Document.ts

import { updateYFragment } from "@getoutline/y-prosemirror";
import removeMarkdown from "@tommoor/remove-markdown";
import invariant from "invariant";
import { compact, find, map, uniq } from "lodash";
import randomstring from "randomstring";
import type { SaveOptions } from "sequelize";
import {
  Transaction,
  Op,
  QueryTypes,
  FindOptions,
  ScopeOptions,
  WhereOptions,
} from "sequelize";
import {
  ForeignKey,
  BelongsTo,
  Column,
  Default,
  PrimaryKey,
  Table,
  BeforeValidate,
  BeforeCreate,
  BeforeUpdate,
  HasMany,
  BeforeSave,
  DefaultScope,
  AfterCreate,
  Scopes,
  DataType,
  Length as SimpleLength,
  IsNumeric,
  IsDate,
} from "sequelize-typescript";
import MarkdownSerializer from "slate-md-serializer";
import isUUID from "validator/lib/isUUID";
import * as Y from "yjs";
import { DateFilter } from "@shared/types";
import getTasks from "@shared/utils/getTasks";
import parseTitle from "@shared/utils/parseTitle";
import unescape from "@shared/utils/unescape";
import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers";
import { DocumentValidation } from "@shared/validations";
import { parser } from "@server/editor";
import slugify from "@server/utils/slugify";
import Backlink from "./Backlink";
import Collection from "./Collection";
import Revision from "./Revision";
import Share from "./Share";
import Star from "./Star";
import Team from "./Team";
import User from "./User";
import View from "./View";
import ParanoidModel from "./base/ParanoidModel";
import Fix from "./decorators/Fix";
import Length from "./validators/Length";

export type SearchResponse = {
  results: {
    ranking: number;
    context: string;
    document: Document;
  }[];
  totalCount: number;
};

type SearchOptions = {
  limit?: number;
  offset?: number;
  collectionId?: string;
  share?: Share;
  dateFilter?: DateFilter;
  collaboratorIds?: string[];
  includeArchived?: boolean;
  includeDrafts?: boolean;
  snippetMinWords?: number;
  snippetMaxWords?: number;
};

const serializer = new MarkdownSerializer();

export const DOCUMENT_VERSION = 2;

@DefaultScope(() => ({
  attributes: {
    exclude: ["state"],
  },
  include: [
    {
      model: User,
      as: "createdBy",
      paranoid: false,
    },
    {
      model: User,
      as: "updatedBy",
      paranoid: false,
    },
  ],
  where: {
    publishedAt: {
      [Op.ne]: null,
    },
  },
}))
@Scopes(() => ({
  withCollectionPermissions: (userId: string, paranoid = true) => {
    if (userId) {
      return {
        include: [
          {
            attributes: ["id", "permission", "sharing", "teamId", "deletedAt"],
            model: Collection.scope({
              method: ["withMembership", userId],
            }),
            as: "collection",
            paranoid,
          },
        ],
      };
    }

    return {
      include: [
        {
          attributes: ["id", "permission", "sharing", "teamId", "deletedAt"],
          model: Collection,
          as: "collection",
          paranoid,
        },
      ],
    };
  },
  withoutState: {
    attributes: {
      exclude: ["state"],
    },
  },
  withCollection: {
    include: [
      {
        model: Collection,
        as: "collection",
      },
    ],
  },
  withState: {
    attributes: {
      // resets to include the state column
      exclude: [],
    },
  },
  withDrafts: {
    include: [
      {
        model: User,
        as: "createdBy",
        paranoid: false,
      },
      {
        model: User,
        as: "updatedBy",
        paranoid: false,
      },
    ],
  },
  withViews: (userId: string) => {
    if (!userId) {
      return {};
    }
    return {
      include: [
        {
          model: View,
          as: "views",
          where: {
            userId,
          },
          required: false,
          separate: true,
        },
      ],
    };
  },
}))
@Table({ tableName: "documents", modelName: "document" })
@Fix
class Document extends ParanoidModel {
  @SimpleLength({
    min: 10,
    max: 10,
    msg: `urlId must be 10 characters`,
  })
  @PrimaryKey
  @Column
  urlId: string;

  @Length({
    max: DocumentValidation.maxTitleLength,
    msg: `Document title must be ${DocumentValidation.maxTitleLength} characters or less`,
  })
  @Column
  title: string;

  @Column(DataType.ARRAY(DataType.STRING))
  previousTitles: string[] = [];

  @IsNumeric
  @Column(DataType.SMALLINT)
  version: number;

  @Column
  template: boolean;

  @Column
  fullWidth: boolean;

  @SimpleLength({
    max: 255,
    msg: `editorVersion must be 255 characters or less`,
  })
  @Column
  editorVersion: string;

  @Column
  emoji: string | null;

  @Column(DataType.TEXT)
  text: string;

  @Length({
    max: DocumentValidation.maxStateLength,
    msg: `Document collaborative state is too large, you must create a new document`,
  })
  @Column(DataType.BLOB)
  state: Uint8Array;

  @Default(false)
  @Column
  isWelcome: boolean;

  @IsNumeric
  @Default(0)
  @Column(DataType.INTEGER)
  revisionCount: number;

  @IsDate
  @Column
  archivedAt: Date | null;

  @IsDate
  @Column
  publishedAt: Date | null;

  @Column(DataType.ARRAY(DataType.UUID))
  collaboratorIds: string[] = [];

  // getters

  get url() {
    if (!this.title) {
      return `/doc/untitled-${this.urlId}`;
    }
    const slugifiedTitle = slugify(this.title);
    return `/doc/${slugifiedTitle}-${this.urlId}`;
  }

  get tasks() {
    return getTasks(this.text || "");
  }

  // hooks

  @BeforeSave
  static async updateTitleInCollectionStructure(
    model: Document,
    { transaction }: SaveOptions<Document>
  ) {
    // templates, drafts, and archived documents don't appear in the structure
    // and so never need to be updated when the title changes
    if (
      model.archivedAt ||
      model.template ||
      !model.publishedAt ||
      !model.changed("title")
    ) {
      return;
    }

    const collection = await Collection.findByPk(model.collectionId, {
      transaction,
      lock: Transaction.LOCK.UPDATE,
    });
    if (!collection) {
      return;
    }

    await collection.updateDocument(model, { transaction });
    model.collection = collection;
  }

  @AfterCreate
  static async addDocumentToCollectionStructure(model: Document) {
    if (model.archivedAt || model.template || !model.publishedAt) {
      return;
    }

    return this.sequelize!.transaction(async (transaction: Transaction) => {
      const collection = await Collection.findByPk(model.collectionId, {
        transaction,
        lock: transaction.LOCK.UPDATE,
      });
      if (!collection) {
        return;
      }

      await collection.addDocumentToStructure(model, 0, { transaction });
      model.collection = collection;
    });
  }

  @BeforeValidate
  static createUrlId(model: Document) {
    return (model.urlId = model.urlId || randomstring.generate(10));
  }

  @BeforeCreate
  static setDocumentVersion(model: Document) {
    if (model.version === undefined) {
      model.version = DOCUMENT_VERSION;
    }

    return this.processUpdate(model);
  }

  @BeforeUpdate
  static processUpdate(model: Document) {
    const { emoji } = parseTitle(model.text);
    // emoji in the title is split out for easier display
    model.emoji = emoji || null;

    // ensure documents have a title
    model.title = model.title || "";

    if (model.previous("title") && model.previous("title") !== model.title) {
      if (!model.previousTitles) {
        model.previousTitles = [];
      }

      model.previousTitles = uniq(
        model.previousTitles.concat(model.previous("title"))
      );
    }

    // add the current user as a collaborator on this doc
    if (!model.collaboratorIds) {
      model.collaboratorIds = [];
    }

    model.collaboratorIds = uniq(
      model.collaboratorIds.concat(model.lastModifiedById)
    );

    // increment revision
    model.revisionCount += 1;
  }

  // associations

  @BelongsTo(() => Document, "parentDocumentId")
  parentDocument: Document | null;

  @ForeignKey(() => Document)
  @Column(DataType.UUID)
  parentDocumentId: string | null;

  @BelongsTo(() => User, "lastModifiedById")
  updatedBy: User;

  @ForeignKey(() => User)
  @Column(DataType.UUID)
  lastModifiedById: string;

  @BelongsTo(() => User, "createdById")
  createdBy: User;

  @ForeignKey(() => User)
  @Column(DataType.UUID)
  createdById: string;

  @BelongsTo(() => Document, "templateId")
  document: Document;

  @ForeignKey(() => Document)
  @Column(DataType.UUID)
  templateId: string;

  @BelongsTo(() => Team, "teamId")
  team: Team;

  @ForeignKey(() => Team)
  @Column(DataType.UUID)
  teamId: string;

  @BelongsTo(() => Collection, "collectionId")
  collection: Collection;

  @ForeignKey(() => Collection)
  @Column(DataType.UUID)
  collectionId: string;

  @HasMany(() => Revision)
  revisions: Revision[];

  @HasMany(() => Backlink)
  backlinks: Backlink[];

  @HasMany(() => Star)
  starred: Star[];

  @HasMany(() => View)
  views: View[];

  static defaultScopeWithUser(userId: string) {
    const collectionScope: Readonly<ScopeOptions> = {
      method: ["withCollectionPermissions", userId],
    };
    const viewScope: Readonly<ScopeOptions> = {
      method: ["withViews", userId],
    };
    return this.scope(["defaultScope", collectionScope, viewScope]);
  }

  static async findByPk(
    id: string,
    options: FindOptions<Document> & {
      userId?: string;
      includeState?: boolean;
    } = {}
  ): Promise<Document | null> {
    // allow default preloading of collection membership if `userId` is passed in find options
    // almost every endpoint needs the collection membership to determine policy permissions.
    const scope = this.scope([
      ...(options.includeState ? [] : ["withoutState"]),
      "withDrafts",
      {
        method: ["withCollectionPermissions", options.userId, options.paranoid],
      },
      {
        method: ["withViews", options.userId],
      },
    ]);

    if (isUUID(id)) {
      return scope.findOne({
        where: {
          id,
        },
        ...options,
      });
    }

    const match = id.match(SLUG_URL_REGEX);
    if (match) {
      return scope.findOne({
        where: {
          urlId: match[1],
        },
        ...options,
      });
    }

    return null;
  }

  static async searchForTeam(
    team: Team,
    query: string,
    options: SearchOptions = {}
  ): Promise<SearchResponse> {
    const wildcardQuery = `${escape(query)}:*`;
    const {
      snippetMinWords = 20,
      snippetMaxWords = 30,
      limit = 15,
      offset = 0,
    } = options;

    // restrict to specific collection if provided
    // enables search in private collections if specified
    let collectionIds;
    if (options.collectionId) {
      collectionIds = [options.collectionId];
    } else {
      collectionIds = await team.collectionIds();
    }

    // short circuit if no relevant collections
    if (!collectionIds.length) {
      return {
        results: [],
        totalCount: 0,
      };
    }

    // restrict to documents in the tree of a shared document when one is provided
    let documentIds;

    if (options.share?.includeChildDocuments) {
      const sharedDocument = await options.share.$get("document");
      invariant(sharedDocument, "Cannot find document for share");

      const childDocumentIds = await sharedDocument.getChildDocumentIds({
        archivedAt: {
          [Op.is]: null,
        },
      });
      documentIds = [sharedDocument.id, ...childDocumentIds];
    }

    const documentClause = documentIds ? `"id" IN(:documentIds) AND` : "";

    // Build the SQL query to get result documentIds, ranking, and search term context
    const whereClause = `
  "searchVector" @@ to_tsquery('english', :query) AND
    "teamId" = :teamId AND
    "collectionId" IN(:collectionIds) AND
    ${documentClause}
    "deletedAt" IS NULL AND
    "publishedAt" IS NOT NULL
  `;
    const selectSql = `
    SELECT
      id,
      ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
      ts_headline('english', "text", to_tsquery('english', :query), :headlineOptions) as "searchContext"
    FROM documents
    WHERE ${whereClause}
    ORDER BY
      "searchRanking" DESC,
      "updatedAt" DESC
    LIMIT :limit
    OFFSET :offset;
  `;
    const countSql = `
    SELECT COUNT(id)
    FROM documents
    WHERE ${whereClause}
  `;
    const queryReplacements = {
      teamId: team.id,
      query: wildcardQuery,
      collectionIds,
      documentIds,
      headlineOptions: `MaxFragments=1, MinWords=${snippetMinWords}, MaxWords=${snippetMaxWords}`,
    };
    const resultsQuery = this.sequelize!.query(selectSql, {
      type: QueryTypes.SELECT,
      replacements: { ...queryReplacements, limit, offset },
    });
    const countQuery = this.sequelize!.query(countSql, {
      type: QueryTypes.SELECT,
      replacements: queryReplacements,
    });
    const [results, [{ count }]]: [any, any] = await Promise.all([
      resultsQuery,
      countQuery,
    ]);

    // Final query to get associated document data
    const documents = await this.findAll({
      where: {
        id: map(results, "id"),
        teamId: team.id,
      },
      include: [
        {
          model: Collection,
          as: "collection",
        },
      ],
    });

    return {
      results: map(results, (result: any) => ({
        ranking: result.searchRanking,
        context: removeMarkdown(unescape(result.searchContext), {
          stripHTML: false,
        }),
        document: find(documents, {
          id: result.id,
        }) as Document,
      })),
      totalCount: count,
    };
  }

  static async searchForUser(
    user: User,
    query: string,
    options: SearchOptions = {}
  ): Promise<SearchResponse> {
    const {
      snippetMinWords = 20,
      snippetMaxWords = 30,
      limit = 15,
      offset = 0,
    } = options;
    const wildcardQuery = `${escape(query)}:*`;

    // Ensure we're filtering by the users accessible collections. If
    // collectionId is passed as an option it is assumed that the authorization
    // has already been done in the router
    let collectionIds;

    if (options.collectionId) {
      collectionIds = [options.collectionId];
    } else {
      collectionIds = await user.collectionIds();
    }

    // If the user has access to no collections then shortcircuit the rest of this
    if (!collectionIds.length) {
      return {
        results: [],
        totalCount: 0,
      };
    }

    let dateFilter;

    if (options.dateFilter) {
      dateFilter = `1 ${options.dateFilter}`;
    }

    // Build the SQL query to get documentIds, ranking, and search term context
    const whereClause = `
  "searchVector" @@ to_tsquery('english', :query) AND
    "teamId" = :teamId AND
    "collectionId" IN(:collectionIds) AND
    ${
      options.dateFilter ? '"updatedAt" > now() - interval :dateFilter AND' : ""
    }
    ${
      options.collaboratorIds
        ? '"collaboratorIds" @> ARRAY[:collaboratorIds]::uuid[] AND'
        : ""
    }
    ${options.includeArchived ? "" : '"archivedAt" IS NULL AND'}
    "deletedAt" IS NULL AND
    ${
      options.includeDrafts
        ? '("publishedAt" IS NOT NULL OR "createdById" = :userId)'
        : '"publishedAt" IS NOT NULL'
    }
  `;
    const selectSql = `
  SELECT
    id,
    ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
    ts_headline('english', "text", to_tsquery('english', :query), :headlineOptions) as "searchContext"
  FROM documents
  WHERE ${whereClause}
  ORDER BY
    "searchRanking" DESC,
    "updatedAt" DESC
  LIMIT :limit
  OFFSET :offset;
  `;
    const countSql = `
    SELECT COUNT(id)
    FROM documents
    WHERE ${whereClause}
  `;
    const queryReplacements = {
      teamId: user.teamId,
      userId: user.id,
      collaboratorIds: options.collaboratorIds,
      query: wildcardQuery,
      collectionIds,
      dateFilter,
      headlineOptions: `MaxFragments=1, MinWords=${snippetMinWords}, MaxWords=${snippetMaxWords}`,
    };
    const resultsQuery = this.sequelize!.query(selectSql, {
      type: QueryTypes.SELECT,
      replacements: { ...queryReplacements, limit, offset },
    });
    const countQuery = this.sequelize!.query(countSql, {
      type: QueryTypes.SELECT,
      replacements: queryReplacements,
    });
    const [results, [{ count }]]: [any, any] = await Promise.all([
      resultsQuery,
      countQuery,
    ]);

    // Final query to get associated document data
    const documents = await this.scope([
      "withoutState",
      "withDrafts",
      {
        method: ["withViews", user.id],
      },
      {
        method: ["withCollectionPermissions", user.id],
      },
    ]).findAll({
      where: {
        teamId: user.teamId,
        id: map(results, "id"),
      },
    });

    return {
      results: map(results, (result: any) => ({
        ranking: result.searchRanking,
        context: removeMarkdown(unescape(result.searchContext), {
          stripHTML: false,
        }),
        document: find(documents, {
          id: result.id,
        }) as Document,
      })),
      totalCount: count,
    };
  }

  // instance methods

  updateFromMarkdown = (text: string, append = false) => {
    this.text = append ? this.text + text : text;

    if (this.state) {
      const ydoc = new Y.Doc();
      Y.applyUpdate(ydoc, this.state);
      const type = ydoc.get("default", Y.XmlFragment) as Y.XmlFragment;
      const doc = parser.parse(this.text);

      if (!type.doc) {
        throw new Error("type.doc not found");
      }

      // apply new document to existing ydoc
      updateYFragment(type.doc, type, doc, new Map());

      const state = Y.encodeStateAsUpdate(ydoc);
      this.state = Buffer.from(state);
      this.changed("state", true);
    }
  };

  toMarkdown = () => {
    const text = unescape(this.text);

    if (this.version) {
      return `# ${this.title}\n\n${text}`;
    }

    return text;
  };

  migrateVersion = () => {
    let migrated = false;

    // migrate from document version 0 -> 1
    if (!this.version) {
      // removing the title from the document text attribute
      this.text = this.text.replace(/^#\s(.*)\n/, "");
      this.version = 1;
      migrated = true;
    }

    // migrate from document version 1 -> 2
    if (this.version === 1) {
      const nodes = serializer.deserialize(this.text);
      this.text = serializer.serialize(nodes, {
        version: 2,
      });
      this.version = 2;
      migrated = true;
    }

    if (migrated) {
      return this.save({
        silent: true,
        hooks: false,
      });
    }

    return undefined;
  };

  /**
   * Calculate all of the document ids that are children of this document by
   * iterating through parentDocumentId references in the most efficient way.
   *
   * @param options FindOptions
   * @returns A promise that resolves to a list of document ids
   */
  getChildDocumentIds = async (
    where?: Omit<WhereOptions<Document>, "parentDocumentId">,
    options?: FindOptions<Document>
  ): Promise<string[]> => {
    const getChildDocumentIds = async (
      ...parentDocumentId: string[]
    ): Promise<string[]> => {
      const childDocuments = await (this
        .constructor as typeof Document).findAll({
        attributes: ["id"],
        where: {
          parentDocumentId,
          ...where,
        },
        ...options,
      });

      const childDocumentIds = childDocuments.map((doc) => doc.id);

      if (childDocumentIds.length > 0) {
        return [
          ...childDocumentIds,
          ...(await getChildDocumentIds(...childDocumentIds)),
        ];
      }

      return childDocumentIds;
    };

    return getChildDocumentIds(this.id);
  };

  archiveWithChildren = async (
    userId: string,
    options?: FindOptions<Document>
  ) => {
    const archivedAt = new Date();

    // Helper to archive all child documents for a document
    const archiveChildren = async (parentDocumentId: string) => {
      const childDocuments = await (this
        .constructor as typeof Document).findAll({
        where: {
          parentDocumentId,
        },
      });
      childDocuments.forEach(async (child) => {
        await archiveChildren(child.id);
        child.archivedAt = archivedAt;
        child.lastModifiedById = userId;
        await child.save(options);
      });
    };

    await archiveChildren(this.id);
    this.archivedAt = archivedAt;
    this.lastModifiedById = userId;
    return this.save(options);
  };

  publish = async (userId: string, { transaction }: SaveOptions<Document>) => {
    // If the document is already published then calling publish should act like
    // a regular save
    if (this.publishedAt) {
      return this.save({ transaction });
    }

    if (!this.template) {
      const collection = await Collection.findByPk(this.collectionId, {
        transaction,
        lock: Transaction.LOCK.UPDATE,
      });

      if (collection) {
        await collection.addDocumentToStructure(this, 0, { transaction });
        this.collection = collection;
      }
    }

    this.lastModifiedById = userId;
    this.publishedAt = new Date();
    return this.save({ transaction });
  };

  unpublish = async (userId: string) => {
    // If the document is already a draft then calling unpublish should act like
    // a regular save
    if (!this.publishedAt) {
      return this.save();
    }

    await this.sequelize.transaction(async (transaction: Transaction) => {
      const collection = await Collection.findByPk(this.collectionId, {
        transaction,
        lock: transaction.LOCK.UPDATE,
      });

      if (collection) {
        await collection.removeDocumentInStructure(this, { transaction });
        this.collection = collection;
      }
    });

    // unpublishing a document converts the ownership to yourself, so that it
    // will appear in your drafts rather than the original creators
    this.createdById = userId;
    this.lastModifiedById = userId;
    this.publishedAt = null;
    return this.save();
  };

  // Moves a document from being visible to the team within a collection
  // to the archived area, where it can be subsequently restored.
  archive = async (userId: string) => {
    await this.sequelize.transaction(async (transaction: Transaction) => {
      const collection = await Collection.findByPk(this.collectionId, {
        transaction,
        lock: transaction.LOCK.UPDATE,
      });

      if (collection) {
        await collection.removeDocumentInStructure(this, { transaction });
        this.collection = collection;
      }
    });

    await this.archiveWithChildren(userId);
    return this;
  };

  // Restore an archived document back to being visible to the team
  unarchive = async (userId: string) => {
    await this.sequelize.transaction(async (transaction: Transaction) => {
      const collection = await Collection.findByPk(this.collectionId, {
        transaction,
        lock: transaction.LOCK.UPDATE,
      });

      // check to see if the documents parent hasn't been archived also
      // If it has then restore the document to the collection root.
      if (this.parentDocumentId) {
        const parent = await (this.constructor as typeof Document).findOne({
          where: {
            id: this.parentDocumentId,
            archivedAt: {
              [Op.is]: null,
            },
          },
        });
        if (!parent) {
          this.parentDocumentId = null;
        }
      }

      if (!this.template && collection) {
        await collection.addDocumentToStructure(this, undefined, {
          transaction,
        });
        this.collection = collection;
      }
    });

    if (this.deletedAt) {
      await this.restore();
    }

    this.archivedAt = null;
    this.lastModifiedById = userId;
    await this.save();
    return this;
  };

  // Delete a document, archived or otherwise.
  delete = (userId: string) => {
    return this.sequelize.transaction(async (transaction: Transaction) => {
      if (!this.archivedAt && !this.template) {
        // delete any children and remove from the document structure
        const collection = await Collection.findByPk(this.collectionId, {
          transaction,
          lock: transaction.LOCK.UPDATE,
        });
        await collection?.deleteDocument(this, { transaction });
      } else {
        await this.destroy({
          transaction,
        });
      }

      await Revision.destroy({
        where: {
          documentId: this.id,
        },
        transaction,
      });
      await this.update(
        {
          lastModifiedById: userId,
        },
        {
          transaction,
        }
      );
      return this;
    });
  };

  getTimestamp = () => {
    return Math.round(new Date(this.updatedAt).getTime() / 1000);
  };

  getSummary = () => {
    const plain = removeMarkdown(unescape(this.text), {
      stripHTML: false,
    });
    const lines = compact(plain.split("\n"));
    const notEmpty = lines.length >= 1;

    if (this.version) {
      return notEmpty ? lines[0] : "";
    }

    return notEmpty ? lines[1] : "";
  };

  toJSON = () => {
    // Warning: only use for new documents as order of children is
    // handled in the collection's documentStructure
    return {
      id: this.id,
      title: this.title,
      url: this.url,
      children: [],
    };
  };
}

function escape(query: string): string {
  // replace "\" with escaped "\\" because sequelize.escape doesn't do it
  // https://github.com/sequelize/sequelize/issues/2950
  return Document.sequelize!.escape(query).replace(/\\/g, "\\\\");
}

export default Document;