diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/drive-migrator-service.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/drive-migrator-service.ts index 819062484..2c28b2a13 100644 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/drive-migrator-service.ts +++ b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/drive-migrator-service.ts @@ -2,8 +2,9 @@ import { logger } from "../../../../core/platform/framework"; import { ExecutionContext, Pagination } from "../../../../core/platform/framework/api/crud-service"; import { TwakePlatform } from "../../../../core/platform/platform"; import Repository from "../../../../core/platform/services/database/services/orm/repository/repository"; -import { DriveFile } from "../../../../services/documents/entities/drive-file"; +import { DriveFile, AccessInformation } from "../../../../services/documents/entities/drive-file"; import { + generateAccessToken, getDefaultDriveItem, getDefaultDriveItemVersion, } from "../../../../services/documents/utils"; @@ -12,6 +13,9 @@ import Company from "../../../../services/user/entities/company"; import Workspace from "../../../../services/workspaces/entities/workspace"; import { PhpDriveFile } from "./php-drive-file-entity"; import { PhpDriveFileService } from "./php-drive-service"; +import mimes from "../../../../utils/mime"; +import WorkspaceUser from "../../../../services/workspaces/entities/workspace_user"; +import CompanyUser from "src/services/user/entities/company_user"; interface CompanyExecutionContext extends ExecutionContext { company: { @@ -51,8 +55,6 @@ class DriveMigrator { }, }; - console.debug("Starting migration"); - do { const companyListResult = await globalResolver.services.companies.getCompanies(page); page = companyListResult.nextPage as Pagination; @@ -75,12 +77,20 @@ class DriveMigrator { company: Company, context: CompanyExecutionContext, ): Promise => { - console.debug(`Migrating company ${company.id}`); + logger.info(`Migrating company ${company.id}`); + const companyAdminOrOwnerId = await this.getCompanyOwnerOrAdminId(company.id, context); const workspaceList = await globalResolver.services.workspaces.getAllForCompany(company.id); for (const workspace of workspaceList) { - await this.migrateWorkspace(workspace, { ...context, workspace_id: workspace.id }); + const wsContext = { + ...context, + workspace_id: workspace.id, + user: { id: companyAdminOrOwnerId, server_request: true }, + }; + const access = await this.getWorkspaceAccess(workspace, company, wsContext); + + await this.migrateWorkspace(workspace, access, wsContext); } }; @@ -91,11 +101,15 @@ class DriveMigrator { */ private migrateWorkspace = async ( workspace: Workspace, + access: AccessInformation, context: WorkspaceExecutionContext, ): Promise => { let page: Pagination = { limitStr: "100" }; - console.debug(`Migrating workspace ${workspace.id} root folder`); + console.debug(`Migrating workspace ${workspace.id} of company ${context.company.id}`); + logger.info(`Migrating workspace ${workspace.id} root folder`); + + const workspaceFolder = await this.createWorkspaceFolder(workspace, access, context); // Migrate the root folder. do { const phpDriveFiles = await this.phpDriveService.listDirectory( @@ -107,11 +121,11 @@ class DriveMigrator { page = phpDriveFiles.nextPage as Pagination; for (const phpDriveFile of phpDriveFiles.getEntities()) { - await this.migrateDriveFile(phpDriveFile, "", context); + await this.migrateDriveFile(phpDriveFile, workspaceFolder.id, access, context); } } while (page.page_token); - console.debug(`Migrating workspace ${workspace.id} trash`); + logger.info(`Migrating workspace ${workspace.id} trash`); // Migrate the trash. page = { limitStr: "100" }; @@ -120,7 +134,7 @@ class DriveMigrator { page = phpDriveFiles.nextPage as Pagination; for (const phpDriveFile of phpDriveFiles.getEntities()) { - await this.migrateDriveFile(phpDriveFile, "trash", context); + await this.migrateDriveFile(phpDriveFile, "trash", access, context); } } while (page.page_token); }; @@ -133,32 +147,50 @@ class DriveMigrator { private migrateDriveFile = async ( item: PhpDriveFile, parentId: string, + access: AccessInformation, context: WorkspaceExecutionContext, ): Promise => { - logger.info(`Migrating php drive item ${item.id} - parent: ${parentId}`); - console.debug(`Migrating php drive item ${item.id} - parent: ${parentId}`); + logger.info(`Migrating php drive item ${item.id} - parent: ${parentId ?? "root"}`); try { + const migrationRecord = await this.phpDriveService.getMigrationRecord( + item.id, + context.company.id, + ); + const newDriveItem = getDefaultDriveItem( { - name: item.name, + name: item.name || item.id, extension: item.extension, added: item.added.toString(), - content_keywords: item.content_keywords, - creator: item.creator, + content_keywords: + item.content_keywords && item.content_keywords.length + ? item.content_keywords.join(",") + : "", + creator: item.creator || context.user.id, is_directory: item.isdirectory, is_in_trash: item.isintrash, description: item.description, - tags: item.tags, + tags: item.tags || [], parent_id: parentId, company_id: context.company.id, + access_info: access, }, context, ); - await this.nodeRepository.save(newDriveItem); + if (migrationRecord && migrationRecord.company_id === context.company.id) { + console.debug(`${item.id} is already migrated`); + } else { + await this.nodeRepository.save(newDriveItem); + } if (item.isdirectory) { + const newParentId = + migrationRecord && migrationRecord.company_id === context.company.id + ? migrationRecord.new_id + : newDriveItem.id; + let page: Pagination = { limitStr: "100" }; do { @@ -171,7 +203,7 @@ class DriveMigrator { for (const child of directoryChildren.getEntities()) { try { - await this.migrateDriveFile(child, newDriveItem.id, context); + await this.migrateDriveFile(child, newParentId, access, context); } catch (error) { logger.error(`Failed to migrate drive item ${child.id}`); console.error(`Failed to migrate drive item ${child.id}`); @@ -180,11 +212,20 @@ class DriveMigrator { } while (page.page_token); } else { let versionPage: Pagination = { limitStr: "100" }; - if ((item.hidden_data as any)?.migrated) { - logger.info(`item is already migrated - ${item.id} - skipping`); + if ( + migrationRecord && + migrationRecord.item_id === item.id && + migrationRecord.company_id === context.company.id + ) { logger.info(`item is already migrated - ${item.id} - skipping`); + console.log(`item is already migrated - ${item.id} - skipping`); + return; } + const mime = mimes[item.extension]; + + let createdVersions = 0; + do { const itemVersions = await this.phpDriveService.listItemVersions( versionPage, @@ -194,11 +235,10 @@ class DriveMigrator { versionPage = itemVersions.nextPage as Pagination; for (const version of itemVersions.getEntities()) { - console.debug(`Migrating version ${version.id}`); try { const newVersion = getDefaultDriveItemVersion( { - creator_id: version.creator_id, + creator_id: version.creator_id || context.user.id, data: version.data, date_added: +version.date_added, drive_item_id: newDriveItem.id, @@ -215,18 +255,23 @@ class DriveMigrator { const file = await this.phpDriveService.migrate( version.file_id, item.workspace_id, + version.id, { filename: version.filename, - userId: version.creator_id, + userId: version.creator_id || context.user.id, totalSize: version.file_size, waitForThumbnail: true, chunkNumber: 1, totalChunks: 1, - type: undefined, + type: mime, }, context, ); + if (!file) { + throw Error("cannot download file version"); + } + newVersion.file_metadata = { external_id: file.id, mime: file.metadata.mime, @@ -239,28 +284,252 @@ class DriveMigrator { newVersion, context, ); + + createdVersions++; } catch (error) { logger.error(`Failed to migrate version ${version.id} for drive item ${item.id}`); console.error(`Failed to migrate version ${version.id} for drive item ${item.id}`); - throw Error(error); } } } while (versionPage.page_token); - } - if (!(item.hidden_data as any)?.migrated) { - item.hidden_data = { - ...((item.hidden_data as any) || {}), - migrated: true, - }; + if (createdVersions === 0) { + await this.nodeRepository.remove(newDriveItem); + return; + } + } - await this.phpDriveService.save(item); + if (!migrationRecord) { + await this.phpDriveService.markAsMigrated(item.id, newDriveItem.id, context.company.id); } } catch (error) { - logger.error(`Failed to migrate Drive item ${item.id}`, error); + logger.error( + `Failed to migrate Drive item ${item.id} / workspace ${item.workspace_id} / company_id: ${context.company.id}`, + error, + ); console.error(`Failed to migrate Drive item ${item.id}`, error); } }; + + /** + * Fetches the first found company owner or admin identifier. + * + * @param {string} companyId - the companyId + * @param {ExecutionContext} context - the execution context + * @returns {Promise} + */ + private getCompanyOwnerOrAdminId = async ( + companyId: string, + context: ExecutionContext, + ): Promise => { + let pagination: Pagination = { limitStr: "100" }; + let companyOwnerOrAdminId = null; + + do { + const companyUsers = await globalResolver.services.companies.companyUserRepository.find( + { group_id: companyId }, + { pagination }, + context, + ); + + pagination = companyUsers.nextPage as Pagination; + + const companyAdminOrOwner = companyUsers + .getEntities() + .find(({ role }) => ["admin", "owner"].includes(role)); + + if (companyAdminOrOwner) { + companyOwnerOrAdminId = companyAdminOrOwner.id; + } + } while (pagination && !companyOwnerOrAdminId); + + return companyOwnerOrAdminId; + }; + + /** + * Compute the Access Information for the workspace folder to be created. + * + * @param {Workspace} workspace - the target workspace + * @param {Company} company - the target company + * @param {WorkspaceExecutionContext} context - the execution context + * @returns {Promise} + */ + private getWorkspaceAccess = async ( + workspace: Workspace, + company: Company, + context: WorkspaceExecutionContext, + ): Promise => { + const companyUsersCount = await globalResolver.services.companies.getUsersCount(company.id); + const workspaceUsersCount = await globalResolver.services.workspaces.getUsersCount( + workspace.id, + ); + + if (companyUsersCount === workspaceUsersCount) { + return { + entities: [ + { + id: "parent", + type: "folder", + level: "manage", + }, + { + id: company.id, + type: "company", + level: "none", + }, + { + id: context.user?.id, + type: "user", + level: "manage", + }, + ], + public: { + level: "none", + token: generateAccessToken(), + }, + }; + } + + let workspaceUsers: WorkspaceUser[] = []; + let wsUsersPagination: Pagination = { limitStr: "100" }; + + do { + const wsUsersQuery = await globalResolver.services.workspaces.getUsers( + { workspaceId: workspace.id }, + wsUsersPagination, + context, + ); + wsUsersPagination = wsUsersQuery.nextPage as Pagination; + + workspaceUsers = [...workspaceUsers, ...wsUsersQuery.getEntities()]; + } while (wsUsersPagination.page_token); + + if (companyUsersCount < 30 || workspaceUsersCount < 30) { + return { + entities: [ + { + id: "parent", + type: "folder", + level: "none", + }, + { + id: company.id, + type: "company", + level: "none", + }, + { + id: context.user?.id, + type: "user", + level: "manage", + }, + ...workspaceUsers.reduce((acc, curr) => { + acc = [ + ...acc, + { + id: curr.userId, + type: "user", + level: "manage", + }, + ]; + + return acc; + }, []), + ], + public: { + level: "none", + token: generateAccessToken(), + }, + }; + } + + let companyUsers: CompanyUser[] = []; + let companyUsersPaginations: Pagination = { limitStr: "100" }; + do { + const companyUsersQuery = await globalResolver.services.companies.getUsers( + { group_id: company.id }, + companyUsersPaginations, + {}, + context, + ); + companyUsersPaginations = companyUsersQuery.nextPage as Pagination; + companyUsers = [...companyUsers, ...companyUsersQuery.getEntities()]; + } while (companyUsersPaginations.page_token); + return { + entities: [ + { + id: "parent", + type: "folder", + level: "none", + }, + { + id: company.id, + type: "company", + level: "manage", + }, + { + id: context.user?.id, + type: "user", + level: "manage", + }, + ...companyUsers.reduce((acc, curr) => { + if (workspaceUsers.find(({ userId }) => curr.user_id === userId)) { + return acc; + } + + acc = [ + ...acc, + { + id: curr.user_id, + type: "user", + level: "none", + }, + ]; + + return acc; + }, []), + ], + public: { + level: "none", + token: generateAccessToken(), + }, + }; + }; + + /** + * Creates a folder for the workspace to migrate. + * + * @param {Workspace} workspace - the workspace to migrate. + * @param {AccessInformation} access - the access information. + * @param {WorkspaceExecutionContext} context - the execution context + * @returns {Promise} + */ + private createWorkspaceFolder = async ( + workspace: Workspace, + access: AccessInformation, + context: WorkspaceExecutionContext, + ): Promise => { + const workspaceFolder = getDefaultDriveItem( + { + name: workspace.name || workspace.id, + extension: "", + content_keywords: "", + creator: context.user.id, + is_directory: true, + is_in_trash: false, + description: "", + tags: [], + parent_id: "root", + company_id: context.company.id, + access_info: access, + }, + context, + ); + + await this.nodeRepository.save(workspaceFolder); + await this.phpDriveService.markAsMigrated(workspace.id, workspaceFolder.id, context.company.id); + + return workspaceFolder; + }; } export default DriveMigrator; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-entity.ts index 757a37e7e..05b326801 100644 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-entity.ts +++ b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-entity.ts @@ -4,7 +4,7 @@ import { Entity, } from "../../../../core/platform/services/database/services/orm/decorators"; -export const TYPE = "php_drive_files"; +export const TYPE = "drive_file"; @Entity(TYPE, { primaryKey: [["workspace_id"], "parent_id", "id"], @@ -33,7 +33,7 @@ export class PhpDriveFile { attachements: unknown; @Column("content_keywords", "encoded_json") - content_keywords: string; + content_keywords: string[] | null; @Column("creator", "string") creator: string; @@ -53,12 +53,12 @@ export class PhpDriveFile { @Column("last_modified", "string") last_modified: string; - @Column("name", "string") + @Column("name", "encoded_string") name: string; @Column("size", "number") size: number; @Column("tags", "encoded_json") - tags: string[]; + tags: string[] | null; } diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-version-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-version-entity.ts index 3a0ddb511..69ecdd052 100644 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-version-entity.ts +++ b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-version-entity.ts @@ -4,7 +4,7 @@ import { Entity, } from "../../../../core/platform/services/database/services/orm/decorators"; -export const TYPE = "php_drive_file_versions"; +export const TYPE = "drive_file_version"; @Entity(TYPE, { primaryKey: [["file_id"], "id"], @@ -24,7 +24,7 @@ export class PhpDriveFileVersion { creator_id: string; @Type(() => String) - @Column("realname", "string") + @Column("realname", "encoded_string") realname: string; @Type(() => String) @@ -44,7 +44,7 @@ export class PhpDriveFileVersion { date_added: string; @Type(() => String) - @Column("filename", "string") + @Column("filename", "encoded_string") filename: string; @Type(() => String) diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-migration-record-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-migration-record-entity.ts new file mode 100644 index 000000000..0461d6b05 --- /dev/null +++ b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-migration-record-entity.ts @@ -0,0 +1,25 @@ +import { Type } from "class-transformer"; +import { + Column, + Entity, +} from "../../../../core/platform/services/database/services/orm/decorators"; + +export const TYPE = "php_drive_migration_record"; + +@Entity(TYPE, { + primaryKey: [["company_id"], "item_id"], + type: TYPE, +}) +export class phpDriveMigrationRecord { + @Type(() => String) + @Column("item_id", "timeuuid") + item_id: string; + + @Type(() => String) + @Column("company_id", "uuid") + company_id: string; + + @Type(() => String) + @Column("new_id", "string") + new_id: string; +} diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-service.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-service.ts index 59a18096c..9397a0318 100644 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-service.ts +++ b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-service.ts @@ -12,11 +12,14 @@ import { TYPE as DRIVE_FILE_VERSION_TABLE, } from "./php-drive-file-version-entity"; import axios from "axios"; -import { Stream } from "stream"; import { Multipart } from "fastify-multipart"; import { CompanyExecutionContext } from "../../../../services/files/web/types"; import { File } from "../../../../services/files/entities/file"; import { UploadOptions } from "../../../../services/files/types"; +import { + phpDriveMigrationRecord, + TYPE as MIGRATION_RECORD_TABLE, +} from "./php-drive-migration-record-entity"; export interface MigrateOptions extends UploadOptions { userId: string; @@ -28,6 +31,7 @@ export class PhpDriveFileService implements PhpDriveServiceAPI { version: "1"; public repository: Repository; public versionRepository: Repository; + public migrationRepository: Repository; /** * Init the service. @@ -45,6 +49,11 @@ export class PhpDriveFileService implements PhpDriveServiceAPI { PhpDriveFileVersion, ); + this.migrationRepository = await globalResolver.database.getRepository( + MIGRATION_RECORD_TABLE, + phpDriveMigrationRecord, + ); + return this; } @@ -94,10 +103,11 @@ export class PhpDriveFileService implements PhpDriveServiceAPI { ); /** - * Downloads a file from the old drive and uploads it to the new Drive. + * Downloads a file version from the old drive and uploads it to the new Drive. * * @param {string} fileId - the old file id * @param {string} workspaceId - the workspace id + * @param {string} versionId - the version id * @param {MigrateOptions} options - the file upload / migration options. * @param {CompanyExecutionContext} context - the company execution context. * @param {string} public_access_key - the file public access key. @@ -106,19 +116,24 @@ export class PhpDriveFileService implements PhpDriveServiceAPI { migrate = async ( fileId: string, workspaceId: string, + versionId: string, options: MigrateOptions, context: CompanyExecutionContext, public_access_key?: string, ): Promise => { try { - const url = `https://staging-web.twake.app/ajax/drive/download?workspace_id=${workspaceId}&element_id=${fileId}&download=1${ + const url = `https://web.twake.app/ajax/drive/download?workspace_id=${workspaceId}&element_id=${fileId}&version_id=${versionId}&download=1${ public_access_key ? `&public_access_key=${public_access_key}` : "" }`; - const response = await axios.get(url, { + const response = await axios.get(url, { responseType: "stream", }); + if (!response.data) { + throw Error("invalid download response"); + } + const file = { file: response.data, }; @@ -142,4 +157,33 @@ export class PhpDriveFileService implements PhpDriveServiceAPI { * @returns {Promise} */ save = async (item: PhpDriveFile): Promise => await this.repository.save(item); + + /** + * Marks a drive item as migrated. + * + * @param {string} itemId - the drive item. + * @param {string} newId - the new drive item id. + * @param {string} companyId - the company id. + */ + markAsMigrated = async (itemId: string, newId: string, companyId: string): Promise => { + const migrationRecord = new phpDriveMigrationRecord(); + migrationRecord.item_id = itemId; + migrationRecord.new_id = newId; + migrationRecord.company_id = companyId; + + await this.migrationRepository.save(migrationRecord); + }; + + /** + * Fetches the drive item migration record. + * + * @param {string} itemId - the drive item id. + * @param {string} companyId - the company id. + * @returns {Promise} + */ + getMigrationRecord = async ( + itemId: string, + companyId: string, + ): Promise => + await this.migrationRepository.findOne({ item_id: itemId, company_id: companyId }); } diff --git a/twake/backend/node/src/services/documents/services/index.ts b/twake/backend/node/src/services/documents/services/index.ts index c7fd11e21..6dc760b8a 100644 --- a/twake/backend/node/src/services/documents/services/index.ts +++ b/twake/backend/node/src/services/documents/services/index.ts @@ -200,7 +200,7 @@ export class DocumentsService { ): Promise => { try { const driveItem = getDefaultDriveItem(content, context); - let driveItemVersion = getDefaultDriveItemVersion(version, context); + const driveItemVersion = getDefaultDriveItemVersion(version, context); const hasAccess = await checkAccess( driveItem.parent_id, @@ -574,6 +574,7 @@ export class DocumentsService { await this.repository.save(item); this.notifyWebsocket(item.parent_id, context); + await updateItemSize(item.parent_id, this.repository, context); globalResolver.platformServices.messageQueue.publish( "services:documents:process", diff --git a/twake/backend/node/src/services/documents/utils.ts b/twake/backend/node/src/services/documents/utils.ts index 7a42657cf..c622606ff 100644 --- a/twake/backend/node/src/services/documents/utils.ts +++ b/twake/backend/node/src/services/documents/utils.ts @@ -122,7 +122,7 @@ export const getDefaultDriveItemVersion = ( * * @returns {String} - the random access token ( sha1 hex digest ). */ -const generateAccessToken = (): string => { +export const generateAccessToken = (): string => { const randomBytes = crypto.randomBytes(64); return crypto.createHash("sha1").update(randomBytes).digest("hex"); @@ -674,7 +674,7 @@ export const getFileMetadata = async ( company_id: context.company.id, }, context, - { waitForThumbnail: true }, + { ...(context.user.server_request ? {} : { waitForThumbnail: true }) }, ); if (!file) {