Skip to content

Commit

Permalink
馃洜 Fix drive migrator (#2788)
Browse files Browse the repository at this point in the history 
* Fix migrator

* Fix tests
  • Loading branch information
@RomaricMourgues
RomaricMourgues authored and Labels Bot committed Mar 31, 2023
1 parent a042307 commit 6783f3e
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 37 deletions.
28 changes: 26 additions & 2 deletions twake/backend/node/src/cli/cmds/migration_cmds/drive.ts
View file
Expand Up @@ -35,15 +35,39 @@ const command: yargs.CommandModule<unknown, unknown> = {
type: "string",
description: "Migrate only this company ID",
},
ignoreThumbnails: {
default: false,
type: "boolean",
description: "Ignore thumbnails",
},
fromItem: {
default: null,
type: "string",
description: "Start migration from this item ID",
},
fromWorkspace: {
default: null,
type: "string",
description: "Start migration from this workspace ID",
},
},
handler: async argv => {
const from = argv.from as string | null;
const fromCompany = argv.from as string | null;
const onlyCompany = argv.onlyCompany as string | null;
const ignoreThumbnails = argv.ignoreThumbnails as boolean;
const fromItem = argv.fromItem as string | null;
const fromWorkspace = argv.fromWorkspace as string | null;

const spinner = ora({ text: "Migrating php drive - " }).start();
const platform = await twake.run(services);
await globalResolver.doInit(platform);
const migrator = new DriveMigrator(platform, { fromCompany: from, onlyCompany });
const migrator = new DriveMigrator(platform, {
fromCompany,
onlyCompany,
ignoreThumbnails,
fromItem,
fromWorkspace,
});

await migrator.run();

Expand Down
58 changes: 42 additions & 16 deletions twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/drive-migrator-service.ts
View file
Expand Up @@ -17,6 +17,9 @@ import mimes from "../../../../utils/mime";
import WorkspaceUser from "../../../../services/workspaces/entities/workspace_user";
import CompanyUser from "src/services/user/entities/company_user";

let didPassFromWorkspace = false;
let didPassFromItem = false;

interface CompanyExecutionContext extends ExecutionContext {
company: {
id: string;
Expand All @@ -31,13 +34,19 @@ class DriveMigrator {
private phpDriveService: PhpDriveFileService;
private nodeRepository: Repository<DriveFile>;
private options: {
ignoreThumbnails?: boolean;
fromItem?: string;
fromWorkspace?: string;
fromCompany?: string;
onlyCompany?: string;
};

constructor(
readonly _platform: TwakePlatform,
options?: {
ignoreThumbnails?: boolean;
fromItem?: string;
fromWorkspace?: string;
fromCompany?: string;
onlyCompany?: string;
},
Expand Down Expand Up @@ -66,27 +75,29 @@ class DriveMigrator {
},
};

let didPassFromCompany = false;
do {
const companyListResult = await globalResolver.services.companies.getCompanies(page);
page = companyListResult.nextPage as Pagination;

let didPassFromCompany = false;

for (const company of companyListResult.getEntities()) {
const companies = companyListResult.getEntities();
for (let i = 0; i < companies.length; i++) {
const company = companies[i];
if (this.options.onlyCompany && this.options.onlyCompany !== company.id) continue;
if (this.options.fromCompany && this.options.fromCompany === company.id) {
didPassFromCompany = true;
}
if (this.options.fromCompany && !didPassFromCompany) continue;

console.log(`Migrating company ${company.id} (next will be ${companies[i + 1]?.id})`);
await this.migrateCompany(company, {
...context,
company: { id: company.id },
});
}

console.log("Loop over companies...", page.page_token);
} while (page.page_token);

console.log("Migration done");
};

/**
Expand All @@ -98,8 +109,6 @@ class DriveMigrator {
company: Company,
context: CompanyExecutionContext,
): Promise<void> => {
logger.info(`Migrating company ${company.id}`);

const companyAdminOrOwnerId = await this.getCompanyOwnerOrAdminId(company.id, context);
if (!companyAdminOrOwnerId) {
return;
Expand All @@ -108,15 +117,25 @@ class DriveMigrator {
if (!workspaceList || workspaceList.length === 0) {
return;
}
for (let i = 0; i < workspaceList.length; i++) {
const workspace = workspaceList[i];
if (this.options.fromWorkspace && this.options.fromWorkspace === workspace.id) {
didPassFromWorkspace = true;
}
if (this.options.fromWorkspace && !didPassFromWorkspace) continue;

for (const workspace of workspaceList) {
const wsContext = {
...context,
workspace_id: workspace.id,
user: { id: companyAdminOrOwnerId, server_request: true },
};
const access = await this.getWorkspaceAccess(workspace, company, wsContext);

console.log(
`Migrating workspace ${workspace.id} root folder (next will be ${
workspaceList[i + 1]?.id
})`,
);
await this.migrateWorkspace(workspace, access, wsContext);
}
};
Expand All @@ -133,9 +152,6 @@ class DriveMigrator {
): Promise<void> => {
let page: Pagination = { limitStr: "100" };

console.debug(`Migrating workspace ${workspace.id} of company ${context.company.id}`);
logger.info(`Migrating workspace ${workspace.id} root folder`);

const workspaceFolder = await this.createWorkspaceFolder(workspace, access, context);
// Migrate the root folder.
do {
Expand All @@ -147,7 +163,19 @@ class DriveMigrator {
);
page = phpDriveFiles.nextPage as Pagination;

for (const phpDriveFile of phpDriveFiles.getEntities()) {
const driveFiles = phpDriveFiles.getEntities();
for (let i = 0; i < driveFiles.length; i++) {
const phpDriveFile = driveFiles[i];
if (this.options.fromItem && this.options.fromItem === phpDriveFile.id) {
didPassFromItem = true;
}
if (this.options.fromItem && !didPassFromItem) continue;

logger.info(
`Migrating php drive item ${phpDriveFile.id} - parent: ${
workspaceFolder.id ?? "root"
} (next php file will be ${driveFiles[i + 1]?.id})`,
);
await this.migrateDriveFile(phpDriveFile, workspaceFolder.id, access, context);
}
} while (page.page_token);
Expand Down Expand Up @@ -177,8 +205,6 @@ class DriveMigrator {
access: AccessInformation,
context: WorkspaceExecutionContext,
): Promise<void> => {
logger.info(`Migrating php drive item ${item.id} - parent: ${parentId ?? "root"}`);

try {
await new Promise(async (resolve, reject) => {
const migrationRecord = await this.phpDriveService.getMigrationRecord(
Expand All @@ -188,7 +214,7 @@ class DriveMigrator {

const newDriveItem = getDefaultDriveItem(
{
name: item.name || item.id,
name: item.name || item.added.toString()?.split(" ")[0] || "Untitled",
extension: item.extension,
added: item.added.toString(),
content_keywords:
Expand Down Expand Up @@ -246,7 +272,6 @@ class DriveMigrator {
migrationRecord.company_id === context.company.id
) {
logger.info(`item is already migrated - ${item.id} - skipping`);
console.log(`item is already migrated - ${item.id} - skipping`);
resolve(true);
return;
}
Expand Down Expand Up @@ -298,6 +323,7 @@ class DriveMigrator {
chunkNumber: 1,
totalChunks: 1,
type: mime,
ignoreThumbnails: this.options.ignoreThumbnails || false,
},
context,
);
Expand Down
40 changes: 21 additions & 19 deletions twake/backend/node/src/services/files/services/index.ts
View file
Expand Up @@ -143,29 +143,31 @@ export class FileServiceImpl {
entity.metadata.thumbnails_status = "waiting";
await this.repository.save(entity, context);

try {
await gr.platformServices.messageQueue.publish<PreviewMessageQueueRequest>(
"services:preview",
{
data: { document, output },
},
);

if (options.waitForThumbnail) {
entity = await gr.services.files.getFile(
if (!options?.ignoreThumbnails) {
try {
await gr.platformServices.messageQueue.publish<PreviewMessageQueueRequest>(
"services:preview",
{
id: entity.id,
company_id: context.company.id,
data: { document, output },
},
context,
{ waitForThumbnail: true },
);
}
} catch (err) {
entity.metadata.thumbnails_status = "error";
await this.repository.save(entity, context);

logger.warn({ err }, "Previewing - Error while sending ");
if (options.waitForThumbnail) {
entity = await gr.services.files.getFile(
{
id: entity.id,
company_id: context.company.id,
},
context,
{ waitForThumbnail: true },
);
}
} catch (err) {
entity.metadata.thumbnails_status = "error";
await this.repository.save(entity, context);

logger.warn({ err }, "Previewing - Error while sending ");
}
}
}

Expand Down
1 change: 1 addition & 0 deletions twake/backend/node/src/services/files/types.ts
View file
Expand Up @@ -5,4 +5,5 @@ export type UploadOptions = {
totalChunks: number;
chunkNumber: number;
waitForThumbnail: boolean;
ignoreThumbnails?: boolean;
};

0 comments on commit 6783f3e

Please sign in to comment.