Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
馃洜 Fix drive migration (#2784)
* 馃洜 Fix Drive migration

* Move timeout
  • Loading branch information
RomaricMourgues authored and Labels Bot committed Mar 25, 2023
1 parent d44c166 commit 599f397
Showing 1 changed file with 150 additions and 140 deletions.
Expand Up @@ -65,6 +65,8 @@ class DriveMigrator {
company: { id: company.id },
});
}

console.log("Loop over companies...", page.page_token);
} while (page.page_token);
};

Expand Down Expand Up @@ -159,158 +161,166 @@ class DriveMigrator {
logger.info(`Migrating php drive item ${item.id} - parent: ${parentId ?? "root"}`);

try {
const migrationRecord = await this.phpDriveService.getMigrationRecord(
item.id,
context.company.id,
);
await new Promise(async (resolve, reject) => {
const migrationRecord = await this.phpDriveService.getMigrationRecord(
item.id,
context.company.id,
);

const newDriveItem = getDefaultDriveItem(
{
name: item.name || item.id,
extension: item.extension,
added: item.added.toString(),
content_keywords:
item.content_keywords && item.content_keywords.length
? item.content_keywords.join(",")
: "",
creator: item.creator || context.user.id,
is_directory: item.isdirectory,
is_in_trash: item.isintrash,
description: item.description,
tags: item.tags || [],
parent_id: parentId,
company_id: context.company.id,
access_info: access,
},
context,
);
const newDriveItem = getDefaultDriveItem(
{
name: item.name || item.id,
extension: item.extension,
added: item.added.toString(),
content_keywords:
item.content_keywords && item.content_keywords.length
? item.content_keywords.join(",")
: "",
creator: item.creator || context.user.id,
is_directory: item.isdirectory,
is_in_trash: item.isintrash,
description: item.description,
tags: item.tags || [],
parent_id: parentId,
company_id: context.company.id,
access_info: access,
},
context,
);

if (migrationRecord && migrationRecord.company_id === context.company.id) {
console.debug(`${item.id} is already migrated`);
} else {
await this.nodeRepository.save(newDriveItem);
}
if (migrationRecord && migrationRecord.company_id === context.company.id) {
console.debug(`${item.id} is already migrated`);
} else {
await this.nodeRepository.save(newDriveItem);
}

if (item.isdirectory) {
const newParentId =
migrationRecord && migrationRecord.company_id === context.company.id
? migrationRecord.new_id
: newDriveItem.id;

let page: Pagination = { limitStr: "100" };

do {
const directoryChildren = await this.phpDriveService.listDirectory(
page,
item.id,
context.workspace_id,
);
page = directoryChildren.nextPage as Pagination;

for (const child of directoryChildren.getEntities()) {
try {
await this.migrateDriveFile(child, newParentId, access, context);
} catch (error) {
logger.error(`Failed to migrate drive item ${child.id}`);
console.error(`Failed to migrate drive item ${child.id}`);
if (item.isdirectory) {
const newParentId =
migrationRecord && migrationRecord.company_id === context.company.id
? migrationRecord.new_id
: newDriveItem.id;

let page: Pagination = { limitStr: "100" };

do {
const directoryChildren = await this.phpDriveService.listDirectory(
page,
item.id,
context.workspace_id,
);
page = directoryChildren.nextPage as Pagination;

for (const child of directoryChildren.getEntities()) {
try {
await this.migrateDriveFile(child, newParentId, access, context);
} catch (error) {
logger.error(`Failed to migrate drive item ${child.id}`);
console.error(`Failed to migrate drive item ${child.id}`);
}
}
} while (page.page_token);
} else {
let versionPage: Pagination = { limitStr: "100" };
if (
migrationRecord &&
migrationRecord.item_id === item.id &&
migrationRecord.company_id === context.company.id
) {
logger.info(`item is already migrated - ${item.id} - skipping`);
console.log(`item is already migrated - ${item.id} - skipping`);
return;
}
} while (page.page_token);
} else {
let versionPage: Pagination = { limitStr: "100" };
if (
migrationRecord &&
migrationRecord.item_id === item.id &&
migrationRecord.company_id === context.company.id
) {
logger.info(`item is already migrated - ${item.id} - skipping`);
console.log(`item is already migrated - ${item.id} - skipping`);
return;
}

const mime = mimes[item.extension];

let createdVersions = 0;

do {
const itemVersions = await this.phpDriveService.listItemVersions(
versionPage,
item.id,
context,
);
versionPage = itemVersions.nextPage as Pagination;

for (const version of itemVersions.getEntities()) {
try {
const newVersion = getDefaultDriveItemVersion(
{
creator_id: version.creator_id || context.user.id,
data: version.data,
date_added: +version.date_added,
drive_item_id: newDriveItem.id,
file_size: version.file_size,
filename: version.filename,
key: version.key,
provider: version.provider,
realname: version.realname,
mode: version.mode,
},
context,
);

logger.info(
`Migrating version ${version.id} of item ${item.id}... (downloading then uploading...)`,
);
const file = await this.phpDriveService.migrate(
version.file_id,
item.workspace_id,
version.id,
{
filename: version.filename,
userId: version.creator_id || context.user.id,
totalSize: version.file_size,
waitForThumbnail: true,
chunkNumber: 1,
totalChunks: 1,
type: mime,
},
context,
);

if (!file) {
throw Error("cannot download file version");
const mime = mimes[item.extension];

let createdVersions = 0;

const timeout = setTimeout(() => reject("Timeout"), 60000);

do {
const itemVersions = await this.phpDriveService.listItemVersions(
versionPage,
item.id,
context,
);
versionPage = itemVersions.nextPage as Pagination;

for (const version of itemVersions.getEntities()) {
try {
const newVersion = getDefaultDriveItemVersion(
{
creator_id: version.creator_id || context.user.id,
data: version.data,
date_added: +version.date_added,
drive_item_id: newDriveItem.id,
file_size: version.file_size,
filename: version.filename,
key: version.key,
provider: version.provider,
realname: version.realname,
mode: version.mode,
},
context,
);

logger.info(
`Migrating version ${version.id} of item ${item.id}... (downloading then uploading...)`,
);
const file = await this.phpDriveService.migrate(
version.file_id,
item.workspace_id,
version.id,
{
filename: version.filename,
userId: version.creator_id || context.user.id,
totalSize: version.file_size,
waitForThumbnail: true,
chunkNumber: 1,
totalChunks: 1,
type: mime,
},
context,
);

if (!file) {
throw Error("cannot download file version");
}

newVersion.file_metadata = {
external_id: file.id,
mime: file.metadata.mime,
name: file.metadata.name || version.filename,
size: file.upload_data.size || version.file_size,
};

await globalResolver.services.documents.documents.createVersion(
newDriveItem.id,
newVersion,
context,
);

createdVersions++;
} catch (error) {
logger.error(`Failed to migrate version ${version.id} for drive item ${item.id}`);
console.error(`Failed to migrate version ${version.id} for drive item ${item.id}`);
}

newVersion.file_metadata = {
external_id: file.id,
mime: file.metadata.mime,
name: file.metadata.name || version.filename,
size: file.upload_data.size || version.file_size,
};

await globalResolver.services.documents.documents.createVersion(
newDriveItem.id,
newVersion,
context,
);

createdVersions++;
} catch (error) {
logger.error(`Failed to migrate version ${version.id} for drive item ${item.id}`);
console.error(`Failed to migrate version ${version.id} for drive item ${item.id}`);
}
} while (versionPage.page_token);

clearTimeout(timeout);

if (createdVersions === 0) {
await this.nodeRepository.remove(newDriveItem);
return;
}
} while (versionPage.page_token);
}

if (createdVersions === 0) {
await this.nodeRepository.remove(newDriveItem);
return;
if (!migrationRecord) {
await this.phpDriveService.markAsMigrated(item.id, newDriveItem.id, context.company.id);
}
}

if (!migrationRecord) {
await this.phpDriveService.markAsMigrated(item.id, newDriveItem.id, context.company.id);
}
resolve(true);
});
} catch (error) {
logger.error(
`Failed to migrate Drive item ${item.id} / workspace ${item.workspace_id} / company_id: ${context.company.id}`,
Expand Down

0 comments on commit 599f397

Please sign in to comment.