From 1638c6917a11bc8b906d91ae4f96c85746b2c55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 7 Mar 2024 17:50:03 +0000 Subject: [PATCH] feat: add `blobs-migrate` recipe (#6418) * feat: add `blobs-migrate` recipe * chore: update `@netlify/blobs` * refactor: check for delete errors * chore: fix test * chore: simplify test * chore: update snapshot * chore: pin version --- package-lock.json | 250 ++++++++++-------- package.json | 2 +- src/commands/recipes/recipes.ts | 17 +- src/lib/blobs/blobs.ts | 2 +- src/lib/edge-functions/editor-helper.ts | 2 +- src/recipes/blobs-migrate/index.ts | 107 ++++++++ .../integration/commands/blobs/blobs.test.ts | 10 +- .../__snapshots__/recipes.test.js.snap | 15 +- 8 files changed, 275 insertions(+), 130 deletions(-) create mode 100644 src/recipes/blobs-migrate/index.ts diff --git a/package-lock.json b/package-lock.json index 8420582f087..9e6567a32ea 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@bugsnag/js": "7.20.2", "@fastify/static": "6.10.2", - "@netlify/blobs": "6.5.0", + "@netlify/blobs": "7.0.0", "@netlify/build": "29.36.1", "@netlify/build-info": "7.13.0", "@netlify/config": "20.12.1", @@ -2220,9 +2220,9 @@ "integrity": "sha512-4wMPu9iN3/HL97QblBsBay3E1etIciR84izI3U+4iALY+JHCrI+a2jO0qbAZ/nxKoegypYEaiiqWXylm+/zfrw==" }, "node_modules/@netlify/blobs": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-6.5.0.tgz", - "integrity": "sha512-wRFlNnL/Qv3WNLZd3OT/YYqF1zb6iPSo8T31sl9ccL1ahBxW1fBqKgF4b1XL7Z+6mRIkatvcsVPkWBcO+oJMNA==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-7.0.0.tgz", + "integrity": "sha512-JHYlZzF7LNRSZv8vDoChpEtrfe/P9m+GoWkllWsy6Pn68ZWRB2FcDdXjvqq3i9psbf3BLVEECmvxYbdYDhlIGQ==", "engines": { "node": "^14.16.0 || >=16.0.0" } @@ -2386,6 +2386,14 @@ "node": ">= 14" } }, + "node_modules/@netlify/build/node_modules/@netlify/blobs": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-6.5.0.tgz", + "integrity": "sha512-wRFlNnL/Qv3WNLZd3OT/YYqF1zb6iPSo8T31sl9ccL1ahBxW1fBqKgF4b1XL7Z+6mRIkatvcsVPkWBcO+oJMNA==", + "engines": { + "node": "^14.16.0 || >=16.0.0" + } + }, "node_modules/@netlify/build/node_modules/@netlify/zip-it-and-ship-it": { "version": "9.29.2", "resolved": "https://registry.npmjs.org/@netlify/zip-it-and-ship-it/-/zip-it-and-ship-it-9.29.2.tgz", @@ -14483,6 +14491,94 @@ "ipx": "bin/ipx.mjs" } }, + "node_modules/ipx/node_modules/@netlify/blobs": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-6.5.0.tgz", + "integrity": "sha512-wRFlNnL/Qv3WNLZd3OT/YYqF1zb6iPSo8T31sl9ccL1ahBxW1fBqKgF4b1XL7Z+6mRIkatvcsVPkWBcO+oJMNA==", + "optional": true, + "peer": true, + "engines": { + "node": "^14.16.0 || >=16.0.0" + } + }, + "node_modules/ipx/node_modules/lru-cache": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", + "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==", + "engines": { + "node": "14 || >=16.14" + } + }, + "node_modules/ipx/node_modules/unstorage": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/unstorage/-/unstorage-1.10.1.tgz", + "integrity": "sha512-rWQvLRfZNBpF+x8D3/gda5nUCQL2PgXy2jNG4U7/Rc9BGEv9+CAJd0YyGCROUBKs9v49Hg8huw3aih5Bf5TAVw==", + "dependencies": { + "anymatch": "^3.1.3", + "chokidar": "^3.5.3", + "destr": "^2.0.2", + "h3": "^1.8.2", + "ioredis": "^5.3.2", + "listhen": "^1.5.5", + "lru-cache": "^10.0.2", + "mri": "^1.2.0", + "node-fetch-native": "^1.4.1", + "ofetch": "^1.3.3", + "ufo": "^1.3.1" + }, + "peerDependencies": { + "@azure/app-configuration": "^1.4.1", + "@azure/cosmos": "^4.0.0", + "@azure/data-tables": "^13.2.2", + "@azure/identity": "^3.3.2", + "@azure/keyvault-secrets": "^4.7.0", + "@azure/storage-blob": "^12.16.0", + "@capacitor/preferences": "^5.0.6", + "@netlify/blobs": "^6.2.0", + "@planetscale/database": "^1.11.0", + "@upstash/redis": "^1.23.4", + "@vercel/kv": "^0.2.3", + "idb-keyval": "^6.2.1" + }, + "peerDependenciesMeta": { + "@azure/app-configuration": { + "optional": true + }, + "@azure/cosmos": { + "optional": true + }, + "@azure/data-tables": { + "optional": true + }, + "@azure/identity": { + "optional": true + }, + "@azure/keyvault-secrets": { + "optional": true + }, + "@azure/storage-blob": { + "optional": true + }, + "@capacitor/preferences": { + "optional": true + }, + "@netlify/blobs": { + "optional": true + }, + "@planetscale/database": { + "optional": true + }, + "@upstash/redis": { + "optional": true + }, + "@vercel/kv": { + "optional": true + }, + "idb-keyval": { + "optional": true + } + } + }, "node_modules/iron-webcrypto": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/iron-webcrypto/-/iron-webcrypto-1.0.0.tgz", @@ -22279,84 +22375,6 @@ "node": ">=0.10.0" } }, - "node_modules/unstorage": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/unstorage/-/unstorage-1.10.1.tgz", - "integrity": "sha512-rWQvLRfZNBpF+x8D3/gda5nUCQL2PgXy2jNG4U7/Rc9BGEv9+CAJd0YyGCROUBKs9v49Hg8huw3aih5Bf5TAVw==", - "dependencies": { - "anymatch": "^3.1.3", - "chokidar": "^3.5.3", - "destr": "^2.0.2", - "h3": "^1.8.2", - "ioredis": "^5.3.2", - "listhen": "^1.5.5", - "lru-cache": "^10.0.2", - "mri": "^1.2.0", - "node-fetch-native": "^1.4.1", - "ofetch": "^1.3.3", - "ufo": "^1.3.1" - }, - "peerDependencies": { - "@azure/app-configuration": "^1.4.1", - "@azure/cosmos": "^4.0.0", - "@azure/data-tables": "^13.2.2", - "@azure/identity": "^3.3.2", - "@azure/keyvault-secrets": "^4.7.0", - "@azure/storage-blob": "^12.16.0", - "@capacitor/preferences": "^5.0.6", - "@netlify/blobs": "^6.2.0", - "@planetscale/database": "^1.11.0", - "@upstash/redis": "^1.23.4", - "@vercel/kv": "^0.2.3", - "idb-keyval": "^6.2.1" - }, - "peerDependenciesMeta": { - "@azure/app-configuration": { - "optional": true - }, - "@azure/cosmos": { - "optional": true - }, - "@azure/data-tables": { - "optional": true - }, - "@azure/identity": { - "optional": true - }, - "@azure/keyvault-secrets": { - "optional": true - }, - "@azure/storage-blob": { - "optional": true - }, - "@capacitor/preferences": { - "optional": true - }, - "@netlify/blobs": { - "optional": true - }, - "@planetscale/database": { - "optional": true - }, - "@upstash/redis": { - "optional": true - }, - "@vercel/kv": { - "optional": true - }, - "idb-keyval": { - "optional": true - } - } - }, - "node_modules/unstorage/node_modules/lru-cache": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", - "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==", - "engines": { - "node": "14 || >=16.14" - } - }, "node_modules/untildify": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/untildify/-/untildify-3.0.3.tgz", @@ -24989,9 +25007,9 @@ "integrity": "sha512-4wMPu9iN3/HL97QblBsBay3E1etIciR84izI3U+4iALY+JHCrI+a2jO0qbAZ/nxKoegypYEaiiqWXylm+/zfrw==" }, "@netlify/blobs": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-6.5.0.tgz", - "integrity": "sha512-wRFlNnL/Qv3WNLZd3OT/YYqF1zb6iPSo8T31sl9ccL1ahBxW1fBqKgF4b1XL7Z+6mRIkatvcsVPkWBcO+oJMNA==" + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-7.0.0.tgz", + "integrity": "sha512-JHYlZzF7LNRSZv8vDoChpEtrfe/P9m+GoWkllWsy6Pn68ZWRB2FcDdXjvqq3i9psbf3BLVEECmvxYbdYDhlIGQ==" }, "@netlify/build": { "version": "29.36.1", @@ -25058,6 +25076,11 @@ "yargs": "^17.6.0" }, "dependencies": { + "@netlify/blobs": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-6.5.0.tgz", + "integrity": "sha512-wRFlNnL/Qv3WNLZd3OT/YYqF1zb6iPSo8T31sl9ccL1ahBxW1fBqKgF4b1XL7Z+6mRIkatvcsVPkWBcO+oJMNA==" + }, "@netlify/zip-it-and-ship-it": { "version": "9.29.2", "resolved": "https://registry.npmjs.org/@netlify/zip-it-and-ship-it/-/zip-it-and-ship-it-9.29.2.tgz", @@ -33721,6 +33744,38 @@ "ufo": "^1.3.2", "unstorage": "^1.10.1", "xss": "^1.0.14" + }, + "dependencies": { + "@netlify/blobs": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-6.5.0.tgz", + "integrity": "sha512-wRFlNnL/Qv3WNLZd3OT/YYqF1zb6iPSo8T31sl9ccL1ahBxW1fBqKgF4b1XL7Z+6mRIkatvcsVPkWBcO+oJMNA==", + "optional": true, + "peer": true + }, + "lru-cache": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", + "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==" + }, + "unstorage": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/unstorage/-/unstorage-1.10.1.tgz", + "integrity": "sha512-rWQvLRfZNBpF+x8D3/gda5nUCQL2PgXy2jNG4U7/Rc9BGEv9+CAJd0YyGCROUBKs9v49Hg8huw3aih5Bf5TAVw==", + "requires": { + "anymatch": "^3.1.3", + "chokidar": "^3.5.3", + "destr": "^2.0.2", + "h3": "^1.8.2", + "ioredis": "^5.3.2", + "listhen": "^1.5.5", + "lru-cache": "^10.0.2", + "mri": "^1.2.0", + "node-fetch-native": "^1.4.1", + "ofetch": "^1.3.3", + "ufo": "^1.3.1" + } + } } }, "iron-webcrypto": { @@ -39481,31 +39536,6 @@ } } }, - "unstorage": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/unstorage/-/unstorage-1.10.1.tgz", - "integrity": "sha512-rWQvLRfZNBpF+x8D3/gda5nUCQL2PgXy2jNG4U7/Rc9BGEv9+CAJd0YyGCROUBKs9v49Hg8huw3aih5Bf5TAVw==", - "requires": { - "anymatch": "^3.1.3", - "chokidar": "^3.5.3", - "destr": "^2.0.2", - "h3": "^1.8.2", - "ioredis": "^5.3.2", - "listhen": "^1.5.5", - "lru-cache": "^10.0.2", - "mri": "^1.2.0", - "node-fetch-native": "^1.4.1", - "ofetch": "^1.3.3", - "ufo": "^1.3.1" - }, - "dependencies": { - "lru-cache": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", - "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==" - } - } - }, "untildify": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/untildify/-/untildify-3.0.3.tgz", diff --git a/package.json b/package.json index cf847b5ca7c..59d12a47b98 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "dependencies": { "@bugsnag/js": "7.20.2", "@fastify/static": "6.10.2", - "@netlify/blobs": "6.5.0", + "@netlify/blobs": "7.0.0", "@netlify/build": "29.36.1", "@netlify/build-info": "7.13.0", "@netlify/config": "20.12.1", diff --git a/src/commands/recipes/recipes.ts b/src/commands/recipes/recipes.ts index b3e064826de..40a18cc72e8 100644 --- a/src/commands/recipes/recipes.ts +++ b/src/commands/recipes/recipes.ts @@ -11,11 +11,18 @@ import { getRecipe, listRecipes } from './common.js' const SUGGESTION_TIMEOUT = 1e4 -// @ts-expect-error TS(7031) FIXME: Binding element 'config' implicitly has an 'any' t... Remove this comment to see the full error message -export const runRecipe = async ({ config, recipeName, repositoryRoot }) => { +interface RunRecipeOptions { + args: string[] + command?: BaseCommand + config: unknown + recipeName: string + repositoryRoot: string +} + +export const runRecipe = async ({ args, command, config, recipeName, repositoryRoot }: RunRecipeOptions) => { const recipe = await getRecipe(recipeName) - return recipe.run({ config, repositoryRoot }) + return recipe.run({ args, command, config, repositoryRoot }) } export const recipesCommand = async (recipeName: string, options: OptionValues, command: BaseCommand): Promise => { @@ -26,8 +33,10 @@ export const recipesCommand = async (recipeName: string, options: OptionValues, return command.help() } + const args = command.args.slice(1) + try { - return await runRecipe({ config, recipeName: sanitizedRecipeName, repositoryRoot }) + return await runRecipe({ args, command, config, recipeName: sanitizedRecipeName, repositoryRoot }) } catch (error) { if ( // The ESM loader throws this instead of MODULE_NOT_FOUND diff --git a/src/lib/blobs/blobs.ts b/src/lib/blobs/blobs.ts index 54168eeff4f..bbe92788e5b 100644 --- a/src/lib/blobs/blobs.ts +++ b/src/lib/blobs/blobs.ts @@ -1,7 +1,7 @@ import { Buffer } from 'buffer' import path from 'path' -import { BlobsServer } from '@netlify/blobs' +import { BlobsServer } from '@netlify/blobs/server' import { v4 as uuidv4 } from 'uuid' import { log, NETLIFYDEVLOG } from '../../utils/command-helpers.js' diff --git a/src/lib/edge-functions/editor-helper.ts b/src/lib/edge-functions/editor-helper.ts index 1ee5d998cd6..e656000931a 100644 --- a/src/lib/edge-functions/editor-helper.ts +++ b/src/lib/edge-functions/editor-helper.ts @@ -40,5 +40,5 @@ export const promptEditorHelper = async ({ NETLIFYDEVLOG, chalk, config, log, re return } - await runRecipe({ config, recipeName: 'vscode', repositoryRoot }) + await runRecipe({ args: [], config, recipeName: 'vscode', repositoryRoot }) } diff --git a/src/recipes/blobs-migrate/index.ts b/src/recipes/blobs-migrate/index.ts new file mode 100644 index 00000000000..1179d530856 --- /dev/null +++ b/src/recipes/blobs-migrate/index.ts @@ -0,0 +1,107 @@ +import { getStore, listStores } from '@netlify/blobs' +import inquirer from 'inquirer' +import pMap from 'p-map' + +import BaseCommand from '../../commands/base-command.js' +import { error, log } from '../../utils/command-helpers.js' + +export const description = 'Migrate legacy Netlify Blobs stores' + +const BLOB_OPS_CONCURRENCY = 5 + +interface Options { + args: string[] + command: BaseCommand +} + +export const run = async ({ args, command }: Options) => { + if (args.length !== 1) { + return error(`Usage: netlify recipes blobs-migrate `) + } + + const [storeName] = args + const { api, siteInfo } = command.netlify + const clientOptions = { + apiURL: `${api.scheme}://${api.host}`, + siteID: siteInfo?.id ?? '', + token: api.accessToken ?? '', + } + + // The store we'll copy from. + const oldStore = getStore({ + ...clientOptions, + name: `netlify-internal/legacy-namespace/${storeName}`, + }) + + // The store we'll write to. + const newStore = getStore({ + ...clientOptions, + name: storeName, + }) + const { blobs } = await oldStore.list() + + if (blobs.length === 0) { + return log(`Store '${storeName}' does not exist or is empty, so there's nothing to migrate.`) + } + + const { stores } = await listStores(clientOptions) + + if (stores.includes(storeName)) { + const { confirmExistingStore } = await inquirer.prompt({ + type: 'confirm', + name: 'confirmExistingStore', + message: `The store '${storeName}' already exists in the new format, which means it has already been migrated or it has been used with a newer version of the Netlify Blobs client. If you continue with the migration, any blobs from the legacy store will overwrite newer entries that have the same key. Do you want to proceed?`, + default: false, + }) + + if (!confirmExistingStore) { + return + } + } + + const { confirmMigration } = await inquirer.prompt({ + type: 'confirm', + name: 'confirmMigration', + message: `You're about to migrate the store '${storeName}' with ${blobs.length} blobs. Do you want to proceed?`, + default: true, + }) + + if (!confirmMigration) { + return + } + + await pMap( + blobs, + async (blob) => { + log(`Migrating blob with key '${blob.key}'...`) + + const result = await oldStore.getWithMetadata(blob.key) + + if (result === null) { + return + } + + await newStore.set(blob.key, result.data, { metadata: result.metadata }) + }, + { concurrency: BLOB_OPS_CONCURRENCY }, + ) + + log('Verifying data in the new store...') + + const { blobs: newBlobs } = await newStore.list() + const blobsMap = new Map(newBlobs.map((blob) => [blob.key, blob.etag])) + + // Before deleting anything, let's first verify that all entries that exist + // in the old store are now also on the new store, with the same etag. + if (!blobs.every((blob) => blobsMap.get(blob.key) === blob.etag)) { + return error(`Failed to migrate some blobs. Try running the command again.`) + } + + try { + await pMap(blobs, (blob) => oldStore.delete(blob.key), { concurrency: BLOB_OPS_CONCURRENCY }) + } catch { + return error('Failed to remove legacy store after migration. Try running the command again.') + } + + log(`Store '${storeName}' has been migrated successfully.`) +} diff --git a/tests/integration/commands/blobs/blobs.test.ts b/tests/integration/commands/blobs/blobs.test.ts index 9ab9180a424..80491b72b14 100644 --- a/tests/integration/commands/blobs/blobs.test.ts +++ b/tests/integration/commands/blobs/blobs.test.ts @@ -2,10 +2,10 @@ import { readFile, rm } from 'fs/promises' import { join } from 'path' import { env } from 'process' -import { BlobsServer, ListResultBlob } from '@netlify/blobs' +import { ListResultBlob } from '@netlify/blobs' +import { BlobsServer } from '@netlify/blobs/server' import httpProxy from 'http-proxy' import { temporaryDirectory } from 'tempy' -import { v4 as uuidv4 } from 'uuid' import { afterAll, beforeAll, describe, expect, test } from 'vitest' import { FixtureTestContext, setupFixtureTests } from '../../utils/fixture.js' @@ -38,19 +38,17 @@ describe('blobs:* commands', () => { ] beforeAll(async () => { - const token = uuidv4() - server = new BlobsServer({ debug: true, directory, - token, + token: 'fake-token', }) const address = await server.start() routes.push({ method: 'all', - path: 'sites/site_id/blobs*', + path: 'blobs/*', response: (req, res) => { blobsProxy.web(req, res, { target: `http://localhost:${address.port}` }) }, diff --git a/tests/integration/commands/recipes/__snapshots__/recipes.test.js.snap b/tests/integration/commands/recipes/__snapshots__/recipes.test.js.snap index 8e1a6903bf2..5f69add5d76 100644 --- a/tests/integration/commands/recipes/__snapshots__/recipes.test.js.snap +++ b/tests/integration/commands/recipes/__snapshots__/recipes.test.js.snap @@ -1,13 +1,14 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`commands/recipes > Shows a list of all the available recipes 1`] = ` -".----------------------------------------------------------------------------------. -| Usage: netlify recipes | -|----------------------------------------------------------------------------------| -| Name | Description | -|--------|-------------------------------------------------------------------------| -| vscode | Create VS Code settings for an optimal experience with Netlify projects | -'----------------------------------------------------------------------------------'" +".-----------------------------------------------------------------------------------------. +| Usage: netlify recipes | +|-----------------------------------------------------------------------------------------| +| Name | Description | +|---------------|-------------------------------------------------------------------------| +| blobs-migrate | Migrate legacy Netlify Blobs stores | +| vscode | Create VS Code settings for an optimal experience with Netlify projects | +'-----------------------------------------------------------------------------------------'" `; exports[`commands/recipes > Suggests closest matching recipe on typo 1`] = `