From 85a607c02452f57b797553f78ec94a465c5f29ac Mon Sep 17 00:00:00 2001 From: Nathan Houle Date: Mon, 1 Apr 2024 13:29:12 -0700 Subject: [PATCH] refactor: split dev, build blob upload jobs This shares some, but not all, functionality between the two tasks. We should figure out how to better share functionality between the two jobs but this should be fine for now. --- .../src/plugins_core/blobs_upload/index.ts | 37 ++++---- .../src/plugins_core/blobs_upload/utils.ts | 56 ------------ .../plugins_core/dev_blobs_upload/index.ts | 88 +++++++++++++++++-- packages/build/src/utils/blobs.ts | 66 +++++++++++++- 4 files changed, 158 insertions(+), 89 deletions(-) delete mode 100644 packages/build/src/plugins_core/blobs_upload/utils.ts diff --git a/packages/build/src/plugins_core/blobs_upload/index.ts b/packages/build/src/plugins_core/blobs_upload/index.ts index d49aa7858d..f604a0f22e 100644 --- a/packages/build/src/plugins_core/blobs_upload/index.ts +++ b/packages/build/src/plugins_core/blobs_upload/index.ts @@ -5,11 +5,9 @@ import pMap from 'p-map' import semver from 'semver' import { log, logError } from '../../log/logger.js' -import { anyBlobsToUpload, getBlobsDir } from '../../utils/blobs.js' +import { anyBlobsToUpload, getBlobsDir, getKeysToUpload, uploadBlob } from '../../utils/blobs.js' import { CoreStep, CoreStepCondition, CoreStepFunction } from '../types.js' -import { getKeysToUpload, getFileWithMetadata } from './utils.js' - const coreStep: CoreStepFunction = async function ({ debug, logs, @@ -26,7 +24,7 @@ const coreStep: CoreStepFunction = async function ({ // for cli deploys with `netlify deploy --build` the `NETLIFY_API_HOST` is undefined const apiHost = NETLIFY_API_HOST || 'api.netlify.com' - const storeOpts: { siteID: string; deployID: string; token: string; apiURL: string; fetch?: any } = { + const storeOpts: { siteID: string; deployID: string; token: string; apiURL: string; fetch?: typeof fetch } = { siteID: SITE_ID, deployID: deployId, token: NETLIFY_API_TOKEN, @@ -34,7 +32,7 @@ const coreStep: CoreStepFunction = async function ({ } if (semver.lt(nodeVersion, '18.0.0')) { const nodeFetch = await import('node-fetch') - storeOpts.fetch = nodeFetch.default + storeOpts.fetch = nodeFetch.default as unknown as typeof fetch } const blobStore = getDeployStore(storeOpts) @@ -53,16 +51,17 @@ const coreStep: CoreStepFunction = async function ({ log(logs, `Uploading ${keys.length} blobs to deploy store...`) } - const uploadBlob = async (key) => { - if (debug && !quiet) { - log(logs, `- Uploading blob ${key}`, { indent: true }) - } - const { data, metadata } = await getFileWithMetadata(blobsDir, key) - await blobStore.set(key, data, { metadata }) - } - try { - await pMap(keys, uploadBlob, { concurrency: 10 }) + await pMap( + keys, + async (key) => { + if (debug && !quiet) { + log(logs, `- Uploading blob ${key}`, { indent: true }) + } + await uploadBlob(blobStore, blobsDir, key) + }, + { concurrency: 10 }, + ) } catch (err) { logError(logs, `Error uploading blobs to deploy store: ${err.message}`) @@ -76,12 +75,8 @@ const coreStep: CoreStepFunction = async function ({ return {} } -const deployAndBlobsPresent: CoreStepCondition = async ({ - deployId, - buildDir, - packagePath, - constants: { NETLIFY_API_TOKEN }, -}) => Boolean(NETLIFY_API_TOKEN && deployId && (await anyBlobsToUpload(buildDir, packagePath))) +const condition: CoreStepCondition = async ({ deployId, buildDir, packagePath, constants: { NETLIFY_API_TOKEN } }) => + Boolean(NETLIFY_API_TOKEN && deployId && (await anyBlobsToUpload(buildDir, packagePath))) export const uploadBlobs: CoreStep = { event: 'onPostBuild', @@ -89,5 +84,5 @@ export const uploadBlobs: CoreStep = { coreStepId: 'blobs_upload', coreStepName: 'Uploading blobs', coreStepDescription: () => 'Uploading blobs to deploy store', - condition: deployAndBlobsPresent, + condition, } diff --git a/packages/build/src/plugins_core/blobs_upload/utils.ts b/packages/build/src/plugins_core/blobs_upload/utils.ts deleted file mode 100644 index eafb37ae76..0000000000 --- a/packages/build/src/plugins_core/blobs_upload/utils.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { readFile } from 'node:fs/promises' -import path from 'node:path' - -import { fdir } from 'fdir' - -const METADATA_PREFIX = '$' -const METADATA_SUFFIX = '.json' - -/** Given output directory, find all file paths to upload excluding metadata files */ -export async function getKeysToUpload(blobsDir: string): Promise { - const files = await new fdir() - .withRelativePaths() // we want the relative path from the blobsDir - .filter((fpath) => !path.basename(fpath).startsWith(METADATA_PREFIX)) - .crawl(blobsDir) - .withPromise() - - // normalize the path separators to all use the forward slash - return files.map((f) => f.split(path.sep).join('/')) -} - -/** Read a file and its metadata file from the blobs directory */ -export async function getFileWithMetadata( - blobsDir: string, - key: string, -): Promise<{ data: Buffer; metadata: Record }> { - const contentPath = path.join(blobsDir, key) - const dirname = path.dirname(key) - const basename = path.basename(key) - const metadataPath = path.join(blobsDir, dirname, `${METADATA_PREFIX}${basename}${METADATA_SUFFIX}`) - - const [data, metadata] = await Promise.all([readFile(contentPath), readMetadata(metadataPath)]).catch((err) => { - throw new Error(`Failed while reading '${key}' and its metadata: ${err.message}`) - }) - - return { data, metadata } -} - -async function readMetadata(metadataPath: string): Promise> { - let metadataFile - try { - metadataFile = await readFile(metadataPath, { encoding: 'utf8' }) - } catch (err) { - if (err.code === 'ENOENT') { - // no metadata file found, that's ok - return {} - } - throw err - } - - try { - return JSON.parse(metadataFile) - } catch { - // Normalize the error message - throw new Error(`Error parsing metadata file '${metadataPath}'`) - } -} diff --git a/packages/build/src/plugins_core/dev_blobs_upload/index.ts b/packages/build/src/plugins_core/dev_blobs_upload/index.ts index 97a0b75ac0..6bf53fca7f 100644 --- a/packages/build/src/plugins_core/dev_blobs_upload/index.ts +++ b/packages/build/src/plugins_core/dev_blobs_upload/index.ts @@ -1,16 +1,86 @@ -import { uploadBlobs } from '../blobs_upload/index.js' -import { type CoreStep, type CoreStepCondition } from '../types.js' - -const condition: CoreStepCondition = async (...args) => { - const { - constants: { IS_LOCAL }, - } = args[0] - return IS_LOCAL && ((await uploadBlobs.condition?.(...args)) ?? true) +import { version as nodeVersion } from 'node:process' + +import { getDeployStore } from '@netlify/blobs' +import pMap from 'p-map' +import semver from 'semver' + +import { log, logError } from '../../log/logger.js' +import { anyBlobsToUpload, getBlobsDir, getKeysToUpload, uploadBlob } from '../../utils/blobs.js' +import { type CoreStep, type CoreStepCondition, type CoreStepFunction } from '../types.js' + +const coreStep: CoreStepFunction = async function ({ + debug, + logs, + deployId, + buildDir, + quiet, + packagePath, + constants: { SITE_ID, NETLIFY_API_TOKEN, NETLIFY_API_HOST }, +}) { + // This should never happen due to the condition check + if (!deployId || !NETLIFY_API_TOKEN) { + return {} + } + // for cli deploys with `netlify deploy --build` the `NETLIFY_API_HOST` is undefined + const apiHost = NETLIFY_API_HOST || 'api.netlify.com' + + const storeOpts: { siteID: string; deployID: string; token: string; apiURL: string; fetch?: typeof fetch } = { + siteID: SITE_ID, + deployID: deployId, + token: NETLIFY_API_TOKEN, + apiURL: `https://${apiHost}`, + } + if (semver.lt(nodeVersion, '18.0.0')) { + const nodeFetch = await import('node-fetch') + storeOpts.fetch = nodeFetch.default as unknown as typeof fetch + } + + const blobStore = getDeployStore(storeOpts) + const blobsDir = getBlobsDir(buildDir, packagePath) + const keys = await getKeysToUpload(blobsDir) + + // We checked earlier, but let's be extra safe + if (keys.length === 0) { + if (!quiet) { + log(logs, 'No blobs to upload to development store.') + } + return {} + } + + if (!quiet) { + log(logs, `Uploading ${keys.length} blobs to development store...`) + } + + try { + await pMap( + keys, + async (key) => { + if (debug && !quiet) { + log(logs, `- Uploading blob ${key}`, { indent: true }) + } + await uploadBlob(blobStore, blobsDir, key) + }, + { concurrency: 10 }, + ) + } catch (err) { + logError(logs, `Error uploading blobs to development store: ${err.message}`) + + throw new Error(`Failed while uploading blobs to development store`) + } + + if (!quiet) { + log(logs, `Done uploading blobs to development store.`) + } + + return {} } +const condition: CoreStepCondition = async ({ deployId, buildDir, packagePath, constants: { NETLIFY_API_TOKEN } }) => + Boolean(NETLIFY_API_TOKEN && deployId && (await anyBlobsToUpload(buildDir, packagePath))) + export const devUploadBlobs: CoreStep = { event: 'onDev', - coreStep: uploadBlobs.coreStep, + coreStep, coreStepId: 'dev_blobs_upload', coreStepName: 'Uploading blobs', coreStepDescription: () => 'Uploading blobs to development deploy store', diff --git a/packages/build/src/utils/blobs.ts b/packages/build/src/utils/blobs.ts index 961acd801e..f1db6f3c5e 100644 --- a/packages/build/src/utils/blobs.ts +++ b/packages/build/src/utils/blobs.ts @@ -1,11 +1,17 @@ -import { resolve } from 'node:path' +import { readFile } from 'node:fs/promises' +import path from 'node:path' +import { type Store } from '@netlify/blobs' import { fdir } from 'fdir' +const METADATA_PREFIX = '$' +const METADATA_SUFFIX = '.json' + const BLOBS_PATH = '.netlify/blobs/deploy' /** Retrieve the absolute path of the deploy scoped internal blob directory */ -export const getBlobsDir = (buildDir: string, packagePath?: string) => resolve(buildDir, packagePath || '', BLOBS_PATH) +export const getBlobsDir = (buildDir: string, packagePath?: string) => + path.resolve(buildDir, packagePath || '', BLOBS_PATH) /** * Detect if there are any blobs to upload @@ -13,8 +19,62 @@ export const getBlobsDir = (buildDir: string, packagePath?: string) => resolve(b * @param packagePath An optional package path for mono repositories * @returns */ -export const anyBlobsToUpload = async function (buildDir: string, packagePath?: string) { +export const anyBlobsToUpload = async (buildDir: string, packagePath?: string): Promise => { const blobsDir = getBlobsDir(buildDir, packagePath) const { files } = await new fdir().onlyCounts().crawl(blobsDir).withPromise() return files > 0 } + +/** Given output directory, find all file paths to upload excluding metadata files */ +export const getKeysToUpload = async (blobsDir: string): Promise => { + const files = await new fdir() + .withRelativePaths() // we want the relative path from the blobsDir + .filter((fpath) => !path.basename(fpath).startsWith(METADATA_PREFIX)) + .crawl(blobsDir) + .withPromise() + + // normalize the path separators to all use the forward slash + return files.map((f) => f.split(path.sep).join('/')) +} + +/** Read a file and its metadata file from the blobs directory */ +const getFileWithMetadata = async ( + blobsDir: string, + key: string, +): Promise<{ data: Buffer; metadata: Record }> => { + const contentPath = path.join(blobsDir, key) + const dirname = path.dirname(key) + const basename = path.basename(key) + const metadataPath = path.join(blobsDir, dirname, `${METADATA_PREFIX}${basename}${METADATA_SUFFIX}`) + + const [data, metadata] = await Promise.all([readFile(contentPath), readMetadata(metadataPath)]).catch((err) => { + throw new Error(`Failed while reading '${key}' and its metadata: ${err.message}`) + }) + + return { data, metadata } +} + +const readMetadata = async (metadataPath: string): Promise> => { + let metadataFile: string + try { + metadataFile = await readFile(metadataPath, { encoding: 'utf8' }) + } catch (err) { + if (err.code === 'ENOENT') { + // no metadata file found, that's ok + return {} + } + throw err + } + + try { + return JSON.parse(metadataFile) + } catch { + // Normalize the error message + throw new Error(`Error parsing metadata file '${metadataPath}'`) + } +} + +export const uploadBlob = async (store: Store, blobsDir: string, key: string): Promise => { + const { data, metadata } = await getFileWithMetadata(blobsDir, key) + await store.set(key, data, { metadata }) +}