diff --git a/packages/build/src/plugins_core/blobs_upload/index.ts b/packages/build/src/plugins_core/blobs_upload/index.ts index d49aa7858d..f604a0f22e 100644 --- a/packages/build/src/plugins_core/blobs_upload/index.ts +++ b/packages/build/src/plugins_core/blobs_upload/index.ts @@ -5,11 +5,9 @@ import pMap from 'p-map' import semver from 'semver' import { log, logError } from '../../log/logger.js' -import { anyBlobsToUpload, getBlobsDir } from '../../utils/blobs.js' +import { anyBlobsToUpload, getBlobsDir, getKeysToUpload, uploadBlob } from '../../utils/blobs.js' import { CoreStep, CoreStepCondition, CoreStepFunction } from '../types.js' -import { getKeysToUpload, getFileWithMetadata } from './utils.js' - const coreStep: CoreStepFunction = async function ({ debug, logs, @@ -26,7 +24,7 @@ const coreStep: CoreStepFunction = async function ({ // for cli deploys with `netlify deploy --build` the `NETLIFY_API_HOST` is undefined const apiHost = NETLIFY_API_HOST || 'api.netlify.com' - const storeOpts: { siteID: string; deployID: string; token: string; apiURL: string; fetch?: any } = { + const storeOpts: { siteID: string; deployID: string; token: string; apiURL: string; fetch?: typeof fetch } = { siteID: SITE_ID, deployID: deployId, token: NETLIFY_API_TOKEN, @@ -34,7 +32,7 @@ const coreStep: CoreStepFunction = async function ({ } if (semver.lt(nodeVersion, '18.0.0')) { const nodeFetch = await import('node-fetch') - storeOpts.fetch = nodeFetch.default + storeOpts.fetch = nodeFetch.default as unknown as typeof fetch } const blobStore = getDeployStore(storeOpts) @@ -53,16 +51,17 @@ const coreStep: CoreStepFunction = async function ({ log(logs, `Uploading ${keys.length} blobs to deploy store...`) } - const uploadBlob = async (key) => { - if (debug && !quiet) { - log(logs, `- Uploading blob ${key}`, { indent: true }) - } - const { data, metadata } = await getFileWithMetadata(blobsDir, key) - await blobStore.set(key, data, { metadata }) - } - try { - await pMap(keys, uploadBlob, { concurrency: 10 }) + await pMap( + keys, + async (key) => { + if (debug && !quiet) { + log(logs, `- Uploading blob ${key}`, { indent: true }) + } + await uploadBlob(blobStore, blobsDir, key) + }, + { concurrency: 10 }, + ) } catch (err) { logError(logs, `Error uploading blobs to deploy store: ${err.message}`) @@ -76,12 +75,8 @@ const coreStep: CoreStepFunction = async function ({ return {} } -const deployAndBlobsPresent: CoreStepCondition = async ({ - deployId, - buildDir, - packagePath, - constants: { NETLIFY_API_TOKEN }, -}) => Boolean(NETLIFY_API_TOKEN && deployId && (await anyBlobsToUpload(buildDir, packagePath))) +const condition: CoreStepCondition = async ({ deployId, buildDir, packagePath, constants: { NETLIFY_API_TOKEN } }) => + Boolean(NETLIFY_API_TOKEN && deployId && (await anyBlobsToUpload(buildDir, packagePath))) export const uploadBlobs: CoreStep = { event: 'onPostBuild', @@ -89,5 +84,5 @@ export const uploadBlobs: CoreStep = { coreStepId: 'blobs_upload', coreStepName: 'Uploading blobs', coreStepDescription: () => 'Uploading blobs to deploy store', - condition: deployAndBlobsPresent, + condition, } diff --git a/packages/build/src/plugins_core/blobs_upload/utils.ts b/packages/build/src/plugins_core/blobs_upload/utils.ts deleted file mode 100644 index eafb37ae76..0000000000 --- a/packages/build/src/plugins_core/blobs_upload/utils.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { readFile } from 'node:fs/promises' -import path from 'node:path' - -import { fdir } from 'fdir' - -const METADATA_PREFIX = '$' -const METADATA_SUFFIX = '.json' - -/** Given output directory, find all file paths to upload excluding metadata files */ -export async function getKeysToUpload(blobsDir: string): Promise { - const files = await new fdir() - .withRelativePaths() // we want the relative path from the blobsDir - .filter((fpath) => !path.basename(fpath).startsWith(METADATA_PREFIX)) - .crawl(blobsDir) - .withPromise() - - // normalize the path separators to all use the forward slash - return files.map((f) => f.split(path.sep).join('/')) -} - -/** Read a file and its metadata file from the blobs directory */ -export async function getFileWithMetadata( - blobsDir: string, - key: string, -): Promise<{ data: Buffer; metadata: Record }> { - const contentPath = path.join(blobsDir, key) - const dirname = path.dirname(key) - const basename = path.basename(key) - const metadataPath = path.join(blobsDir, dirname, `${METADATA_PREFIX}${basename}${METADATA_SUFFIX}`) - - const [data, metadata] = await Promise.all([readFile(contentPath), readMetadata(metadataPath)]).catch((err) => { - throw new Error(`Failed while reading '${key}' and its metadata: ${err.message}`) - }) - - return { data, metadata } -} - -async function readMetadata(metadataPath: string): Promise> { - let metadataFile - try { - metadataFile = await readFile(metadataPath, { encoding: 'utf8' }) - } catch (err) { - if (err.code === 'ENOENT') { - // no metadata file found, that's ok - return {} - } - throw err - } - - try { - return JSON.parse(metadataFile) - } catch { - // Normalize the error message - throw new Error(`Error parsing metadata file '${metadataPath}'`) - } -} diff --git a/packages/build/src/plugins_core/dev_blobs_upload/index.ts b/packages/build/src/plugins_core/dev_blobs_upload/index.ts index 97a0b75ac0..6bf53fca7f 100644 --- a/packages/build/src/plugins_core/dev_blobs_upload/index.ts +++ b/packages/build/src/plugins_core/dev_blobs_upload/index.ts @@ -1,16 +1,86 @@ -import { uploadBlobs } from '../blobs_upload/index.js' -import { type CoreStep, type CoreStepCondition } from '../types.js' - -const condition: CoreStepCondition = async (...args) => { - const { - constants: { IS_LOCAL }, - } = args[0] - return IS_LOCAL && ((await uploadBlobs.condition?.(...args)) ?? true) +import { version as nodeVersion } from 'node:process' + +import { getDeployStore } from '@netlify/blobs' +import pMap from 'p-map' +import semver from 'semver' + +import { log, logError } from '../../log/logger.js' +import { anyBlobsToUpload, getBlobsDir, getKeysToUpload, uploadBlob } from '../../utils/blobs.js' +import { type CoreStep, type CoreStepCondition, type CoreStepFunction } from '../types.js' + +const coreStep: CoreStepFunction = async function ({ + debug, + logs, + deployId, + buildDir, + quiet, + packagePath, + constants: { SITE_ID, NETLIFY_API_TOKEN, NETLIFY_API_HOST }, +}) { + // This should never happen due to the condition check + if (!deployId || !NETLIFY_API_TOKEN) { + return {} + } + // for cli deploys with `netlify deploy --build` the `NETLIFY_API_HOST` is undefined + const apiHost = NETLIFY_API_HOST || 'api.netlify.com' + + const storeOpts: { siteID: string; deployID: string; token: string; apiURL: string; fetch?: typeof fetch } = { + siteID: SITE_ID, + deployID: deployId, + token: NETLIFY_API_TOKEN, + apiURL: `https://${apiHost}`, + } + if (semver.lt(nodeVersion, '18.0.0')) { + const nodeFetch = await import('node-fetch') + storeOpts.fetch = nodeFetch.default as unknown as typeof fetch + } + + const blobStore = getDeployStore(storeOpts) + const blobsDir = getBlobsDir(buildDir, packagePath) + const keys = await getKeysToUpload(blobsDir) + + // We checked earlier, but let's be extra safe + if (keys.length === 0) { + if (!quiet) { + log(logs, 'No blobs to upload to development store.') + } + return {} + } + + if (!quiet) { + log(logs, `Uploading ${keys.length} blobs to development store...`) + } + + try { + await pMap( + keys, + async (key) => { + if (debug && !quiet) { + log(logs, `- Uploading blob ${key}`, { indent: true }) + } + await uploadBlob(blobStore, blobsDir, key) + }, + { concurrency: 10 }, + ) + } catch (err) { + logError(logs, `Error uploading blobs to development store: ${err.message}`) + + throw new Error(`Failed while uploading blobs to development store`) + } + + if (!quiet) { + log(logs, `Done uploading blobs to development store.`) + } + + return {} } +const condition: CoreStepCondition = async ({ deployId, buildDir, packagePath, constants: { NETLIFY_API_TOKEN } }) => + Boolean(NETLIFY_API_TOKEN && deployId && (await anyBlobsToUpload(buildDir, packagePath))) + export const devUploadBlobs: CoreStep = { event: 'onDev', - coreStep: uploadBlobs.coreStep, + coreStep, coreStepId: 'dev_blobs_upload', coreStepName: 'Uploading blobs', coreStepDescription: () => 'Uploading blobs to development deploy store', diff --git a/packages/build/src/utils/blobs.ts b/packages/build/src/utils/blobs.ts index 961acd801e..f1db6f3c5e 100644 --- a/packages/build/src/utils/blobs.ts +++ b/packages/build/src/utils/blobs.ts @@ -1,11 +1,17 @@ -import { resolve } from 'node:path' +import { readFile } from 'node:fs/promises' +import path from 'node:path' +import { type Store } from '@netlify/blobs' import { fdir } from 'fdir' +const METADATA_PREFIX = '$' +const METADATA_SUFFIX = '.json' + const BLOBS_PATH = '.netlify/blobs/deploy' /** Retrieve the absolute path of the deploy scoped internal blob directory */ -export const getBlobsDir = (buildDir: string, packagePath?: string) => resolve(buildDir, packagePath || '', BLOBS_PATH) +export const getBlobsDir = (buildDir: string, packagePath?: string) => + path.resolve(buildDir, packagePath || '', BLOBS_PATH) /** * Detect if there are any blobs to upload @@ -13,8 +19,62 @@ export const getBlobsDir = (buildDir: string, packagePath?: string) => resolve(b * @param packagePath An optional package path for mono repositories * @returns */ -export const anyBlobsToUpload = async function (buildDir: string, packagePath?: string) { +export const anyBlobsToUpload = async (buildDir: string, packagePath?: string): Promise => { const blobsDir = getBlobsDir(buildDir, packagePath) const { files } = await new fdir().onlyCounts().crawl(blobsDir).withPromise() return files > 0 } + +/** Given output directory, find all file paths to upload excluding metadata files */ +export const getKeysToUpload = async (blobsDir: string): Promise => { + const files = await new fdir() + .withRelativePaths() // we want the relative path from the blobsDir + .filter((fpath) => !path.basename(fpath).startsWith(METADATA_PREFIX)) + .crawl(blobsDir) + .withPromise() + + // normalize the path separators to all use the forward slash + return files.map((f) => f.split(path.sep).join('/')) +} + +/** Read a file and its metadata file from the blobs directory */ +const getFileWithMetadata = async ( + blobsDir: string, + key: string, +): Promise<{ data: Buffer; metadata: Record }> => { + const contentPath = path.join(blobsDir, key) + const dirname = path.dirname(key) + const basename = path.basename(key) + const metadataPath = path.join(blobsDir, dirname, `${METADATA_PREFIX}${basename}${METADATA_SUFFIX}`) + + const [data, metadata] = await Promise.all([readFile(contentPath), readMetadata(metadataPath)]).catch((err) => { + throw new Error(`Failed while reading '${key}' and its metadata: ${err.message}`) + }) + + return { data, metadata } +} + +const readMetadata = async (metadataPath: string): Promise> => { + let metadataFile: string + try { + metadataFile = await readFile(metadataPath, { encoding: 'utf8' }) + } catch (err) { + if (err.code === 'ENOENT') { + // no metadata file found, that's ok + return {} + } + throw err + } + + try { + return JSON.parse(metadataFile) + } catch { + // Normalize the error message + throw new Error(`Error parsing metadata file '${metadataPath}'`) + } +} + +export const uploadBlob = async (store: Store, blobsDir: string, key: string): Promise => { + const { data, metadata } = await getFileWithMetadata(blobsDir, key) + await store.set(key, data, { metadata }) +}