Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: split dev, build blob upload jobs
This shares some, but not all, functionality between the two tasks. We should figure out how to better share functionality between the two jobs but this should be fine for now.
- Loading branch information
Showing
4 changed files
with
174 additions
and
89 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
111 changes: 101 additions & 10 deletions
111
packages/build/src/plugins_core/dev_blobs_upload/index.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,18 +1,109 @@ | ||
| import { uploadBlobs } from '../blobs_upload/index.js' | ||
| import { type CoreStep, type CoreStepCondition } from '../types.js' | ||
|
|
||
| const condition: CoreStepCondition = async (...args) => { | ||
| const { | ||
| constants: { IS_LOCAL }, | ||
| } = args[0] | ||
| return IS_LOCAL && ((await uploadBlobs.condition?.(...args)) ?? true) | ||
| import { version as nodeVersion } from 'node:process' | ||
|
|
||
| import { getDeployStore } from '@netlify/blobs' | ||
| import pMap from 'p-map' | ||
| import semver from 'semver' | ||
|
|
||
| import { log, logError } from '../../log/logger.js' | ||
| import { getFileWithMetadata, getKeysToUpload, scanForBlobs } from '../../utils/blobs.js' | ||
| import { type CoreStep, type CoreStepCondition, type CoreStepFunction } from '../types.js' | ||
|
|
||
| const coreStep: CoreStepFunction = async function ({ | ||
| debug, | ||
| logs, | ||
| deployId, | ||
| buildDir, | ||
| quiet, | ||
| packagePath, | ||
| constants: { SITE_ID, NETLIFY_API_TOKEN, NETLIFY_API_HOST }, | ||
| }) { | ||
| // This should never happen due to the condition check | ||
| if (!deployId || !NETLIFY_API_TOKEN) { | ||
| return {} | ||
| } | ||
| // for cli deploys with `netlify deploy --build` the `NETLIFY_API_HOST` is undefined | ||
| const apiHost = NETLIFY_API_HOST || 'api.netlify.com' | ||
|
|
||
| const storeOpts: Parameters<typeof getDeployStore>[0] = { | ||
| siteID: SITE_ID, | ||
| deployID: deployId, | ||
| token: NETLIFY_API_TOKEN, | ||
| apiURL: `https://${apiHost}`, | ||
| } | ||
|
|
||
| // If we don't have native `fetch` in the global scope, add a polyfill. | ||
| if (semver.lt(nodeVersion, '18.0.0')) { | ||
| const nodeFetch = (await import('node-fetch')).default as unknown as typeof fetch | ||
| storeOpts.fetch = nodeFetch | ||
| } | ||
|
|
||
| const blobs = await scanForBlobs(buildDir, packagePath) | ||
|
|
||
| // We checked earlier, but let's be extra safe | ||
| if (blobs === null) { | ||
| if (!quiet) { | ||
| log(logs, 'No blobs to upload to deploy store.') | ||
| } | ||
| return {} | ||
| } | ||
|
|
||
| // If using the deploy config API, configure the store to use the region that | ||
| // was configured for the deploy. | ||
| if (!blobs.isLegacyDirectory) { | ||
| storeOpts.experimentalRegion = 'auto' | ||
| } | ||
|
|
||
| const blobStore = getDeployStore(storeOpts) | ||
| const keys = await getKeysToUpload(blobs.directory) | ||
|
|
||
| if (keys.length === 0) { | ||
| if (!quiet) { | ||
| log(logs, 'No blobs to upload to deploy store.') | ||
| } | ||
| return {} | ||
| } | ||
|
|
||
| if (!quiet) { | ||
| log(logs, `Uploading ${keys.length} blobs to deploy store...`) | ||
| } | ||
|
|
||
| try { | ||
| await pMap( | ||
| keys, | ||
| async (key: string) => { | ||
| if (debug && !quiet) { | ||
| log(logs, `- Uploading blob ${key}`, { indent: true }) | ||
| } | ||
| const { data, metadata } = await getFileWithMetadata(blobs.directory, key) | ||
| await blobStore.set(key, data, { metadata }) | ||
| }, | ||
| { concurrency: 10 }, | ||
| ) | ||
| } catch (err) { | ||
| logError(logs, `Error uploading blobs to deploy store: ${err.message}`) | ||
|
|
||
| throw new Error(`Failed while uploading blobs to deploy store`) | ||
| } | ||
|
|
||
| if (!quiet) { | ||
| log(logs, `Done uploading blobs to deploy store.`) | ||
| } | ||
|
|
||
| return {} | ||
| } | ||
|
|
||
| const deployAndBlobsPresent: CoreStepCondition = async ({ | ||
| deployId, | ||
| buildDir, | ||
| packagePath, | ||
| constants: { NETLIFY_API_TOKEN }, | ||
| }) => Boolean(NETLIFY_API_TOKEN && deployId && (await scanForBlobs(buildDir, packagePath))) | ||
|
|
||
| export const devUploadBlobs: CoreStep = { | ||
| event: 'onDev', | ||
| coreStep: uploadBlobs.coreStep, | ||
| coreStep, | ||
| coreStepId: 'dev_blobs_upload', | ||
| coreStepName: 'Uploading blobs', | ||
| coreStepDescription: () => 'Uploading blobs to development deploy store', | ||
| condition, | ||
| condition: deployAndBlobsPresent, | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters