Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: split dev, build blob upload jobs
This shares some, but not all, functionality between the two tasks. We should figure out how to better share functionality between the two jobs but this should be fine for now.
- Loading branch information
Showing
4 changed files
with
158 additions
and
89 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,20 +1,80 @@ | ||
| import { resolve } from 'node:path' | ||
| import { readFile } from 'node:fs/promises' | ||
| import path from 'node:path' | ||
|
|
||
| import { type Store } from '@netlify/blobs' | ||
| import { fdir } from 'fdir' | ||
|
|
||
| const METADATA_PREFIX = '$' | ||
| const METADATA_SUFFIX = '.json' | ||
|
|
||
| const BLOBS_PATH = '.netlify/blobs/deploy' | ||
|
|
||
| /** Retrieve the absolute path of the deploy scoped internal blob directory */ | ||
| export const getBlobsDir = (buildDir: string, packagePath?: string) => resolve(buildDir, packagePath || '', BLOBS_PATH) | ||
| export const getBlobsDir = (buildDir: string, packagePath?: string) => | ||
| path.resolve(buildDir, packagePath || '', BLOBS_PATH) | ||
|
|
||
| /** | ||
| * Detect if there are any blobs to upload | ||
| * @param buildDir The build directory. (current working directory where the build is executed) | ||
| * @param packagePath An optional package path for mono repositories | ||
| * @returns | ||
| */ | ||
| export const anyBlobsToUpload = async function (buildDir: string, packagePath?: string) { | ||
| export const anyBlobsToUpload = async (buildDir: string, packagePath?: string): Promise<boolean> => { | ||
| const blobsDir = getBlobsDir(buildDir, packagePath) | ||
| const { files } = await new fdir().onlyCounts().crawl(blobsDir).withPromise() | ||
| return files > 0 | ||
| } | ||
|
|
||
| /** Given output directory, find all file paths to upload excluding metadata files */ | ||
| export const getKeysToUpload = async (blobsDir: string): Promise<string[]> => { | ||
| const files = await new fdir() | ||
| .withRelativePaths() // we want the relative path from the blobsDir | ||
| .filter((fpath) => !path.basename(fpath).startsWith(METADATA_PREFIX)) | ||
| .crawl(blobsDir) | ||
| .withPromise() | ||
|
|
||
| // normalize the path separators to all use the forward slash | ||
| return files.map((f) => f.split(path.sep).join('/')) | ||
| } | ||
|
|
||
| /** Read a file and its metadata file from the blobs directory */ | ||
| const getFileWithMetadata = async ( | ||
| blobsDir: string, | ||
| key: string, | ||
| ): Promise<{ data: Buffer; metadata: Record<string, string> }> => { | ||
| const contentPath = path.join(blobsDir, key) | ||
| const dirname = path.dirname(key) | ||
| const basename = path.basename(key) | ||
| const metadataPath = path.join(blobsDir, dirname, `${METADATA_PREFIX}${basename}${METADATA_SUFFIX}`) | ||
|
|
||
| const [data, metadata] = await Promise.all([readFile(contentPath), readMetadata(metadataPath)]).catch((err) => { | ||
| throw new Error(`Failed while reading '${key}' and its metadata: ${err.message}`) | ||
| }) | ||
|
|
||
| return { data, metadata } | ||
| } | ||
|
|
||
| const readMetadata = async (metadataPath: string): Promise<Record<string, string>> => { | ||
| let metadataFile: string | ||
| try { | ||
| metadataFile = await readFile(metadataPath, { encoding: 'utf8' }) | ||
| } catch (err) { | ||
| if (err.code === 'ENOENT') { | ||
| // no metadata file found, that's ok | ||
| return {} | ||
| } | ||
| throw err | ||
| } | ||
|
|
||
| try { | ||
| return JSON.parse(metadataFile) | ||
| } catch { | ||
| // Normalize the error message | ||
| throw new Error(`Error parsing metadata file '${metadataPath}'`) | ||
| } | ||
| } | ||
|
|
||
| export const uploadBlob = async (store: Store, blobsDir: string, key: string): Promise<void> => { | ||
| const { data, metadata } = await getFileWithMetadata(blobsDir, key) | ||
| await store.set(key, data, { metadata }) | ||
| } |