Skip to content

Commit

Permalink
perf(@angular-devkit/build-angular): add persistent caching of JavaSc…
Browse files Browse the repository at this point in the history
…ript transformations

When caching is enabled for the Angular CLI, the JavaScript transformer
within the application build system will now perform persistent caching
of its output. This allows for improved warm build times by removing the
need to run the Angular linker and build optimizer steps against unchanged
third-party code within each build. This does not affect hot rebuilds that
would take place during watch mode since the outputs are already cached
within memory. The on-disk storage of the cached data is handled by the
`lmdb` package which provides fast key/value storage and built-in off-thread
compression. This package is currently used by such projects as Gatsby and
parcel for their respective caching subsystems.

Warm production build of a new 18.0.0-rc.2 project with disabled caching:
```
Application bundle generation complete. [3.698 seconds]
```
With enabled caching:
```
Application bundle generation complete. [2.277 seconds]
```
  • Loading branch information
clydin committed May 13, 2024
1 parent 46e5993 commit 1e98f72
Show file tree
Hide file tree
Showing 6 changed files with 206 additions and 5 deletions.
1 change: 1 addition & 0 deletions package.json
Expand Up @@ -159,6 +159,7 @@
"less-loader": "12.2.0",
"license-checker": "^25.0.0",
"license-webpack-plugin": "4.0.2",
"lmdb": "2.9.2",
"loader-utils": "3.2.1",
"lodash": "^4.17.21",
"magic-string": "0.30.10",
Expand Down
1 change: 1 addition & 0 deletions packages/angular/build/BUILD.bazel
Expand Up @@ -83,6 +83,7 @@ ts_library(
"@npm//fast-glob",
"@npm//https-proxy-agent",
"@npm//inquirer",
"@npm//lmdb",
"@npm//magic-string",
"@npm//mrmime",
"@npm//ora",
Expand Down
1 change: 1 addition & 0 deletions packages/angular/build/package.json
Expand Up @@ -31,6 +31,7 @@
"fast-glob": "3.3.2",
"https-proxy-agent": "7.0.4",
"inquirer": "9.2.20",
"lmdb": "3.0.8",
"magic-string": "0.30.10",
"mrmime": "2.0.0",
"ora": "5.4.1",
Expand Down
Expand Up @@ -19,6 +19,7 @@ import assert from 'node:assert';
import * as path from 'node:path';
import { maxWorkers, useTypeChecking } from '../../../utils/environment-options';
import { JavaScriptTransformer } from '../javascript-transformer';
import { LmbdCacheStore } from '../lmdb-cache-store';
import { LoadResultCache, createCachedLoad } from '../load-result-cache';
import { logCumulativeDurations, profileAsync, resetCumulativeDurations } from '../profiling';
import { BundleStylesheetOptions } from '../stylesheets/bundle-options';
Expand Down Expand Up @@ -62,7 +63,17 @@ export function createCompilerPlugin(
const preserveSymlinks = build.initialOptions.preserveSymlinks;

// Initialize a worker pool for JavaScript transformations
const javascriptTransformer = new JavaScriptTransformer(pluginOptions, maxWorkers);
let cacheStore;
if (pluginOptions.sourceFileCache?.persistentCachePath) {
cacheStore = new LmbdCacheStore(
pluginOptions.sourceFileCache.persistentCachePath + '/angular-compiler.db',
);
}
const javascriptTransformer = new JavaScriptTransformer(
pluginOptions,
maxWorkers,
cacheStore?.createCache('jstransformer'),
);

// Setup defines based on the values used by the Angular compiler-cli
build.initialOptions.define ??= {};
Expand Down
59 changes: 59 additions & 0 deletions packages/angular/build/src/tools/esbuild/lmdb-cache-store.ts
@@ -0,0 +1,59 @@
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/

import { RootDatabase, open } from 'lmdb';
import { Cache, CacheStore } from './cache';

export class LmbdCacheStore implements CacheStore<unknown> {
readonly #cacheFileUrl;
#db: RootDatabase | undefined;

constructor(readonly cachePath: string) {
this.#cacheFileUrl = cachePath;
}

#ensureCacheFile(): RootDatabase {
this.#db ??= open({
path: this.#cacheFileUrl,
compression: true,
});

return this.#db;
}

// eslint-disable-next-line @typescript-eslint/no-explicit-any
async get(key: string): Promise<any> {
const db = this.#ensureCacheFile();
const value = db.get(key);

return value;
}

has(key: string): boolean {
return this.#ensureCacheFile().doesExist(key);
}

async set(key: string, value: unknown): Promise<this> {
const db = this.#ensureCacheFile();
await db.put(key, value);

return this;
}

createCache<V = unknown>(namespace: string): Cache<V> {
return new Cache(this, namespace);
}

async close() {
try {
await this.#db?.close();
} catch {
// Failure to close should not be fatal
}
}
}
136 changes: 132 additions & 4 deletions yarn.lock
Expand Up @@ -51,8 +51,7 @@
tslib "^2.3.0"

"@angular/bazel@https://github.com/angular/bazel-builds.git#b7599b1f8390b9ceeccae3ea976fcf6ede335396":
version "18.1.0-next.0+sha-b9c88dc"
uid b7599b1f8390b9ceeccae3ea976fcf6ede335396
version "18.1.0-next.0"
resolved "https://github.com/angular/bazel-builds.git#b7599b1f8390b9ceeccae3ea976fcf6ede335396"
dependencies:
"@microsoft/api-extractor" "^7.24.2"
Expand All @@ -69,7 +68,6 @@

"@angular/build-tooling@https://github.com/angular/dev-infra-private-build-tooling-builds.git#df748cd6d28f098e8f12d0700cad1d3c08b3ccd7":
version "0.0.0-7eb6ad7da2d79cfe08019e6b6d4102ab752a94ca"
uid df748cd6d28f098e8f12d0700cad1d3c08b3ccd7
resolved "https://github.com/angular/dev-infra-private-build-tooling-builds.git#df748cd6d28f098e8f12d0700cad1d3c08b3ccd7"
dependencies:
"@angular/benchpress" "0.3.0"
Expand Down Expand Up @@ -268,7 +266,6 @@

"@angular/ng-dev@https://github.com/angular/dev-infra-private-ng-dev-builds.git#7e4c6ea5fa0ac493132345c97daef1737374702f":
version "0.0.0-7eb6ad7da2d79cfe08019e6b6d4102ab752a94ca"
uid "7e4c6ea5fa0ac493132345c97daef1737374702f"
resolved "https://github.com/angular/dev-infra-private-ng-dev-builds.git#7e4c6ea5fa0ac493132345c97daef1737374702f"
dependencies:
"@yarnpkg/lockfile" "^1.1.0"
Expand Down Expand Up @@ -1920,6 +1917,36 @@
dependencies:
call-bind "^1.0.7"

"@lmdb/lmdb-darwin-arm64@2.9.2":
version "2.9.2"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-2.9.2.tgz#da42cda48018eabfd678b9698e68a0102221b4ba"
integrity sha512-+GX51Fi8nZOrEXCFiQHnrCpKAzkfDA2sY5+M6Ry4wZEu711o2qlvg+7xXP+j7OT7+JsfB9ayGCdhra2AAaX02g==

"@lmdb/lmdb-darwin-x64@2.9.2":
version "2.9.2"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-2.9.2.tgz#fb24813693175a858727d61b3dc33c277a739987"
integrity sha512-ajkq2oZTd/RXXpgaZqVm6LHoJYf4A42q+S+U4gYKRYpeR4ERGvG+VGCK9bi9MXInQfeq0KM1yv6rsYpvCOoNhQ==

"@lmdb/lmdb-linux-arm64@2.9.2":
version "2.9.2"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-2.9.2.tgz#1fc10dfd165b7199b47c41ce9de99b22f46a8da4"
integrity sha512-WqQqWwFyL8JPVpKJyKnyyg7tnsVlD08PHEyxSMxDQC2EkPpvZuUz2oMqasDoy5tmYB0jANOI13/Qz3Mbh9endQ==

"@lmdb/lmdb-linux-arm@2.9.2":
version "2.9.2"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-2.9.2.tgz#2d4203c85e895cb75ffd6488791cc18de871a6b2"
integrity sha512-AAdmxDIh1tMYzXOUuDP+TNhvl9pLgvS63M6xhwgVArr79As4msraUSjIJ8J0jlhFKsN7nVoXzPB/jvpp8aK49w==

"@lmdb/lmdb-linux-x64@2.9.2":
version "2.9.2"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-2.9.2.tgz#dc0f71c092c005b2ad9ddd3502151ecce7692702"
integrity sha512-rB4tE80EOxXwTJr9rsATWZghOVP8+mV085P5u/dBdttJSq3TLxY0CMZ8NKB/WJpryNnsfCI4OvjOAibF/fg+GQ==

"@lmdb/lmdb-win32-x64@2.9.2":
version "2.9.2"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-2.9.2.tgz#84f561d05329c671f6e4119b483ce54410772bb6"
integrity sha512-VRrM/Zq/k8YEZlGuDvFi3NU753cm+vOa1kUcq4iNyeAEVXzjrSg5K3sHI0d6Od5gLsKctjlQeaFn6+21inU4bw==

"@material/animation@15.0.0-canary.7f224ddd4.0":
version "15.0.0-canary.7f224ddd4.0"
resolved "https://registry.yarnpkg.com/@material/animation/-/animation-15.0.0-canary.7f224ddd4.0.tgz#14b4f80718f9d405953dfca4376f9bcef609adc6"
Expand Down Expand Up @@ -2665,6 +2692,36 @@
resolved "https://registry.yarnpkg.com/@microsoft/tsdoc/-/tsdoc-0.14.2.tgz#c3ec604a0b54b9a9b87e9735dfc59e1a5da6a5fb"
integrity sha512-9b8mPpKrfeGRuhFH5iO1iwCLeIIsV6+H1sRfxbkoGXIyQE2BTsPd9zqSqQJ+pv5sJ/hT5M1zvOFL02MnEezFug==

"@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz#44d752c1a2dc113f15f781b7cc4f53a307e3fa38"
integrity sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==

"@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz#f954f34355712212a8e06c465bc06c40852c6bb3"
integrity sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==

"@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz#45c63037f045c2b15c44f80f0393fa24f9655367"
integrity sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==

"@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz#35707efeafe6d22b3f373caf9e8775e8920d1399"
integrity sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==

"@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz#091b1218b66c341f532611477ef89e83f25fae4f"
integrity sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==

"@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz#0f164b726869f71da3c594171df5ebc1c4b0a407"
integrity sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==

"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
Expand Down Expand Up @@ -6342,6 +6399,11 @@ destroy@~1.0.4:
resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80"
integrity sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==

detect-libc@^2.0.1:
version "2.0.2"
resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.2.tgz#8ccf2ba9315350e1241b88d0ac3b0e1fbd99605d"
integrity sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==

detect-node@^2.0.4:
version "2.1.0"
resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.1.0.tgz#c9c70775a49c3d03bc2c06d9a73be550f978f8b1"
Expand Down Expand Up @@ -9312,6 +9374,24 @@ lines-and-columns@^1.1.6:
resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==

lmdb@2.9.2:
version "2.9.2"
resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.9.2.tgz#a67ed24cad282ba7ad21daf2a8a13c08dcb33f56"
integrity sha512-Q5SQzu4u4sdz4U8QT1uCS04beS7hS/1YYb1suJwaijqVETGAkrPBKr0ERxTeza/u2F6ei5+8UTnzm4ae3PJG3w==
dependencies:
msgpackr "^1.9.9"
node-addon-api "^6.1.0"
node-gyp-build-optional-packages "5.1.1"
ordered-binary "^1.4.1"
weak-lru-cache "^1.2.2"
optionalDependencies:
"@lmdb/lmdb-darwin-arm64" "2.9.2"
"@lmdb/lmdb-darwin-x64" "2.9.2"
"@lmdb/lmdb-linux-arm" "2.9.2"
"@lmdb/lmdb-linux-arm64" "2.9.2"
"@lmdb/lmdb-linux-x64" "2.9.2"
"@lmdb/lmdb-win32-x64" "2.9.2"

loader-runner@^4.2.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.3.0.tgz#c1b4a163b99f614830353b16755e7149ac2314e1"
Expand Down Expand Up @@ -9882,6 +9962,27 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.2:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==

msgpackr-extract@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz#e05ec1bb4453ddf020551bcd5daaf0092a2c279d"
integrity sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==
dependencies:
node-gyp-build-optional-packages "5.0.7"
optionalDependencies:
"@msgpackr-extract/msgpackr-extract-darwin-arm64" "3.0.2"
"@msgpackr-extract/msgpackr-extract-darwin-x64" "3.0.2"
"@msgpackr-extract/msgpackr-extract-linux-arm" "3.0.2"
"@msgpackr-extract/msgpackr-extract-linux-arm64" "3.0.2"
"@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
"@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"

msgpackr@^1.9.9:
version "1.10.1"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555"
integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==
optionalDependencies:
msgpackr-extract "^3.0.2"

multicast-dns@^7.2.5:
version "7.2.5"
resolved "https://registry.yarnpkg.com/multicast-dns/-/multicast-dns-7.2.5.tgz#77eb46057f4d7adbd16d9290fa7299f6fa64cced"
Expand Down Expand Up @@ -9995,6 +10096,11 @@ node-addon-api@^3.0.0:
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==

node-addon-api@^6.1.0:
version "6.1.0"
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76"
integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==

node-fetch@2.6.7, node-fetch@cjs:
version "2.6.7"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
Expand All @@ -10014,6 +10120,18 @@ node-forge@^1:
resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==

node-gyp-build-optional-packages@5.0.7:
version "5.0.7"
resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz#5d2632bbde0ab2f6e22f1bbac2199b07244ae0b3"
integrity sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==

node-gyp-build-optional-packages@5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.1.1.tgz#52b143b9dd77b7669073cbfe39e3f4118bfc603c"
integrity sha512-+P72GAjVAbTxjjwUmwjVrqrdZROD4nf8KgpBoDxqXXTiYZZt/ud60dE5yvCSr9lRO8e8yv6kgJIC0K0PfZFVQw==
dependencies:
detect-libc "^2.0.1"

node-gyp-build@^4.2.2:
version "4.8.0"
resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.8.0.tgz#3fee9c1731df4581a3f9ead74664369ff00d26dd"
Expand Down Expand Up @@ -10558,6 +10676,11 @@ ora@5.4.1, ora@^5.1.0, ora@^5.4.1:
strip-ansi "^6.0.0"
wcwidth "^1.0.1"

ordered-binary@^1.4.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.5.1.tgz#94ccbf14181711081ee23931db0dc3f58aaa0df6"
integrity sha512-5VyHfHY3cd0iza71JepYG50My+YUbrFtGoUz2ooEydPyPM7Aai/JW098juLr+RG6+rDJuzNNTsEQu2DZa1A41A==

os-homedir@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3"
Expand Down Expand Up @@ -13512,6 +13635,11 @@ wcwidth@^1.0.0, wcwidth@^1.0.1:
dependencies:
defaults "^1.0.3"

weak-lru-cache@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19"
integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==

webdriver-js-extender@2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/webdriver-js-extender/-/webdriver-js-extender-2.1.0.tgz#57d7a93c00db4cc8d556e4d3db4b5db0a80c3bb7"
Expand Down

0 comments on commit 1e98f72

Please sign in to comment.