Parse AST in worker

.eslintrc.js

@@ -116,12 +116,13 @@ module.exports = {
 				// 'fsevents' is only available on macOS, and not installed on linux/windows
 				ignore: [
 					'fsevents',
-					'help.md',
+					'help\\.md',
 					'is-reference',
 					'package.json',
 					'types',
-					'examples.json',
-					'locate-character'
+					'examples\\.json',
+					'locate-character',
+					'^emit:'
 				]
 			}
 		],

browser/src/parseAstAsync.ts

@@ -0,0 +1,11 @@
+import type { AstNode, ParseAst } from '../../src/rollup/types';
+import { parseAst } from '../../src/utils/parseAst';
+
+// We could also do things in a web worker here, but that would make bundling
+// the browser build much more complicated
+const parseAstAsync = async (
+	code: Parameters<ParseAst>[0],
+	options?: Parameters<ParseAst>[1]
+): Promise<AstNode> => new Promise(resolve => resolve(parseAst(code, options)));
+
+export const getParseAstAsync = () => parseAstAsync;

build-plugins/emit-file.ts

@@ -0,0 +1,33 @@
+import type { Plugin } from 'rollup';
+
+const EMIT_PREFIX = 'emit:';
+
+export default function emitFile(): Plugin {
+	return {
+		load(id) {
+			if (id.startsWith(EMIT_PREFIX)) {
+				const entryId = id.slice(EMIT_PREFIX.length);
+				const referenceId = this.emitFile({
+					id: entryId,
+					type: 'chunk'
+				});
+				return `export default import.meta.ROLLUP_FILE_URL_${referenceId};`;
+			}
+		},
+		name: 'emit-file',
+		resolveFileUrl({ moduleId, relativePath, format }) {
+			if (moduleId.startsWith(EMIT_PREFIX)) {
+				return format === 'cjs'
+					? `__dirname + '/${relativePath}'`
+					: `new URL('${relativePath}', import.meta.url)`;
+			}
+		},
+		async resolveId(source, importer) {
+			if (source.startsWith(EMIT_PREFIX)) {
+				return `${EMIT_PREFIX}${
+					(await this.resolve(source.slice(EMIT_PREFIX.length), importer))!.id
+				}`;
+			}
+		}
+	};
+}

build-plugins/replace-browser-modules.ts

@@ -12,7 +12,8 @@ const JS_REPLACED_MODULES = [
 	'performance',
 	'process',
 	'resolveId',
-	'initWasm'
+	'initWasm',
+	'parseAstAsync'
 ];
 
 type ModulesMap = [string, string][];
@@ -41,7 +42,7 @@ export default function replaceBrowserModules(): Plugin & RollupPlugin {
 			}
 		},
 		transformIndexHtml(html) {
-			// Unfortunately, picomatch sneaks as a dedendency into the dev bundle.
+			// Unfortunately, picomatch sneaks as a dependency into the dev bundle.
 			// This fixes an error.
 			return html.replace('</head>', '<script>window.process={}</script></head>');
 		}

docs/.vitepress/config.ts

@@ -1,4 +1,5 @@
 import alias from '@rollup/plugin-alias';
+import type { Plugin } from 'vite';
 import { defineConfig } from 'vitepress';
 import { moduleAliases } from '../../build-plugins/aliases';
 import replaceBrowserModules from '../../build-plugins/replace-browser-modules';
@@ -147,7 +148,7 @@ export default defineConfig({
 				}
 			},
 			examplesPlugin(),
-			alias(moduleAliases)
+			alias(moduleAliases) as unknown as Plugin
 		]
 	}
 });

docs/repl/components/ReplEditor.vue

@@ -56,7 +56,7 @@ onMounted(async () => {
 			const relevantMessages = messages.filter(
 				(message): message is RollupLog & { pos: number } =>
 					typeof message.pos === 'number' &&
-					getFileNameFromMessage(message) === properties.moduleName
+					getFileNameFromMessage(message) === `/${properties.moduleName}`
 			);
 			editor.dispatch({ effects: [addWarningsEffect.of({ messages: relevantMessages, type })] });
 		};

rollup.config.ts

@@ -12,6 +12,7 @@ import addCliEntry from './build-plugins/add-cli-entry';
 import { moduleAliases } from './build-plugins/aliases';
 import cleanBeforeWrite from './build-plugins/clean-before-write';
 import { copyBrowserTypes, copyNodeTypes } from './build-plugins/copy-types';
+import emitFile from './build-plugins/emit-file';
 import emitModulePackageFile from './build-plugins/emit-module-package-file';
 import { emitNativeEntry } from './build-plugins/emit-native-entry';
 import emitWasmFile from './build-plugins/emit-wasm-file';
@@ -50,7 +51,8 @@ const nodePlugins: readonly Plugin[] = [
 	}),
 	typescript(),
 	cleanBeforeWrite('dist'),
-	externalNativeImport()
+	externalNativeImport(),
+	emitFile()
 ];
 
 export default async function (

src/Graph.ts

@@ -25,6 +25,7 @@ import {
 	logImplicitDependantIsNotIncluded,
 	logMissingExport
 } from './utils/logs';
+import { getParseAstAsync } from './utils/parseAstAsync';
 import type { PureFunctions } from './utils/pureFunctions';
 import { getPureFunctions } from './utils/pureFunctions';
 import { timeEnd, timeStart } from './utils/timers';
@@ -60,6 +61,7 @@ export default class Graph {
 	readonly moduleLoader: ModuleLoader;
 	readonly modulesById = new Map<string, Module | ExternalModule>();
 	needsTreeshakingPass = false;
+	readonly parseAstAsync = getParseAstAsync();
 	phase: BuildPhase = BuildPhase.LOAD_AND_PARSE;
 	readonly pluginDriver: PluginDriver;
 	readonly pureFunctions: PureFunctions;

src/Module.ts

@@ -785,7 +785,7 @@ export default class Module {
 		return { source, usesTopLevelAwait };
 	}
 
-	setSource({
+	async setSource({
 		ast,
 		code,
 		customTransformCache,
@@ -799,7 +799,7 @@ export default class Module {
 	}: TransformModuleJSON & {
 		resolvedIds?: ResolvedIdMap;
 		transformFiles?: EmittedFile[] | undefined;
-	}): void {
+	}): Promise<void> {
 		if (code.startsWith('#!')) {
 			const shebangEndPosition = code.indexOf('\n');
 			this.shebang = code.slice(2, shebangEndPosition);
@@ -831,7 +831,7 @@ export default class Module {
 		this.transformDependencies = transformDependencies;
 		this.customTransformCache = customTransformCache;
 		this.updateOptions(moduleOptions);
-		const moduleAst = ast ?? this.tryParse();
+		const moduleAst = ast ?? (await this.tryParseAsync());
 
 		timeEnd('generate ast', 3);
 		timeStart('analyze ast', 3);
@@ -1334,6 +1334,14 @@ export default class Module {
 			return this.error(logModuleParseError(error_, this.id), error_.pos);
 		}
 	}
+
+	private async tryParseAsync(): Promise<ProgramAst> {
+		try {
+			return (await this.graph.parseAstAsync(this.info.code!)) as ProgramAst;
+		} catch (error_: any) {
+			return this.error(logModuleParseError(error_, this.id), error_.pos);
+		}
+	}
 }
 
 // if there is a cyclic import in the reexport chain, we should not

src/ModuleLoader.ts

@@ -245,17 +245,23 @@ export class ModuleLoader {
 				async entryModule => {
 					addChunkNamesToModule(entryModule, unresolvedModule, false, chunkNamePriority);
 					if (!entryModule.info.isEntry) {
-						this.implicitEntryModules.add(entryModule);
 						const implicitlyLoadedAfterModules = await Promise.all(
 							implicitlyLoadedAfter.map(id =>
 								this.loadEntryModule(id, false, unresolvedModule.importer, entryModule.id)
 							)
 						);
-						for (const module of implicitlyLoadedAfterModules) {
-							entryModule.implicitlyLoadedAfter.add(module);
-						}
-						for (const dependant of entryModule.implicitlyLoadedAfter) {
-							dependant.implicitlyLoadedBefore.add(entryModule);
+						// We need to check again if this is still an entry module as these
+						// changes need to be performed atomically to avoid race conditions
+						// if the same module is re-emitted as an entry module.
+						// The inverse changes happen in "handleExistingModule"
+						if (!entryModule.info.isEntry) {
+							this.implicitEntryModules.add(entryModule);
+							for (const module of implicitlyLoadedAfterModules) {
+								entryModule.implicitlyLoadedAfter.add(module);
+							}
+							for (const dependant of entryModule.implicitlyLoadedAfter) {
+								dependant.implicitlyLoadedBefore.add(entryModule);
+							}
 						}
 					}
 					return entryModule;
@@ -311,10 +317,10 @@ export class ModuleLoader {
 				for (const emittedFile of cachedModule.transformFiles)
 					this.pluginDriver.emitFile(emittedFile);
 			}
-			module.setSource(cachedModule);
+			await module.setSource(cachedModule);
 		} else {
 			module.updateOptions(sourceDescription);
-			module.setSource(
+			await module.setSource(
 				await transform(sourceDescription, module, this.pluginDriver, this.options.onLog)
 			);
 		}
@@ -615,6 +621,8 @@ export class ModuleLoader {
 				: loadPromise;
 		}
 		if (isEntry) {
+			// This reverts the changes in addEntryWithImplicitDependants and needs to
+			// be performed atomically
 			module.info.isEntry = true;
 			this.implicitEntryModules.delete(module);
 			for (const dependant of module.implicitlyLoadedAfter) {

src/utils/logs.ts

@@ -824,7 +824,8 @@ export function logModuleParseError(error: Error, moduleId: string): RollupLog {
 		cause: error,
 		code: PARSE_ERROR,
 		id: moduleId,
-		message
+		message,
+		stack: error.stack
 	};
 }
 

src/utils/parseAstAsync.ts

@@ -0,0 +1,58 @@
+import { Worker } from 'node:worker_threads';
+import parseWorkerUrl from 'emit:./parseAstWorkerEntry.ts';
+import type { AstNode, ParseAst } from '../rollup/types';
+import { convertProgram } from './convert-ast';
+import getReadStringFunction from './getReadStringFunction';
+
+type MessageResolvers = Map<number, [(buffer: Buffer) => void, (error: unknown) => void]>;
+
+export const getParseAstAsync = () => {
+	let workerWithResolvers: { worker: Worker; resolvers: MessageResolvers } | null = null;
+	let nextId = 0;
+
+	const parseToBuffer = (code: string, allowReturnOutsideFunction: boolean): Promise<Buffer> => {
+		if (!workerWithResolvers) {
+			const worker = new Worker(parseWorkerUrl);
+			const resolvers: MessageResolvers = new Map();
+
+			worker.on('message', ([id, buffer]: [id: number, buffer: Buffer]) => {
+				resolvers.get(id)![0](buffer);
+				resolvers.delete(id);
+				if (resolvers.size === 0) {
+					// We wait one macro task tick to no close the worker if there are
+					// additional tasks directly queued up
+					setTimeout(async () => {
+						if (resolvers.size === 0) {
+							workerWithResolvers = null;
+							await worker.terminate();
+						}
+					});
+				}
+			});
+
+			worker.on('error', error => {
+				if (workerWithResolvers?.worker === worker) {
+					workerWithResolvers = null;
+					for (const [, reject] of resolvers.values()) {
+						reject(error);
+					}
+					resolvers.clear();
+				}
+			});
+			workerWithResolvers = { resolvers, worker };
+		}
+		const id = nextId++;
+		return new Promise<Buffer>((resolve, reject) => {
+			workerWithResolvers!.resolvers.set(id, [resolve, reject]);
+			workerWithResolvers!.worker.postMessage([id, code, allowReturnOutsideFunction]);
+		});
+	};
+
+	return async (
+		code: Parameters<ParseAst>[0],
+		{ allowReturnOutsideFunction = false }: Parameters<ParseAst>[1] = {}
+	): Promise<AstNode> => {
+		const astBuffer = await parseToBuffer(code, allowReturnOutsideFunction);
+		return convertProgram(astBuffer.buffer, getReadStringFunction(astBuffer));
+	};
+};

src/utils/parseAstWorkerEntry.ts

@@ -0,0 +1,14 @@
+import { parentPort } from 'node:worker_threads';
+import { parse } from '../../native';
+
+parentPort!.on(
+	'message',
+	([id, code, allowReturnOutsideFunction]: [
+		id: number,
+		code: string,
+		allowReturnOutsideFunction: boolean
+	]) => {
+		const buffer = parse(code, allowReturnOutsideFunction);
+		parentPort!.postMessage([id, buffer], [buffer.buffer]);
+	}
+);

typings/declarations.d.ts

@@ -4,6 +4,11 @@ declare module 'help.md' {
 	export default value;
 }
 
+declare module 'emit:*' {
+	const value: string;
+	export default value;
+}
+
 // external libs
 declare module 'rollup-plugin-string' {
 	import type { PluginImpl } from 'rollup';