Support partial manifests in build output (#15388)

Author: jamesopstad

.changeset/resolve-partial-build-output-manifests.md

@@ -0,0 +1,8 @@
+---
+"@cloudflare/build-output-utils": minor
+"@cloudflare/config": minor
+---
+
+Support partial manifests in the experimental Build Output Specification
+
+Build output producers now declare whether their module inventory is complete. `readBuildOutput()` resolves partial manifests by discovering `.js`, `.mjs`, and `.map` files while preserving explicit module type overrides.

packages/build-output-utils/src/__tests__/read.test.ts

@@ -14,7 +14,8 @@ import { readBuildOutput } from "../read";
 import { writeSettingsConfig, writeWorkerConfig } from "../write";
 import type { ParsedOutputWorkerConfig } from "@cloudflare/config";
 
-const manifest: ParsedOutputWorkerConfig["manifest"] = {
+const completeManifest: ParsedOutputWorkerConfig["manifest"] = {
+	type: "complete",
 	mainModule: "index.js",
 	modules: { "index.js": { type: "esm" } },
 };
@@ -34,6 +35,18 @@ function inputWorkerConfig(name: string) {
 	});
 }
 
+async function writeBundleFiles(
+	root: string,
+	files: Record<string, string>
+): Promise<void> {
+	const bundleDir = getWorkerBundleDir(root);
+	for (const [fileName, contents] of Object.entries(files)) {
+		const filePath = path.join(bundleDir, fileName);
+		await fsp.mkdir(path.dirname(filePath), { recursive: true });
+		await fsp.writeFile(filePath, contents);
+	}
+}
+
 /**
  * Seed a Worker into the Build Output Specification tree, optionally creating
  * the `bundle/` and `assets/` directories on disk.
@@ -62,7 +75,7 @@ async function seedWorker(
 	await writeWorkerConfig({
 		root,
 		config: inputWorkerConfig(name),
-		manifest: hasBundle ? manifest : undefined,
+		manifest: hasBundle ? completeManifest : undefined,
 		workerDirectoryName,
 	});
 	if (bundleDir) {
@@ -95,7 +108,7 @@ describe("readBuildOutput", () => {
 		expect(output.workers.default.assetsDir).toBeUndefined();
 
 		expect(output.workers.default.config.name).toBe("my-worker");
-		expect(output.workers.default.config.manifest).toEqual(manifest);
+		expect(output.workers.default.config.manifest).toEqual(completeManifest);
 		expect(output.workers.default.config).not.toHaveProperty("entrypoint");
 	});
 
@@ -116,6 +129,103 @@ describe("readBuildOutput", () => {
 		);
 	});
 
+	it("resolves a partial manifest from bundle files and explicit overrides", async ({
+		expect,
+	}) => {
+		const root = process.cwd();
+		await seedWorker(root);
+		await writeWorkerConfig({
+			root,
+			config: inputWorkerConfig("my-worker"),
+			manifest: {
+				type: "partial",
+				mainModule: "index.js",
+				modules: {
+					"chunks/worker.mjs": { type: "cjs" },
+					"data.txt": { type: "text" },
+				},
+			},
+		});
+		await writeBundleFiles(root, {
+			"index.js": "module.exports = {};",
+			"chunks/worker.mjs": "export default {};",
+			"chunks/worker.mjs.map": "{}",
+			"data.txt": "data",
+			"ignored.json": "{}",
+		});
+
+		const { manifest: resolvedManifest } = (await readBuildOutput(root)).workers
+			.default.config;
+
+		expect(resolvedManifest).toEqual({
+			type: "complete",
+			mainModule: "index.js",
+			modules: {
+				"chunks/worker.mjs": { type: "cjs" },
+				"chunks/worker.mjs.map": { type: "sourcemap" },
+				"data.txt": { type: "text" },
+				"index.js": { type: "esm" },
+			},
+		});
+	});
+
+	it("does not scan bundle files for a complete manifest", async ({
+		expect,
+	}) => {
+		const root = process.cwd();
+		await seedWorker(root);
+		await writeBundleFiles(root, {
+			"index.js": "export default {};",
+			"unlisted.js": "export default {};",
+		});
+
+		const { manifest: resolvedManifest } = (await readBuildOutput(root)).workers
+			.default.config;
+
+		expect(resolvedManifest).toEqual(completeManifest);
+	});
+
+	it("throws when a partial manifest's main module cannot be resolved", async ({
+		expect,
+	}) => {
+		const root = process.cwd();
+		await seedWorker(root);
+		await writeWorkerConfig({
+			root,
+			config: inputWorkerConfig("my-worker"),
+			manifest: {
+				type: "partial",
+				mainModule: "missing.js",
+				modules: {},
+			},
+		});
+
+		await expect(readBuildOutput(root)).rejects.toThrow(
+			/partial manifest .* has main module "missing\.js", but it was not found as an ES module/
+		);
+	});
+
+	it("throws when a partial manifest's main module is a source map", async ({
+		expect,
+	}) => {
+		const root = process.cwd();
+		await seedWorker(root);
+		await writeWorkerConfig({
+			root,
+			config: inputWorkerConfig("my-worker"),
+			manifest: {
+				type: "partial",
+				mainModule: "index.js.map",
+				modules: {},
+			},
+		});
+		await writeBundleFiles(root, { "index.js.map": "{}" });
+
+		await expect(readBuildOutput(root)).rejects.toThrow(
+			/partial manifest .* has main module "index\.js\.map", but it was not found as an ES module/
+		);
+	});
+
 	it("resolves the assets directory when present and leaves bundle undefined for assets-only Workers", async ({
 		expect,
 	}) => {

packages/build-output-utils/src/__tests__/write.test.ts

@@ -97,9 +97,10 @@ describe("writeWorkerConfig", () => {
 	}) => {
 		const root = process.cwd();
 		const manifest = {
+			type: "complete",
 			mainModule: "index.js",
-			modules: { "index.js": { type: "esm" as const } },
-		};
+			modules: { "index.js": { type: "esm" } },
+		} as const;
 
 		await writeWorkerConfig({ root, config: parsedWorkerConfig, manifest });
 

packages/build-output-utils/src/index.ts

@@ -22,4 +22,5 @@ export type {
 	BuildOutput,
 	BuildOutputWorker,
 	BuildOutputWorkers,
+	ResolvedOutputWorkerConfig,
 } from "./read";

packages/build-output-utils/src/read.ts

@@ -1,5 +1,6 @@
 import * as fs from "node:fs";
 import * as fsp from "node:fs/promises";
+import * as path from "node:path";
 import { OutputSettingsSchema, OutputWorkerSchema } from "@cloudflare/config";
 import { BuildOutputError } from "./errors";
 import {
@@ -12,15 +13,33 @@ import {
 	getWorkersDir,
 } from "./paths";
 import type {
+	ModuleType,
 	ParsedOutputSettingsConfig,
 	ParsedOutputWorkerConfig,
 } from "@cloudflare/config";
 
+type ManifestModules = NonNullable<
+	ParsedOutputWorkerConfig["manifest"]
+>["modules"];
+
+type CompleteManifest = Omit<
+	NonNullable<ParsedOutputWorkerConfig["manifest"]>,
+	"type"
+> & { type: "complete" };
+
+/** A schema-validated Worker config whose manifest has been fully resolved. */
+export type ResolvedOutputWorkerConfig = Omit<
+	ParsedOutputWorkerConfig,
+	"manifest"
+> & {
+	manifest?: CompleteManifest;
+};
+
 interface BuildOutputWorkerBase {
 	/** Absolute path to the Worker's `config.json`. */
 	configPath: string;
-	/** The parsed, schema-validated Worker config, including its `manifest`. */
-	config: ParsedOutputWorkerConfig;
+	/** The parsed Worker config, including its fully resolved `manifest`. */
+	config: ResolvedOutputWorkerConfig;
 }
 
 /**
@@ -81,7 +100,8 @@ export interface BuildOutput {
  *
  * Reads the optional top-level settings `config.json`, then reads and
  * schema-validates the Worker's `config.json` and resolves its
- * `bundle/` / `assets/` directories.
+ * `bundle/` / `assets/` directories. Partial manifests are resolved into
+ * complete manifests using the files in `bundle/`.
  *
  * @throws {BuildOutputError} if the top-level `config.json` is invalid, or if
  * the Worker config is missing, is not valid JSON, or fails schema validation.
@@ -155,18 +175,20 @@ async function readWorker(
 	}
 
 	if (hasBundleDir) {
+		const config = await resolveManifest(result.data, configPath, bundleDir);
 		return {
 			configPath,
-			config: result.data,
+			config,
 			bundleDir,
 			assetsDir: hasAssetsDir ? assetsDir : undefined,
 		};
 	}
 
 	if (hasAssetsDir) {
+		const { manifest: _manifest, ...config } = result.data;
 		return {
 			configPath,
-			config: result.data,
+			config,
 			bundleDir: undefined,
 			assetsDir,
 		};
@@ -177,6 +199,78 @@ async function readWorker(
 	);
 }
 
+/** Resolve the manifest against the bundle contents. */
+async function resolveManifest(
+	config: ParsedOutputWorkerConfig,
+	configPath: string,
+	bundleDir: string
+): Promise<ResolvedOutputWorkerConfig> {
+	const { manifest, ...workerConfig } = config;
+	if (manifest === undefined) {
+		return workerConfig;
+	}
+	if (manifest.type === "complete") {
+		return {
+			...workerConfig,
+			manifest: { ...manifest, type: "complete" },
+		};
+	}
+
+	const inferredModules = await scanModules(bundleDir);
+	if (inferredModules[manifest.mainModule]?.type !== "esm") {
+		throw new BuildOutputError(
+			`partial manifest at ${configPath} has main module "${manifest.mainModule}", but it was not found as an ES module in the bundle.`
+		);
+	}
+
+	const modules = {
+		...inferredModules,
+		...manifest.modules,
+	};
+
+	return {
+		...workerConfig,
+		manifest: {
+			type: "complete",
+			mainModule: manifest.mainModule,
+			modules,
+		},
+	};
+}
+
+/** Infer JavaScript modules and source maps from a bundle directory. */
+async function scanModules(bundleDir: string): Promise<ManifestModules> {
+	const modules: ManifestModules = {};
+	const entries = await fsp.readdir(bundleDir, {
+		recursive: true,
+		withFileTypes: true,
+	});
+
+	for (const entry of entries) {
+		const type = entry.isFile() ? inferModuleType(entry.name) : undefined;
+		if (type !== undefined) {
+			const modulePath = path
+				.relative(bundleDir, path.join(entry.parentPath, entry.name))
+				.split(path.sep)
+				.join("/");
+			modules[modulePath] = { type };
+		}
+	}
+
+	return modules;
+}
+
+/** Infer the module type for extensions supported by partial manifests. */
+function inferModuleType(modulePath: string): ModuleType | undefined {
+	switch (path.extname(modulePath)) {
+		case ".js":
+		case ".mjs":
+			return "esm";
+		case ".map":
+			return "sourcemap";
+	}
+}
+
 /**
  * Read and schema-validate the optional top-level `config.json` holding the
  * project-level settings shared by every Worker, including the mode the build