feat: enhance sourcemap uploader with batching and error handling options

LuggaPugga · LuggaPugga · commit bd0e18050c23 · 2026-03-02T17:58:18.000+01:00
diff --git a/.changeset/backend-ingest-body-limit.md b/.changeset/backend-ingest-body-limit.md
@@ -0,0 +1,5 @@
+---
+"@sourcemaps/backend": patch
+---
+
+Increase the public sourcemap ingest request body limit to 50MB to avoid 413 errors when uploading larger sourcemap payloads.
diff --git a/.changeset/plugin-batching-fail-on-error.md b/.changeset/plugin-batching-fail-on-error.md
@@ -0,0 +1,5 @@
+---
+"@faststats/sourcemap-uploader-plugin": patch
+---
+
+Add upload payload batching with a configurable `maxUploadBodyBytes` limit and introduce a `failOnError` option to control whether upload failures should fail the build.
diff --git a/apps/backend/.env.example b/apps/backend/.env.example
@@ -9,6 +9,6 @@ S3_ACCESS_KEY_ID=
 S3_SECRET_ACCESS_KEY=
 HOST=0.0.0.0
 PORT=3000
-INTERNAL_HOST=127.0.0.1
+INTERNAL_HOST=[::]
 INTERNAL_PORT=9485
 ADMIN_TOKEN=replace-with-long-random-token
diff --git a/apps/backend/src/config.rs b/apps/backend/src/config.rs
@@ -23,7 +23,7 @@ impl Config {
             .unwrap_or_else(|_| "3000".into())
             .parse()
             .expect("PORT must be a valid u16");
-        let internal_host = std::env::var("INTERNAL_HOST").unwrap_or_else(|_| "127.0.0.1".into());
+        let internal_host = std::env::var("INTERNAL_HOST").unwrap_or_else(|_| "[::]".into());
         let internal_port: u16 = std::env::var("INTERNAL_PORT")
             .unwrap_or_else(|_| "3001".into())
             .parse()
@@ -46,7 +46,7 @@ impl Config {
             s3_secret_access_key: std::env::var("S3_SECRET_ACCESS_KEY")?,
             listen_addr: SocketAddr::new(host.parse().expect("HOST must be a valid IP"), port),
             internal_listen_addr: SocketAddr::new(
-                internal_host
+                normalize_ip_literal(&internal_host)
                     .parse()
                     .expect("INTERNAL_HOST must be a valid IP"),
                 internal_port,
@@ -55,3 +55,9 @@ impl Config {
         })
     }
 }
+
+fn normalize_ip_literal(host: &str) -> &str {
+    host.strip_prefix('[')
+        .and_then(|h| h.strip_suffix(']'))
+        .unwrap_or(host)
+}
diff --git a/apps/backend/src/routes.rs b/apps/backend/src/routes.rs
@@ -13,7 +13,7 @@ use crate::SharedState;
 use crate::auth::{AdminAuthenticatedProject, AuthenticatedProject};
 use crate::error::AppError;
 
-const INGEST_MAX_BODY_BYTES: usize = 50 * 1024 * 1024; // 50MB
+const INGEST_MAX_BODY_BYTES: usize = 50 * 1024 * 1024;
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
diff --git a/packages/bundler-plugin/src/index.ts b/packages/bundler-plugin/src/index.ts
@@ -10,6 +10,7 @@ import type {
 import { createUnplugin } from "unplugin";
 
 const DEFAULT_ENDPOINT = "https://sourcemaps.faststats.dev/api/sourcemaps";
+const DEFAULT_MAX_UPLOAD_BODY_BYTES = 50 * 1024 * 1024;
 
 type BundlerName =
 	| "vite"
@@ -87,6 +88,8 @@ export type BundlerPluginOptions = {
 	endpoint?: string;
 	authToken?: string;
 	buildId?: string;
+	maxUploadBodyBytes?: number;
+	failOnError?: boolean;
 	deleteAfterUpload?: boolean;
 	globalKey?: string;
 	fetchImpl?: typeof fetch;
@@ -180,6 +183,72 @@ const postSourcemaps = async (
 	}
 };
 
+const payloadSizeBytes = (payload: SourcemapUploadPayload): number =>
+	Buffer.byteLength(JSON.stringify(payload), "utf8");
+
+const createUploadBatches = (
+	buildId: string,
+	bundler: BundlerName,
+	sourcemaps: SourcemapUpload[],
+	maxUploadBodyBytes: number,
+): SourcemapUploadPayload[] => {
+	if (!Number.isFinite(maxUploadBodyBytes) || maxUploadBodyBytes <= 0) {
+		throw new Error("maxUploadBodyBytes must be a positive number");
+	}
+
+	const uploadedAt = new Date().toISOString();
+	const batches: SourcemapUploadPayload[] = [];
+	let currentBatch: SourcemapUpload[] = [];
+
+	const toPayload = (batch: SourcemapUpload[]): SourcemapUploadPayload => ({
+		buildId,
+		bundler,
+		uploadedAt,
+		sourcemaps: batch,
+	});
+
+	for (const sourcemap of sourcemaps) {
+		const nextBatch = [...currentBatch, sourcemap];
+		const nextPayload = toPayload(nextBatch);
+
+		if (payloadSizeBytes(nextPayload) <= maxUploadBodyBytes) {
+			currentBatch = nextBatch;
+			continue;
+		}
+
+		if (currentBatch.length === 0) {
+			throw new Error(
+				`Sourcemap "${sourcemap.fileName}" exceeds maxUploadBodyBytes limit`,
+			);
+		}
+
+		batches.push(toPayload(currentBatch));
+		currentBatch = [sourcemap];
+
+		if (payloadSizeBytes(toPayload(currentBatch)) > maxUploadBodyBytes) {
+			throw new Error(
+				`Sourcemap "${sourcemap.fileName}" exceeds maxUploadBodyBytes limit`,
+			);
+		}
+	}
+
+	if (currentBatch.length > 0) {
+		batches.push(toPayload(currentBatch));
+	}
+
+	return batches;
+};
+
+const handleUploadError = async (
+	options: BundlerPluginOptions,
+	error: unknown,
+): Promise<void> => {
+	await options.onUploadError?.(error);
+	if (options.failOnError ?? true) {
+		throw error;
+	}
+};
+
 const deleteFiles = async (
 	baseDir: string,
 	fileNames: string[],
@@ -293,15 +362,16 @@ const uploadAndMaybeDelete = async (
 		return;
 	}
 
-	const payload: SourcemapUploadPayload = {
+	const batches = createUploadBatches(
 		buildId,
 		bundler,
-		uploadedAt: new Date().toISOString(),
 		sourcemaps,
-	};
-
-	await postSourcemaps(options, payload);
-	await options.onUploadSuccess?.(payload);
+		options.maxUploadBodyBytes ?? DEFAULT_MAX_UPLOAD_BODY_BYTES,
+	);
+	for (const payload of batches) {
+		await postSourcemaps(options, payload);
+		await options.onUploadSuccess?.(payload);
+	}
 
 	if (options.deleteAfterUpload && baseDirForDeletion) {
 		await deleteFiles(
@@ -347,8 +417,7 @@ const rollupWriteBundle = (
 				outputDir,
 			);
 		} catch (error) {
-			await options.onUploadError?.(error);
-			throw error;
+			await handleUploadError(options, error);
 		}
 	};
 
@@ -419,8 +488,7 @@ const applyWebpackLikeHooks = (
 						}
 					}
 				} catch (error) {
-					await options.onUploadError?.(error);
-					throw error;
+					await handleUploadError(options, error);
 				}
 			},
 		);
@@ -522,8 +590,7 @@ const unpluginInstance = createUnplugin<BundlerPluginOptions>(
 								outdir,
 							);
 						} catch (error) {
-							await options.onUploadError?.(error);
-							throw error;
+							await handleUploadError(options, error);
 						}
 					});
 				},
@@ -553,8 +620,7 @@ const unpluginInstance = createUnplugin<BundlerPluginOptions>(
 						outputDir,
 					);
 				} catch (error) {
-					await options.onUploadError?.(error);
-					throw error;
+					await handleUploadError(options, error);
 				}
 			},
 		};
diff --git a/packages/bundler-plugin/tests/plugin.test.ts b/packages/bundler-plugin/tests/plugin.test.ts
@@ -4,9 +4,10 @@ import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { rspack } from "@rspack/core";
 import { build as esbuildBuild } from "esbuild";
+import type { NormalizedOutputOptions, OutputBundle } from "rollup";
 import { rollup as createRollupBundle } from "rollup";
 import { build as viteBuild } from "vite";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import webpack from "webpack";
 import bunPlugin from "../src/bun";
 import esbuildPlugin from "../src/esbuild";
@@ -273,6 +274,166 @@ describe("sourcemaps bundler plugin", () => {
 		await rm(cwd, { recursive: true, force: true });
 	});
 
+	it("batches uploads when payload exceeds maxUploadBodyBytes", async () => {
+		const postedPayloads: UploadPayload[] = [];
+		const fetchImpl = (async (
+			_input: Parameters<typeof fetch>[0],
+			init?: Parameters<typeof fetch>[1],
+		) => {
+			const body = init?.body;
+			if (!body || typeof body !== "string") {
+				throw new Error("Expected JSON string body");
+			}
+			postedPayloads.push(JSON.parse(body) as UploadPayload);
+			return new Response(JSON.stringify({ ok: true }), { status: 200 });
+		}) as typeof fetch;
+		const plugin = firstPlugin(
+			rollupPlugin({
+				endpoint,
+				buildId: "batch-build-id",
+				maxUploadBodyBytes: 600,
+				fetchImpl,
+			}),
+		) as {
+			writeBundle?: (
+				outputOptions: NormalizedOutputOptions,
+				bundle: OutputBundle,
+			) => Promise<void>;
+		};
+
+		if (!plugin.writeBundle) {
+			throw new Error("Expected rollup writeBundle hook");
+		}
+
+		const bundle = {
+			"first.js.map": {
+				type: "asset",
+				fileName: "first.js.map",
+				source: JSON.stringify({ version: 3, mappings: "AAAA".repeat(35) }),
+			},
+			"second.js.map": {
+				type: "asset",
+				fileName: "second.js.map",
+				source: JSON.stringify({ version: 3, mappings: "BBBB".repeat(35) }),
+			},
+			"third.js.map": {
+				type: "asset",
+				fileName: "third.js.map",
+				source: JSON.stringify({ version: 3, mappings: "CCCC".repeat(35) }),
+			},
+		} as unknown as OutputBundle;
+
+		await plugin.writeBundle(
+			{ dir: process.cwd() } as NormalizedOutputOptions,
+			bundle,
+		);
+
+		expect(postedPayloads.length).toBeGreaterThan(1);
+		expect(
+			postedPayloads.every((payload) => payload.buildId === "batch-build-id"),
+		).toBe(true);
+		expect(
+			postedPayloads.every((payload) => payload.bundler === "rollup"),
+		).toBe(true);
+		expect(
+			postedPayloads.every((payload) => payload.sourcemaps.length > 0),
+		).toBe(true);
+		expect(
+			postedPayloads.flatMap((payload) =>
+				payload.sourcemaps.map((sourcemap) => sourcemap.fileName),
+			),
+		).toEqual(["first.js.map", "second.js.map", "third.js.map"]);
+	});
+
+	it("throws on upload error by default", async () => {
+		const onUploadError = vi.fn();
+		const fetchImpl = (async (
+			_input: Parameters<typeof fetch>[0],
+			_init?: Parameters<typeof fetch>[1],
+		) =>
+			new Response(JSON.stringify({ ok: false }), {
+				status: 500,
+			})) as unknown as typeof fetch;
+		const plugin = firstPlugin(
+			rollupPlugin({
+				endpoint,
+				buildId: "fail-default-build-id",
+				fetchImpl,
+				onUploadError,
+			}),
+		) as {
+			writeBundle?: (
+				outputOptions: NormalizedOutputOptions,
+				bundle: OutputBundle,
+			) => Promise<void>;
+		};
+
+		if (!plugin.writeBundle) {
+			throw new Error("Expected rollup writeBundle hook");
+		}
+
+		const bundle = {
+			"bundle.js.map": {
+				type: "asset",
+				fileName: "bundle.js.map",
+				source: JSON.stringify({ version: 3, mappings: "AAAA" }),
+			},
+		} as unknown as OutputBundle;
+
+		await expect(
+			plugin.writeBundle(
+				{ dir: process.cwd() } as NormalizedOutputOptions,
+				bundle,
+			),
+		).rejects.toThrow("Sourcemap upload failed with status 500");
+		expect(onUploadError).toHaveBeenCalledTimes(1);
+	});
+
+	it("does not throw on upload error when failOnError is false", async () => {
+		const onUploadError = vi.fn();
+		const fetchImpl = (async (
+			_input: Parameters<typeof fetch>[0],
+			_init?: Parameters<typeof fetch>[1],
+		) =>
+			new Response(JSON.stringify({ ok: false }), {
+				status: 500,
+			})) as unknown as typeof fetch;
+		const plugin = firstPlugin(
+			rollupPlugin({
+				endpoint,
+				buildId: "fail-soft-build-id",
+				fetchImpl,
+				failOnError: false,
+				onUploadError,
+			}),
+		) as {
+			writeBundle?: (
+				outputOptions: NormalizedOutputOptions,
+				bundle: OutputBundle,
+			) => Promise<void>;
+		};
+
+		if (!plugin.writeBundle) {
+			throw new Error("Expected rollup writeBundle hook");
+		}
+
+		const bundle = {
+			"bundle.js.map": {
+				type: "asset",
+				fileName: "bundle.js.map",
+				source: JSON.stringify({ version: 3, mappings: "AAAA" }),
+			},
+		} as unknown as OutputBundle;
+
+		await expect(
+			plugin.writeBundle(
+				{ dir: process.cwd() } as NormalizedOutputOptions,
+				bundle,
+			),
+		).resolves.toBeUndefined();
+		expect(onUploadError).toHaveBeenCalledTimes(1);
+	});
+
 	it("prefers webpack native build hash when buildId is not provided", async () => {
 		const cwd = await mkdtemp(join(tmpdir(), "sourcemaps-webpack-native-id-"));
 		await cp(join(fixturesDir, "webpack"), cwd, { recursive: true });

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +"@sourcemaps/backend": patch
 +---
++
 +Increase the public sourcemap ingest request body limit to 50MB to avoid 413 errors when uploading larger sourcemap payloads.