Generate multiple types, one for each compatibility date that changes type signatures

src/workerd/tools/BUILD.bazel

@@ -13,13 +13,42 @@ wd_cc_binary(
     ],
 )
 
-run_binary(
+compat_dates = [
+    # Oldest compatibility date, with no flags enabled
+    ("2021-01-01", "oldest"),
+    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#formdata-parsing-supports-file
+    ("2021-11-03", "2021-11-03"),
+    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#settersgetters-on-api-object-prototypes
+    ("2022-01-31", "2022-01-31"),
+    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#global-navigator
+    ("2022-03-21", "2022-03-21"),
+    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#global-navigator
+    ("2022-08-04", "2022-08-04"),
+    # Latest compatibility date
+    ("2030-01-01", "experimental"),
+]
+
+filegroup(
     name = "api_encoder",
-    outs = ["api.capnp.bin"],
-    args = [
-        "--output",
-        "$(location api.capnp.bin)",
+    srcs = [
+        "//src/workerd/tools:api_encoder_" + label
+        for (date, label) in compat_dates
     ],
-    tool = "api_encoder_bin",
     visibility = ["//visibility:public"],
 )
+
+[
+    run_binary(
+        name = "api_encoder_" + label,
+        outs = [label + ".api.capnp.bin"],
+        args = [
+            "--output",
+            "$(location " + label + ".api.capnp.bin)",
+            "--compatibility-date",
+            date,
+        ],
+        tool = "api_encoder_bin",
+        visibility = ["//visibility:public"],
+    )
+    for (date, label) in compat_dates
+]

src/workerd/tools/api-encoder.c++

@@ -64,6 +64,10 @@ struct ApiEncoderMain {
     return kj::MainBuilder(context, "<unknown>", "API Encoder")
         .addOptionWithArg({"o", "output"}, KJ_BIND_METHOD(*this, setOutput),
                           "<file>", "Output to <file>")
+        .addOptionWithArg(
+            {"c", "compatibility-date"},
+            KJ_BIND_METHOD(*this, setCompatibilityDate), "<date>",
+            "Set the compatibility date of the generated types to <date>")
         .callAfterParsing(KJ_BIND_METHOD(*this, run))
         .build();
   }
@@ -73,6 +77,11 @@ struct ApiEncoderMain {
     return true;
   }
 
+  kj::MainBuilder::Validity setCompatibilityDate(kj::StringPtr value) {
+    compatibilityDate = value;
+    return true;
+  }
+
   CompatibilityFlags::Reader
   compileFlags(capnp::MessageBuilder &message, kj::StringPtr compatDate,
                kj::ArrayPtr<const kj::StringPtr> compatFlags) {
@@ -102,11 +111,13 @@ struct ApiEncoderMain {
 
   bool run() {
     // Create RTTI builder with all non-experimental compatibility flags enabled
-    // TODO(soon): generate different types for different flags, for now, we
-    //  set the compatibility date in the future such that all flags with a
-    //  $compatEnableDate are enabled.
     capnp::MallocMessageBuilder flagsMessage;
-    auto flags = compileFlags(flagsMessage, "2023-01-01", {});
+    CompatibilityFlags::Reader flags;
+    KJ_IF_MAYBE (date, compatibilityDate) {
+      flags = compileFlags(flagsMessage, *date, {});
+    } else {
+      flags = compileFlags(flagsMessage, "2021-01-01", {});
+    }
     auto builder = rtti::Builder(flags);
 
     // Build structure groups
@@ -153,8 +164,7 @@ struct ApiEncoderMain {
 
   template <typename... Types>
   void writeGroup(
-      capnp::List<rtti::StructureGroups::StructureGroup>::Builder
-          &groups,
+      capnp::List<rtti::StructureGroups::StructureGroup>::Builder &groups,
       rtti::Builder<CompatibilityFlags::Reader> &builder, kj::StringPtr name) {
     auto group = groups[groupsIndex++];
     group.setName(name);
@@ -169,6 +179,7 @@ struct ApiEncoderMain {
 private:
   kj::ProcessContext &context;
   kj::Maybe<kj::StringPtr> output;
+  kj::Maybe<kj::StringPtr> compatibilityDate;
 
   unsigned int groupsIndex = 0;
   unsigned int structureIndex = 0;

types/BUILD.bazel

@@ -28,13 +28,13 @@ js_run_binary(
     srcs = [
         "//src/workerd/tools:api_encoder",
     ],
-    outs = ["api.d.ts", "api.ts"],  # TODO(soon) switch to out_dirs when generating multiple files
     args = [
-        "src/workerd/tools/api.capnp.bin",
+        "src/workerd/tools",
         "--output",
-        "types/api.d.ts",
+        "types/definitions",
         "--format",
     ],
+    out_dirs = ["definitions"],
     silent_on_success = False,  # Always enable logging for debugging
     tool = ":types_bin",
 )

types/src/index.ts

@@ -1,8 +1,7 @@
 #!/usr/bin/env node
 import assert from "assert";
-import { mkdir, readFile, writeFile } from "fs/promises";
+import { appendFile, mkdir, readFile, readdir, writeFile } from "fs/promises";
 import path from "path";
-import { arrayBuffer } from "stream/consumers";
 import util from "util";
 import { StructureGroups } from "@workerd/jsg/rtti.capnp.js";
 import { Message } from "capnp-ts";
@@ -141,13 +140,14 @@ function printDefinitions(
 // Usage: types [options] [input]
 //
 // Options:
-//  -o, --output <file>
-//    File path to write TypeScript to, defaults to stdout if omitted
+//  -o, --output <dir>
+//    Directory to write types to, in folders based on compat date
 //  -f, --format
 //    Formats generated types with Prettier
 //
 // Input:
-//    Binary Cap’n Proto file path, defaults to reading from stdin if omitted
+//    Directory containing binary Cap’n Proto file paths, in the format <label>.api.capnp.bin
+//      <label> should relate to the compatibility date
 export async function main(args?: string[]) {
   const { values: options, positionals } = util.parseArgs({
     options: {
@@ -158,15 +158,12 @@ export async function main(args?: string[]) {
     allowPositionals: true,
     args,
   });
-  const maybeInputPath = positionals[0];
-
-  const buffer =
-    maybeInputPath === undefined
-      ? await arrayBuffer(process.stdin)
-      : await readFile(maybeInputPath);
-  const message = new Message(buffer, /* packed */ false);
-  const root = message.getRoot(StructureGroups);
-
+  const inputDir = positionals[0];
+  assert(
+    inputDir,
+    "This script requires a positional argument pointing to a directory containing binary Cap’n Proto file paths, in the format <label>.api.capnp.bin"
+  );
+  const files = await readdir(inputDir);
   const standards = await collateStandards(
     path.join(
       path.dirname(require.resolve("typescript")),
@@ -177,21 +174,23 @@ export async function main(args?: string[]) {
       "lib.webworker.iterable.d.ts"
     )
   );
-
-  let { ambient, importable } = printDefinitions(root, standards);
-
-  if (options.format) {
-    ambient = prettier.format(ambient, { parser: "typescript" });
-    importable = prettier.format(importable, { parser: "typescript" });
-  }
-  if (options.output !== undefined) {
-    console.log(options.output);
-    const output = path.resolve(options.output);
-    await mkdir(path.dirname(output), { recursive: true });
-    await writeFile(output, ambient);
-
-    const importableFile = path.join(path.dirname(output), "api.ts");
-    await writeFile(importableFile, importable);
+  for (const file of files) {
+    const buffer = await readFile(path.join(inputDir, file));
+    const message = new Message(buffer, /* packed */ false);
+    const root = message.getRoot(StructureGroups);
+    let { ambient, importable } = printDefinitions(root, standards);
+    if (options.format) {
+      ambient = prettier.format(ambient, { parser: "typescript" });
+      importable = prettier.format(importable, { parser: "typescript" });
+    }
+    if (options.output !== undefined) {
+      const output = path.resolve(options.output);
+      const [date] = file.split(".api.capnp.bin");
+      await mkdir(path.join(output, date), { recursive: true });
+      await writeFile(path.join(output, date, "api.d.ts"), ambient);
+      const importableFile = path.join(output, date, "api.ts");
+      await writeFile(importableFile, importable);
+    }
   }
 }
 

types/src/program.ts

@@ -1,4 +1,3 @@
-import { assert } from "console";
 import path from "path";
 import ts from "typescript";
 

types/test/index.spec.ts

@@ -108,12 +108,16 @@ test("main: generates types", async () => {
   // https://bazel.build/reference/test-encyclopedia#initial-conditions
   const tmpPath = process.env.TEST_TMPDIR;
   assert(tmpPath !== undefined);
-  const inputPath = path.join(tmpPath, "types.capnp.bin");
-  const outputPath = path.join(tmpPath, "types.d.ts");
+  const definitionsDir = path.join(tmpPath, "definitions");
+  await fs.mkdir(definitionsDir);
+  const inputDir = path.join(tmpPath, "capnp");
+  await fs.mkdir(inputDir);
+  const inputPath = path.join(inputDir, "types.api.capnp.bin");
+  const outputPath = path.join(definitionsDir, "types", "api.d.ts");
 
   await fs.writeFile(inputPath, new Uint8Array(message.toArrayBuffer()));
 
-  await main([inputPath, "--output", outputPath]);
+  await main([inputDir, "--output", definitionsDir]);
   let output = await fs.readFile(outputPath, "utf8");
   assert.strictEqual(
     output,
@@ -141,7 +145,7 @@ declare const prop: Promise<number>;
   );
 
   // Test formatted output
-  await main([inputPath, "-o", outputPath, "--format"]);
+  await main([inputDir, "-o", definitionsDir, "--format"]);
   output = await fs.readFile(outputPath, "utf8");
   assert.strictEqual(
     output,