Generate an exportable and ambient version of the TypeScript types

types/BUILD.bazel

@@ -28,7 +28,7 @@ js_run_binary(
     srcs = [
         "//src/workerd/tools:api_encoder",
     ],
-    outs = ["api.d.ts"],  # TODO(soon) switch to out_dirs when generating multiple files
+    outs = ["api.d.ts", "api.ts"],  # TODO(soon) switch to out_dirs when generating multiple files
     args = [
         "src/workerd/tools/api.capnp.bin",
         "--output",

types/src/index.ts

@@ -11,18 +11,75 @@ import ts from "typescript";
 import { generateDefinitions } from "./generator";
 import { printNodeList, printer } from "./print";
 import { createMemoryProgram } from "./program";
+import { ParsedTypeDefinition, collateStandards } from "./standards";
 import {
   compileOverridesDefines,
+  createCommentsTransformer,
   createGlobalScopeTransformer,
   createIteratorTransformer,
   createOverrideDefineTransformer,
 } from "./transforms";
-
+import { createAmbientTransformer } from "./transforms/ambient";
+import { createImportableTransformer } from "./transforms/importable";
 const definitionsHeader = `/* eslint-disable */
 // noinspection JSUnusedGlobalSymbols
 `;
 
-function printDefinitions(root: StructureGroups): string {
+function checkDiagnostics(sources: Map<string, string>) {
+  const host = ts.createCompilerHost(
+    { noEmit: true },
+    /* setParentNodes */ true
+  );
+
+  host.getDefaultLibLocation = () =>
+    path.dirname(require.resolve("typescript"));
+  const program = createMemoryProgram(sources, host, {
+    lib: ["lib.esnext.d.ts"],
+  });
+  const emitResult = program.emit();
+
+  const allDiagnostics = ts
+    .getPreEmitDiagnostics(program)
+    .concat(emitResult.diagnostics);
+
+  allDiagnostics.forEach((diagnostic) => {
+    if (diagnostic.file) {
+      const { line, character } = ts.getLineAndCharacterOfPosition(
+        diagnostic.file,
+        diagnostic.start!
+      );
+      const message = ts.flattenDiagnosticMessageText(
+        diagnostic.messageText,
+        "\n"
+      );
+      console.log(`(${line + 1},${character + 1}): ${message}`);
+    } else {
+      console.log(
+        ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")
+      );
+    }
+  });
+}
+function transform(
+  sources: Map<string, string>,
+  sourcePath: string,
+  transforms: (
+    program: ts.Program,
+    checker: ts.TypeChecker
+  ) => ts.TransformerFactory<ts.SourceFile>[]
+) {
+  const program = createMemoryProgram(sources);
+  const checker = program.getTypeChecker();
+  const sourceFile = program.getSourceFile(sourcePath);
+  assert(sourceFile !== undefined);
+  const result = ts.transform(sourceFile, transforms(program, checker));
+  assert.strictEqual(result.transformed.length, 1);
+  return printer.printFile(result.transformed[0]);
+}
+function printDefinitions(
+  root: StructureGroups,
+  standards: ParsedTypeDefinition
+): { ambient: string; importable: string } {
   // Generate TypeScript nodes from capnp request
   const nodes = generateDefinitions(root);
 
@@ -33,45 +90,46 @@ function printDefinitions(root: StructureGroups): string {
   let source = printNodeList(nodes);
   sources.set(sourcePath, source);
 
-  // Build TypeScript program from source files and overrides. Importantly,
-  // these are in the same program, so we can use nodes from one in the other.
-  let program = createMemoryProgram(sources);
-  let checker = program.getTypeChecker();
-  let sourceFile = program.getSourceFile(sourcePath);
-  assert(sourceFile !== undefined);
-
   // Run post-processing transforms on program
-  let result = ts.transform(sourceFile, [
+  source = transform(sources, sourcePath, (program, checker) => [
     // Run iterator transformer before overrides so iterator-like interfaces are
     // still removed if they're replaced in overrides
     createIteratorTransformer(checker),
     createOverrideDefineTransformer(program, replacements),
   ]);
-  assert.strictEqual(result.transformed.length, 1);
 
   // We need the type checker to respect our updated definitions after applying
   // overrides (e.g. to find the correct nodes when traversing heritage), so
   // rebuild the program to re-run type checking.
   // TODO: is there a way to re-run the type checker on an existing program?
-  source = printer.printFile(result.transformed[0]);
-  program = createMemoryProgram(new Map([[sourcePath, source]]));
-  checker = program.getTypeChecker();
-  sourceFile = program.getSourceFile(sourcePath);
-  assert(sourceFile !== undefined);
+  source = transform(
+    new Map([[sourcePath, source]]),
+    sourcePath,
+    (program, checker) => [
+      // Run global scope transformer after overrides so members added in
+      // overrides are extracted
+      createGlobalScopeTransformer(checker),
+      createCommentsTransformer(standards),
+      createAmbientTransformer(),
+      // TODO(polish): maybe flatten union types?
+    ]
+  );
 
-  result = ts.transform(sourceFile, [
-    // Run global scope transformer after overrides so members added in
-    // overrides are extracted
-    createGlobalScopeTransformer(checker),
-    // TODO(polish): maybe flatten union types?
-  ]);
-  assert.strictEqual(result.transformed.length, 1);
+  checkDiagnostics(new Map([[sourcePath, source]]));
+
+  const importable = transform(
+    new Map([[sourcePath, source]]),
+    sourcePath,
+    () => [createImportableTransformer()]
+  );
 
-  // TODO(polish): maybe log diagnostics with `ts.getPreEmitDiagnostics(program, sourceFile)`?
-  //  (see https://github.com/microsoft/TypeScript/wiki/Using-the-Compiler-API#a-minimal-compiler)
+  checkDiagnostics(new Map([[sourcePath, importable]]));
 
   // Print program to string
-  return definitionsHeader + printer.printFile(result.transformed[0]);
+  return {
+    ambient: definitionsHeader + source,
+    importable: definitionsHeader + importable,
+  };
 }
 
 // Generates TypeScript types from a binary Cap’n Proto file containing encoded
@@ -109,17 +167,31 @@ export async function main(args?: string[]) {
   const message = new Message(buffer, /* packed */ false);
   const root = message.getRoot(StructureGroups);
 
-  let definitions = printDefinitions(root);
+  const standards = await collateStandards(
+    path.join(
+      path.dirname(require.resolve("typescript")),
+      "lib.webworker.d.ts"
+    ),
+    path.join(
+      path.dirname(require.resolve("typescript")),
+      "lib.webworker.iterable.d.ts"
+    )
+  );
+
+  let { ambient, importable } = printDefinitions(root, standards);
+
   if (options.format) {
-    definitions = prettier.format(definitions, { parser: "typescript" });
+    ambient = prettier.format(ambient, { parser: "typescript" });
+    importable = prettier.format(importable, { parser: "typescript" });
   }
   if (options.output !== undefined) {
+    console.log(options.output);
     const output = path.resolve(options.output);
     await mkdir(path.dirname(output), { recursive: true });
-    await writeFile(output, definitions);
-  } else {
-    // Write to stdout without extra newline
-    process.stdout.write(definitions);
+    await writeFile(output, ambient);
+
+    const importableFile = path.join(path.dirname(output), "api.ts");
+    await writeFile(importableFile, importable);
   }
 }
 

types/src/program.ts

@@ -1,3 +1,4 @@
+import { assert } from "console";
 import path from "path";
 import ts from "typescript";
 
@@ -6,11 +7,32 @@ interface MemorySourceFile {
   sourceFile: ts.SourceFile;
 }
 
+// Update compiler host to return in-memory source file
+function patchHostMethod<K extends "fileExists" | "readFile" | "getSourceFile">(
+  host: ts.CompilerHost,
+  sourceFiles: Map<string, MemorySourceFile>,
+  key: K,
+  placeholderResult: (f: MemorySourceFile) => ReturnType<ts.CompilerHost[K]>
+) {
+  const originalMethod: (...args: any[]) => any = host[key];
+  host[key] = (fileName: string, ...args: any[]) => {
+    const sourceFile = sourceFiles.get(path.resolve(fileName));
+    if (sourceFile !== undefined) {
+      return placeholderResult(sourceFile);
+    }
+    return originalMethod.call(host, fileName, ...args);
+  };
+}
+
 // Creates a TypeScript program from in-memory source files. Accepts a Map of
 // fully-resolved "virtual" paths to source code.
-export function createMemoryProgram(sources: Map<string, string>): ts.Program {
-  const options = ts.getDefaultCompilerOptions();
-  const host = ts.createCompilerHost(options, true);
+export function createMemoryProgram(
+  sources: Map<string, string>,
+  host?: ts.CompilerHost,
+  options?: ts.CompilerOptions
+): ts.Program {
+  options ??= ts.getDefaultCompilerOptions();
+  host ??= ts.createCompilerHost(options, true);
 
   const sourceFiles = new Map<string, MemorySourceFile>();
   for (const [sourcePath, source] of sources) {
@@ -24,25 +46,14 @@ export function createMemoryProgram(sources: Map<string, string>): ts.Program {
     sourceFiles.set(sourcePath, { source, sourceFile });
   }
 
-  // Update compiler host to return in-memory source file
-  function patchHostMethod<
-    K extends "fileExists" | "readFile" | "getSourceFile"
-  >(
-    key: K,
-    placeholderResult: (f: MemorySourceFile) => ReturnType<ts.CompilerHost[K]>
-  ) {
-    const originalMethod: (...args: any[]) => any = host[key];
-    host[key] = (fileName: string, ...args: any[]) => {
-      const sourceFile = sourceFiles.get(path.resolve(fileName));
-      if (sourceFile !== undefined) {
-        return placeholderResult(sourceFile);
-      }
-      return originalMethod.call(host, fileName, ...args);
-    };
-  }
-  patchHostMethod("fileExists", () => true);
-  patchHostMethod("readFile", ({ source }) => source);
-  patchHostMethod("getSourceFile", ({ sourceFile }) => sourceFile);
+  patchHostMethod(host, sourceFiles, "fileExists", () => true);
+  patchHostMethod(host, sourceFiles, "readFile", ({ source }) => source);
+  patchHostMethod(
+    host,
+    sourceFiles,
+    "getSourceFile",
+    ({ sourceFile }) => sourceFile
+  );
 
   const rootNames = [...sourceFiles.keys()];
   return ts.createProgram(rootNames, options, host);

types/src/standards.ts

@@ -0,0 +1,68 @@
+import { assert } from "console";
+import { readFile } from "fs/promises";
+import * as ts from "typescript";
+import { createMemoryProgram } from "./program";
+export interface ParsedTypeDefinition {
+  program: ts.Program;
+  source: ts.SourceFile;
+  checker: ts.TypeChecker;
+  parsed: {
+    functions: Map<string, ts.FunctionDeclaration>;
+    interfaces: Map<string, ts.InterfaceDeclaration>;
+    vars: Map<string, ts.VariableDeclaration>;
+    types: Map<string, ts.TypeAliasDeclaration>;
+    classes: Map<string, ts.ClassDeclaration>;
+  };
+}
+
+// Collate standards (to support lib.(dom|webworker).iterable.d.ts being defined separately)
+export async function collateStandards(
+  ...standardTypes: string[]
+): Promise<ParsedTypeDefinition> {
+  const STANDARDS_PATH = "/source.ts";
+  const text: string = (
+    await Promise.all(
+      standardTypes.map(
+        async (s) =>
+          // Remove the Microsoft copyright notices from the file, to prevent them being copied in as TS comments
+          (await readFile(s, "utf-8")).split(`/////////////////////////////`)[2]
+      )
+    )
+  ).join("\n");
+  const program = createMemoryProgram(new Map([[STANDARDS_PATH, text]]));
+  const source = program.getSourceFile(STANDARDS_PATH)!;
+  const checker = program.getTypeChecker();
+  const parsed = {
+    functions: new Map<string, ts.FunctionDeclaration>(),
+    interfaces: new Map<string, ts.InterfaceDeclaration>(),
+    vars: new Map<string, ts.VariableDeclaration>(),
+    types: new Map<string, ts.TypeAliasDeclaration>(),
+    classes: new Map<string, ts.ClassDeclaration>(),
+  };
+  ts.forEachChild(source, (node) => {
+    let name = "";
+    if (node && ts.isFunctionDeclaration(node)) {
+      name = node.name?.text ?? "";
+      parsed.functions.set(name, node);
+    } else if (ts.isVariableStatement(node)) {
+      assert(node.declarationList.declarations.length === 1);
+      name = node.declarationList.declarations[0].name.getText(source);
+      parsed.vars.set(name, node.declarationList.declarations[0]);
+    } else if (ts.isInterfaceDeclaration(node)) {
+      name = node.name.text;
+      parsed.interfaces.set(name, node);
+    } else if (ts.isTypeAliasDeclaration(node)) {
+      name = node.name.text;
+      parsed.types.set(name, node);
+    } else if (ts.isClassDeclaration(node)) {
+      name = node.name?.text ?? "";
+      parsed.classes.set(name, node);
+    }
+  });
+  return {
+    program,
+    source,
+    checker,
+    parsed,
+  };
+}

types/src/transforms/ambient.ts

@@ -0,0 +1,18 @@
+import ts from "typescript";
+import { ensureStatementModifiers } from "./helpers";
+
+// This ensures that all nodes have the `declare` keyword
+export function createAmbientTransformer(): ts.TransformerFactory<ts.SourceFile> {
+  return (ctx) => {
+    return (node) => {
+      const visitor = createVisitor(ctx);
+      return ts.visitEachChild(node, visitor, ctx);
+    };
+  };
+}
+
+function createVisitor(ctx: ts.TransformationContext) {
+  return (node: ts.Node) => {
+    return ensureStatementModifiers(ctx, node, false);
+  };
+}