diff --git a/CHANGELOG.md b/CHANGELOG.md index 4986d1c9..fb7c2da5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - Add `garn.javascript.prettier` plugin to automatically format and check javascript formatting using [prettier](https://prettier.io/). +- Add `garn.callFlake` helper to allow importing flake files by local path. +- Add `garn.importFlake` helper to allow importing flake files by url, e.g. from GitHub. ## v0.0.18 diff --git a/ts/importFlake.test.ts b/ts/importFlake.test.ts new file mode 100644 index 00000000..4e35d5b1 --- /dev/null +++ b/ts/importFlake.test.ts @@ -0,0 +1,288 @@ +import { + afterEach, + beforeEach, + describe, + it, +} from "https://deno.land/std@0.206.0/testing/bdd.ts"; +import { + assertStderrContains, + assertStdout, + assertSuccess, + assertThrowsWith, + buildPackage, + runCheck, + runCommand, + runExecutable, + runInDevShell, +} from "./testUtils.ts"; +import { callFlake, importFlake, importFromGithub } from "./importFlake.ts"; +import * as garn from "./mod.ts"; +import { existsSync } from "https://deno.land/std@0.201.0/fs/mod.ts"; +import { assertEquals } from "https://deno.land/std@0.206.0/assert/mod.ts"; + +describe("callFlake", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = Deno.makeTempDirSync({ prefix: "garn-test" }); + }); + + afterEach(() => { + Deno.removeSync(tempDir, { recursive: true }); + }); + + const writeFlakeToImport = (contents: string) => { + Deno.mkdirSync(`${tempDir}/to-import`, { recursive: true }); + Deno.writeTextFileSync( + `${tempDir}/to-import/flake.nix`, + ` + { + inputs.nixpkgs.url = "github:NixOS/nixpkgs/6fc7203e423bbf1c8f84cccf1c4818d097612566"; + inputs.flake-utils.url = "github:numtide/flake-utils/ff7b65b44d01cf9ba6a71320833626af21126384"; + outputs = { self, nixpkgs, flake-utils }: flake-utils.lib.eachDefaultSystem (system: + let pkgs = import "\${nixpkgs}" { inherit system; }; in ${contents} + ); + } + `, + ); + if (!existsSync(`${tempDir}/to-import/flake.lock`)) { + assertSuccess( + runCommand( + new Deno.Command("nix", { + args: ["flake", "lock"], + cwd: `${tempDir}/to-import`, + }), + ), + ); + } + }; + + describe("getPackage", () => { + it("returns the specified package from the flake file", () => { + writeFlakeToImport(`{ + packages = { + main_pkg = pkgs.runCommand "create-some-files" {} '' + mkdir $out + touch $out/foo $out/bar + ''; + }; + }`); + const flake = callFlake("./to-import"); + const pkg = buildPackage(flake.getPackage("main_pkg"), { dir: tempDir }); + assertEquals(getDirEntryNames(pkg), ["bar", "foo"]); + }); + + it("reflects changes when changing the flake source after importing", () => { + writeFlakeToImport(`{ + packages = { + main_pkg = pkgs.runCommand "create-some-files" {} '' + mkdir $out + touch $out/original + ''; + }; + }`); + const flake = callFlake(`./to-import`); + buildPackage(flake.getPackage("main_pkg"), { dir: tempDir }); + writeFlakeToImport(`{ + packages = { + main_pkg = pkgs.runCommand "create-some-files" {} '' + mkdir $out + touch $out/modified + ''; + }; + }`); + const pkg = buildPackage(flake.getPackage("main_pkg"), { dir: tempDir }); + assertEquals(getDirEntryNames(pkg), ["modified"]); + }); + + it("allows importing relative sources in packages", () => { + Deno.writeTextFileSync( + `${tempDir}/foo.js`, + "console.log('Hello from a js file in tempDir!')", + ); + writeFlakeToImport(`{ + packages = { + foo = pkgs.writeScript "foo" '' + \${pkgs.nodejs}/bin/node \${../foo.js} + ''; + }; + }`); + const flake = callFlake("./to-import"); + const exe = garn.shell`${flake.getPackage("foo")}`; + const output = assertSuccess(runExecutable(exe, { cwd: tempDir })); + assertStdout(output, "Hello from a js file in tempDir!\n"); + }); + + it("displays a helpful error if the specified package does not exist", () => { + writeFlakeToImport(`{ packages = { }; }`); + const flake = callFlake("./to-import"); + const exe = garn.shell`${flake.getPackage("foo")}`; + const output = runExecutable(exe, { cwd: tempDir }); + assertStderrContains( + output, + 'error: The package "foo" was not found in ./to-import', + ); + }); + }); + + describe("getApp", () => { + it("returns the specified executable from the flake file apps", () => { + writeFlakeToImport(`{ + apps = { + hello = { + type = "app"; + program = builtins.toString (pkgs.writeScript "hello" '' + echo hello from flake file + ''); + }; + }; + }`); + const flake = callFlake("./to-import"); + const exe = flake.getApp("hello"); + const output = assertSuccess(runExecutable(exe, { cwd: tempDir })); + assertStdout(output, "hello from flake file\n"); + }); + + it("displays a helpful error if the specified app does not exist", () => { + writeFlakeToImport(`{ apps = { }; }`); + const flake = callFlake("./to-import"); + const exe = garn.shell`${flake.getApp("foo")}`; + const output = runExecutable(exe, { cwd: tempDir }); + assertStderrContains( + output, + 'error: The app "foo" was not found in ./to-import', + ); + }); + }); + + describe("getCheck", () => { + it("returns the specified check from the flake file", () => { + writeFlakeToImport(`{ + checks = { + some-check = pkgs.runCommand "my-check" {} '' + # ${Date.now()} + touch $out + echo running my-check! + ''; + }; + }`); + const flake = callFlake("./to-import"); + const check = flake.getCheck("some-check"); + const output = assertSuccess(runCheck(check, { dir: tempDir })); + assertStderrContains(output, "running my-check!"); + }); + + it("displays a helpful error if the specified check does not exist", () => { + writeFlakeToImport(`{ checks = { }; }`); + const flake = callFlake("./to-import"); + const exe = garn.shell`${flake.getCheck("foo")}`; + const output = runExecutable(exe, { cwd: tempDir }); + assertStderrContains( + output, + 'error: The check "foo" was not found in ./to-import', + ); + }); + }); + + describe("getDevShell", () => { + it("returns the specified environment from the flake file", () => { + writeFlakeToImport(`{ + devShells = { + some-shell = pkgs.mkShell { + nativeBuildInputs = [ pkgs.hello ]; + }; + }; + }`); + const flake = callFlake("./to-import"); + const env = flake.getDevShell("some-shell"); + const output = runInDevShell(env, { cmd: "hello", dir: tempDir }); + assertStdout(output, "Hello, world!\n"); + }); + + it("displays a helpful error if the specified devShell does not exist", () => { + writeFlakeToImport(`{ devShells = { }; }`); + const flake = callFlake("./to-import"); + const exe = garn.shell`${flake.getDevShell("foo")}`; + const output = runExecutable(exe, { cwd: tempDir }); + assertStderrContains( + output, + 'error: The devShell "foo" was not found in ./to-import', + ); + }); + }); + + describe("allPackages", () => { + it("returns a package with symlinks to all found packages", () => { + writeFlakeToImport(`{ + packages = { + foo = pkgs.runCommand "foo" {} "echo foo-pkg > $out"; + bar = pkgs.runCommand "bar" {} "echo bar-pkg > $out"; + }; + }`); + const flake = callFlake("./to-import"); + const pkg = buildPackage(flake.allPackages, { dir: tempDir }); + assertEquals(getDirEntryNames(pkg), ["bar", "foo"]); + assertEquals(Deno.readTextFileSync(`${pkg}/foo`), "foo-pkg\n"); + assertEquals(Deno.readTextFileSync(`${pkg}/bar`), "bar-pkg\n"); + }); + }); + + describe("getAllChecks", () => { + it("returns a check that composes all found checks", () => { + writeFlakeToImport(`{ + checks = { + foo = pkgs.runCommand "check-foo" {} '' + # ${Date.now()} + touch $out + echo running check foo + ''; + bar = pkgs.runCommand "check-bar" {} '' + # ${Date.now()} + touch $out + echo running check bar + ''; + }; + }`); + const flake = callFlake("./to-import"); + const output = assertSuccess(runCheck(flake.allChecks, { dir: tempDir })); + assertStderrContains(output, "running check foo"); + assertStderrContains(output, "running check bar"); + }); + }); +}); + +describe("importFlake", () => { + it("allows importing flakes from url", () => { + const flake = importFlake( + "github:garnix-io/debug-tools/8a4026fa6ccbfec070f96d458ffa96e7fb6112e8", + ); + const exe = flake.getPackage("main_pkg").bin("debug-args"); + const output = assertSuccess(runExecutable(exe)); + assertStdout(output, ""); + assertStderrContains(output, "[]"); + }); +}); + +describe("importFromGithub", () => { + it("allows importing flakes from GitHub repositories", () => { + const flake = importFromGithub({ + repo: "garnix-io/debug-tools", + revOrRef: "8a4026fa6ccbfec070f96d458ffa96e7fb6112e8", + }); + const exe = flake.getPackage("main_pkg").bin("debug-args"); + const output = assertSuccess(runExecutable(exe)); + assertStdout(output, ""); + assertStderrContains(output, "[]"); + }); + + it("throws an error if the repo is malformed", () => { + assertThrowsWith( + "The `repo` of a hosted git service should match /", + () => importFromGithub({ repo: "/foo/bar" }), + ); + }); +}); + +function getDirEntryNames(path: string) { + return [...Deno.readDirSync(path)].map((entry) => entry.name).sort(); +} diff --git a/ts/importFlake.ts b/ts/importFlake.ts new file mode 100644 index 00000000..f92de48b --- /dev/null +++ b/ts/importFlake.ts @@ -0,0 +1,267 @@ +import { Check } from "./check.ts"; +import { Environment, mkEnvironment } from "./environment.ts"; +import { Executable, mkExecutable } from "./executable.ts"; +import { filterNullValues } from "./internal/utils.ts"; +import { + getPathOrError, + NixExpression, + nixFlakeDep, + nixRaw, + nixStrLit, +} from "./nix.ts"; +import { mkPackage, Package } from "./package.ts"; + +/** + * Config for importing flake files from GitHub/GitLab/SourceHut. + */ +export type RepoConfig = { + /** + * The full repo name, i.e. the user handle and the repo name separated by a + * slash, e.g. `garnix-io/garn`. + */ + repo: string; + + /** + * The name of a branch/tag, or a commit hash to import. + */ + revOrRef?: string; + + /** + * The path to a subdirectory in the repository containing the flake file to + * import. + */ + dir?: string; + + /** + * A host to fetch from other than the default for that hosted service + * (useful if pulling from a self-hosted GitHub/GitLab/Sourcehut instance). + */ + host?: string; +}; + +/** + * Methods for obtaining apps, checks, dev-shells, and packages from an imported + * flake file to include in your garn project. + */ +export type ImportedFlake = { + /** + * A check that combines all checks found in the imported flake file. + * + * I.e. `allChecks` will succeed only if *all* checks in the imported flake file + * succeed. + */ + allChecks: Check; + + /** + * A package that combines all packages found in the imported flake file. + * + * When building this package the result will be a directory of symlinks to + * all packages in the imported flake file. + */ + allPackages: Package; + + /** + * Obtain a specific `Executable` from the `apps` section of the imported + * flake file. + * + * `garn` will error if the specified `appName` does not exist. + */ + getApp: (appName: string) => Executable; + + /** + * Obtain a specific `Check` from the `checks` section of the imported flake + * file. + * + * `garn` will error if the specified `checkName` does not exist. + */ + getCheck: (checkName: string) => Check; + + /** + * Obtain a specific `Environment` from the `devShells` section of the + * imported flake file. + * + * `garn` will error if the specified `devShellName` does not exist. + */ + getDevShell: (devShellName: string) => Environment; + + /** + * Obtain a specific `Package` from the `packages` section of the imported + * flake file. + * + * `garn` will error if the specified `packageName` does not exist. + */ + getPackage: (packageName: string) => Package; +}; + +/** + * Imports a flake file from a GitHub repository. + * + * See `importFlake` for details. + */ +export function importFromGithub(options: RepoConfig): ImportedFlake { + return importFlakeFromRepoConfig("github", options); +} + +/** + * Imports a flake file from a GitLab repository. + * + * See `importFlake` for details. + */ +export function importFromGitlab(options: RepoConfig): ImportedFlake { + return importFlakeFromRepoConfig("gitlab", options); +} + +/** + * Imports a flake file from a SourceHut repository. + * + * See `importFlake` for details. + */ +export function importFromSourcehut(options: RepoConfig): ImportedFlake { + return importFlakeFromRepoConfig("sourcehut", options); +} + +/** + * Gets apps, checks, dev-shells, and packages from an existing flake file by + * adding it as a flake input to your project. + * + * This is most useful for bringing in flake files from the internet. The hash + * of the flake file and its referenced sources will be included in your + * project's `flake.lock`. If you want to import a flake file within your + * current project, consider using `callFlake` instead which does not update + * the hash in your flake.lock. + * + * See https://nixos.org/manual/nix/unstable/command-ref/new-cli/nix3-flake.html + * for information on how to format `flakeUrl`. You may also want to use + * `importFromGithub`, `importFromGitlab` or `importFromSourcehut` helpers + * instead which provide a more ergonomic API when importing from these + * services. + * + * Example: + * ```typescript + * const myFlake = importFlake("git+https://example.com/some-repo.git?ref=main"); + * + * const myProject = getGarnProjectSomehow() + * .add(() => ({ + * format: myFlake.getApp("format"), + * env: myFlake.getDevShell("default"), + * bundle: myFlake.getPackage("bundle"), + * allChecks: myFlake.allChecks, + * })); + * ``` + */ +export function importFlake(flakeUrl: string): ImportedFlake { + return processFlake( + nixFlakeDep("importedFlake-" + flakeUrl.replaceAll(/[^a-zA-Z0-9]/g, "_"), { + url: flakeUrl, + }), + flakeUrl, + ); +} + +/** + * Gets apps, checks, dev-shells, and packages from an existing flake file + * within your current project. + * + * @param flakeDir The relative path to the directory containing a `flake.nix` + * + * Absolute `flakeDir`s and paths outside of the garn project are not + * supported. Consider using `importFlake` instead if you want to import an + * absolute flake path or one from a URL. + * + * This can be used to migrate an existing flake project to use garn. + * + * Example: + * ```typescript + * const myFlake = callFlake("./path/to/dir"); + * + * const myProject = getGarnProjectSomehow() + * .add(() => ({ + * format: myFlake.getApp("format"), + * env: myFlake.getDevShell("default"), + * bundle: myFlake.getPackage("bundle"), + * allChecks: myFlake.allChecks, + * })); + * ``` + */ +export function callFlake(flakeDir: string): ImportedFlake { + const callFlake = nixFlakeDep("call-flake", { + url: "github:divnix/call-flake", + }); + return processFlake(nixRaw`(${callFlake} ${nixRaw(flakeDir)})`, flakeDir); +} + +function importFlakeFromRepoConfig( + serviceName: string, + opts: RepoConfig, +): ImportedFlake { + if (!opts.repo.match(/^[^/]+\/[^/]+$/)) { + throw new Error( + "The `repo` of a hosted git service should match /", + ); + } + const query = new URLSearchParams( + filterNullValues({ + dir: opts.dir, + host: opts.host, + }), + ).toString(); + return importFlake( + serviceName + + ":" + + opts.repo + + (opts.revOrRef ? `/${opts.revOrRef}` : "") + + (query ? `?${query}` : ""), + ); +} + +function processFlake(flake: NixExpression, location: string): ImportedFlake { + return { + allChecks: { + tag: "check", + nixExpression: nixRaw`pkgs.linkFarm "all-checks" ${flake}.checks.\${system}`, + }, + + allPackages: mkPackage( + nixRaw`pkgs.linkFarm "all-packages" ${flake}.packages.\${system}`, + `All packages from flake.nix in ${flake}`, + ), + + getApp: (appName) => + mkExecutable( + getPathOrError( + flake, + ["apps", nixRaw`\${system}`, appName, "program"], + nixStrLit`The app "${appName}" was not found in ${location}`, + ), + `Execute ${appName} from flake.nix in ${flake}`, + ), + + getCheck: (checkName) => ({ + tag: "check", + nixExpression: getPathOrError( + flake, + ["checks", nixRaw`\${system}`, checkName], + nixStrLit`The check "${checkName}" was not found in ${location}`, + ), + }), + + getDevShell: (devShellName) => + mkEnvironment({ + nixExpression: getPathOrError( + flake, + ["devShells", nixRaw`\${system}`, devShellName], + nixStrLit`The devShell "${devShellName}" was not found in ${location}`, + ), + }), + + getPackage: (packageName) => + mkPackage( + getPathOrError( + flake, + ["packages", nixRaw`\${system}`, packageName], + nixStrLit`The package "${packageName}" was not found in ${location}`, + ), + `${packageName} from flake.nix in ${flake}`, + ), + }; +} diff --git a/ts/internal/interpolatedString.ts b/ts/internal/interpolatedString.ts index cd00f402..a82948e6 100644 --- a/ts/internal/interpolatedString.ts +++ b/ts/internal/interpolatedString.ts @@ -77,3 +77,10 @@ export function getInterpolations( ): Array { return interpolated.rest.map(([node, _str]) => node); } + +export function join(arr: Array, joiner: string): InterpolatedString { + return { + initial: "", + rest: arr.map((el, idx) => [el, idx < arr.length - 1 ? joiner : ""]), + }; +} diff --git a/ts/internal/utils.ts b/ts/internal/utils.ts index 655af3f5..1b40f5d0 100644 --- a/ts/internal/utils.ts +++ b/ts/internal/utils.ts @@ -41,20 +41,17 @@ export const mapKeys = ( return result; }; -/** - * Maps an object's values. Typescript currently does not keep track of the - * object structure, but this could easily be accomplished in the future in a - * backwards-compatible way. - */ -export const mapValues = ( - f: (i: T, key: string) => R, - x: Record, -): Record => { - const result: Record = {}; - for (const [key, value] of Object.entries(x)) { +export const mapValues = , FnResult>( + f: (i: Obj[keyof Obj], key: keyof Obj) => FnResult, + x: Obj, +): { [key in keyof Obj]: FnResult } => { + const result: Partial<{ [key in keyof Obj]: FnResult }> = {}; + for (const [key, value] of Object.entries(x) as Array< + [keyof Obj, Obj[keyof Obj]] + >) { result[key] = f(value, key); } - return result; + return result as { [key in keyof Obj]: FnResult }; }; export const filterNullValues = ( diff --git a/ts/mod.ts b/ts/mod.ts index c2195fc0..59260431 100644 --- a/ts/mod.ts +++ b/ts/mod.ts @@ -25,3 +25,4 @@ export * as javascript from "./javascript/mod.ts"; export { processCompose } from "./process_compose.ts"; export { deployToGhPages } from "./deployToGhPages.ts"; export * as nix from "./nix.ts"; +export * from "./importFlake.ts"; diff --git a/ts/nix.ts b/ts/nix.ts index 924c33b7..cc432081 100644 --- a/ts/nix.ts +++ b/ts/nix.ts @@ -3,6 +3,7 @@ import { InterpolatedString, interpolatedStringFromString, interpolatedStringFromTemplate, + join, mapStrings, renderInterpolatedString, } from "./internal/interpolatedString.ts"; @@ -210,6 +211,29 @@ export function escapeShellArg(shellArg: NixExpression): NixExpression { return nixRaw`(pkgs.lib.strings.escapeShellArg ${shellArg})`; } +export function getPathOrError( + attrSet: NixExpression, + path: Array, + error: NixExpression, +): NixExpression { + const pathExpr: NixExpression = { + [__nixExpressionTag]: null, + type: "raw", + raw: join( + path.map((el) => (typeof el === "string" ? nixStrLit(el) : el)), + ".", + ), + }; + return nixRaw` + let + x = ${attrSet}; + in + if x ? ${pathExpr} + then x.${pathExpr} + else builtins.throw ${error} + `; +} + /** * Returns a `NixExpression` that renders as an identifier that refers to a * flake input. At the same time it registers the flake input as a dependency, @@ -217,7 +241,7 @@ export function escapeShellArg(shellArg: NixExpression): NixExpression { * `renderFlakeFile` for an example. */ export function nixFlakeDep(name: string, dep: FlakeDep): NixExpression { - if (!name.match(/^[a-zA-Z0-9-]+$/)) { + if (!name.match(/^[a-zA-Z][a-zA-Z0-9-_]+$/)) { throw Error(`flakeDep: "${name}" is not a valid nix variable name`); } return { diff --git a/ts/package.test.ts b/ts/package.test.ts new file mode 100644 index 00000000..645ae2e9 --- /dev/null +++ b/ts/package.test.ts @@ -0,0 +1,16 @@ +import { describe, it } from "https://deno.land/std@0.206.0/testing/bdd.ts"; +import { mkPackage } from "./package.ts"; +import { nixRaw } from "./nix.ts"; +import { assertStdout, assertSuccess, runExecutable } from "./testUtils.ts"; + +describe("bin", () => { + it("escapes executable names", () => { + const pkg = mkPackage( + nixRaw`pkgs.writeScriptBin "bin with spaces" "echo running bin with spaces"`, + "pkg", + ); + const exe = pkg.bin("bin with spaces"); + const output = assertSuccess(runExecutable(exe)); + assertStdout(output, "running bin with spaces\n"); + }); +}); diff --git a/ts/package.ts b/ts/package.ts index dac43f67..b6ad264a 100644 --- a/ts/package.ts +++ b/ts/package.ts @@ -1,8 +1,10 @@ -import { Environment, sandboxScript } from "./environment.ts"; +import { Environment, sandboxScript, shell } from "./environment.ts"; +import { Executable } from "./executable.ts"; import { hasTag } from "./internal/utils.ts"; import { NixExpression, NixStrLitInterpolatable, + escapeShellArg, nixRaw, nixStrLit, toHumanReadable, @@ -26,6 +28,14 @@ export type Package = { * Update the description for this `Package` */ setDescription: (this: Package, newDescription: string) => Package; + + /** + * Get an executable by name within the `bin` directory of this `Package`. + * + * If that executable doesn't exist this will only fail when trying to *run* + * the `Executable`, not before. + */ + bin: (executableName: string) => Executable; }; export function isPackage(x: unknown): x is Package { @@ -50,6 +60,10 @@ export function mkPackage( description: newDescription, }; }, + + bin(this: Package, executableName: string): Executable { + return shell`${this}/bin/${escapeShellArg(nixStrLit(executableName))}`; + }, }; } diff --git a/ts/testUtils.ts b/ts/testUtils.ts index de0124e9..f0e19516 100644 --- a/ts/testUtils.ts +++ b/ts/testUtils.ts @@ -182,7 +182,7 @@ export const buildPackage = ( export const runInDevShell = ( devShell: garn.Environment, options: { cmd?: string; dir?: string } = {}, -): void => { +): Output => { const cmd = options.cmd ?? "true"; const dir = options.dir ?? Deno.makeTempDirSync({ prefix: "garn-test" }); const flakeFile = nix.renderFlakeFile( @@ -201,7 +201,7 @@ export const runInDevShell = ( }), ); Deno.writeTextFileSync(`${dir}/flake.nix`, flakeFile); - assertSuccess( + return assertSuccess( runCommand( new Deno.Command("nix", { args: ["develop", "-L", dir, "-c", "--", "bash", "-c", cmd], @@ -214,3 +214,15 @@ export const runInDevShell = ( export const testPkgs = { hello: mkPackage(nix.nixRaw("pkgs.hello"), "hello"), }; + +export const assertThrowsWith = ( + expectedErrorMessage: string, + fn: () => void, +) => { + try { + fn(); + throw `Expected fn to throw with ${expectedErrorMessage}, but it did not throw`; + } catch (err) { + assertEquals(err.message, expectedErrorMessage); + } +};