diff --git a/packages/evalite-tests/tests/fixtures/vitest-workspace/basics.eval.ts b/packages/evalite-tests/tests/fixtures/vitest-workspace/basics.eval.ts new file mode 100644 index 00000000..99455e41 --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/vitest-workspace/basics.eval.ts @@ -0,0 +1,17 @@ +import { evalite } from "evalite"; +import { Levenshtein } from "autoevals"; + +evalite("Basics", { + data: () => { + return [ + { + input: "abc", + expected: "abcdef", + }, + ]; + }, + task: async (input) => { + return input + "def"; + }, + scorers: [Levenshtein], +}); diff --git a/packages/evalite-tests/tests/fixtures/vitest-workspace/unit.test.ts b/packages/evalite-tests/tests/fixtures/vitest-workspace/unit.test.ts new file mode 100644 index 00000000..c5b57daa --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/vitest-workspace/unit.test.ts @@ -0,0 +1,6 @@ +import { it, expect } from "vitest"; + +it("should not run this unit test when running evalite", () => { + console.log("UNIT_TEST_RAN"); + expect(1 + 1).toBe(2); +}); diff --git a/packages/evalite-tests/tests/fixtures/vitest-workspace/vite.config.ts b/packages/evalite-tests/tests/fixtures/vitest-workspace/vite.config.ts new file mode 100644 index 00000000..aa6102fe --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/vitest-workspace/vite.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + workspace: [ + { + test: { + name: "unit", + include: ["**/*.test.ts"], + }, + }, + ], + }, +}); diff --git a/packages/evalite-tests/tests/vitest-workspace.test.ts b/packages/evalite-tests/tests/vitest-workspace.test.ts new file mode 100644 index 00000000..5d09e51a --- /dev/null +++ b/packages/evalite-tests/tests/vitest-workspace.test.ts @@ -0,0 +1,26 @@ +import { expect, it } from "vitest"; +import { getEvalsAsRecordViaStorage, loadFixture } from "./test-utils.js"; + +it("Should run evalite successfully despite vitest workspace config", async () => { + await using fixture = await loadFixture("vitest-workspace"); + + await fixture.run({ + mode: "run-once-and-exit", + }); + + const output = fixture.getOutput(); + + // Should not have "No suite present" or "No result present" errors + expect(output).not.toContain("No suite present"); + expect(output).not.toContain("No result present"); + + // Should not have run the unit test + // TODO: This currently fails - workspace config causes evalite to pick up .test.ts files + expect(output).not.toContain("UNIT_TEST_RAN"); + + // Should have run the eval successfully + // TODO: This currently fails - workspace config prevents .eval.ts files from being found + const evals = await getEvalsAsRecordViaStorage(fixture.storage); + expect(evals.Basics).toHaveLength(1); + expect(evals.Basics?.[0]?.status).toBe("success"); +});