feat: support extra watch file globs in Evalite

.changeset/busy-sheep-burn.md

@@ -0,0 +1,5 @@
+---
+"evalite": minor
+---
+
+Added forceRerunTriggers to the config to match Vitest's version.

apps/evalite-docs/src/content/docs/api/cli.mdx

@@ -74,6 +74,29 @@ evalite watch path/to/eval.eval.ts
 
 **Note:** `--outputPath` is not supported in watch mode.
 
+#### Watching Additional Files
+
+By default, `evalite watch` only triggers reruns when your `*.eval.ts` files change.
+
+If your evals depend on other files that Vitest can't automatically detect (e.g., prompt templates, external data files, or CLI build outputs), you can configure extra watch globs in `evalite.config.ts`:
+
+```ts
+// evalite.config.ts
+import { defineConfig } from "evalite/config";
+
+export default defineConfig({
+  forceRerunTriggers: [
+    "src/**/*.ts", // helper / model code
+    "prompts/**/*", // prompt templates
+    "data/**/*.json", // test data
+  ],
+});
+```
+
+These globs are passed through to Vitest's [`forceRerunTriggers`](https://vitest.dev/config/#forcereruntriggers) option, so any change to a matching file will trigger a full eval rerun.
+
+> **Note:** Globs are resolved relative to the directory where you run evalite (the Evalite cwd).
+
 **Examples:**
 
 ```bash

apps/evalite-docs/src/content/docs/api/define-config.mdx

@@ -18,6 +18,9 @@ defineConfig(config: {
   maxConcurrency?: number;
   trialCount?: number;
   setupFiles?: string[];
+  cache?: boolean;
+  viteConfig?: ViteUserConfig;
+  forceRerunTriggers?: string[];
 }): Evalite.Config
 ```
 
@@ -167,6 +170,57 @@ export default defineConfig({
 
 **Note:** `.env` files are loaded automatically via `dotenv/config` - no need to configure them here.
 
+### `cache`
+
+**Type:** `boolean`
+
+**Default:** `true`
+
+Enable or disable caching of AI SDK model outputs. See [Vercel AI SDK caching](/tips/vercel-ai-sdk#caching) for details.
+
+```typescript
+export default defineConfig({
+  cache: false, // Disable cache entirely
+});
+```
+
+### `viteConfig`
+
+**Type:** `ViteUserConfig`
+
+Pass-through Vite/Vitest configuration options. This allows you to import and use your existing `vite.config.ts` explicitly.
+
+```typescript
+import { defineConfig } from "evalite/config";
+import viteConfig from "./vite.config.ts";
+
+export default defineConfig({
+  viteConfig: viteConfig,
+});
+```
+
+**Note:** `testTimeout`, `maxConcurrency`, and `setupFiles` must be configured at the root level of `evalite.config.ts`, not in `viteConfig.test`.
+
+### `forceRerunTriggers`
+
+**Type:** `string[]`
+
+**Default:** `[]`
+
+Extra file globs that trigger eval reruns in watch mode. This maps onto Vitest's [`forceRerunTriggers`](https://vitest.dev/config/#forcereruntriggers) option.
+
+```typescript
+export default defineConfig({
+  forceRerunTriggers: [
+    "src/**/*.ts", // helper / model code
+    "prompts/**/*", // prompt templates
+    "data/**/*.json", // test data
+  ],
+});
+```
+
+Useful when your evals depend on files that Vitest can't automatically detect as dependencies (e.g., prompt templates, external data files).
+
 ## Complete Example
 
 ```typescript
@@ -196,6 +250,12 @@ export default defineConfig({
 
   // Setup
   setupFiles: ["./test-setup.ts"],
+
+  // Caching
+  cache: true,
+
+  // Watch mode triggers
+  forceRerunTriggers: ["src/**/*.ts", "prompts/**/*"],
 });
 ```
 

apps/evalite-docs/src/content/docs/api/run-evalite.mdx

@@ -15,6 +15,7 @@ runEvalite(opts: {
   outputPath?: string;
   hideTable?: boolean;
   storage?: Evalite.Storage;
+  forceRerunTriggers?: string[];
 }): Promise<void>
 ```
 
@@ -140,6 +141,23 @@ await runEvalite({
 
 See [Storage](/api/storage) for more details.
 
+### `opts.forceRerunTriggers`
+
+**Type:** `string[]` (optional)
+
+Extra file globs that trigger eval reruns in watch mode. This overrides any `forceRerunTriggers` setting in `evalite.config.ts`.
+
+```typescript
+await runEvalite({
+  mode: "watch-for-file-changes",
+  forceRerunTriggers: ["src/**/*.ts", "prompts/**/*", "data/**/*.json"],
+});
+```
+
+This is useful when your evals depend on files that aren't automatically detected as dependencies (e.g., prompt templates, external data files).
+
+> **Tip:** `forceRerunTriggers` globs are evaluated relative to the `cwd` you pass (or `process.cwd()` if unspecified).
+
 ## Usage Examples
 
 ### Basic CI/CD Script

apps/evalite-docs/src/content/docs/guides/configuration.mdx

@@ -36,6 +36,7 @@ export default defineConfig({
 - **`setupFiles`**: Array of file paths to run before tests (e.g., for loading environment variables).
 - **`cache`**: Enable or disable caching of AI SDK model outputs. Default is true. See [Vercel AI SDK](/tips/vercel-ai-sdk#caching) for details.
 - **`viteConfig`**: Pass through Vite/Vitest configuration options. This allows you to import and use your existing vite.config.ts explicitly.
+- **`forceRerunTriggers`**: Extra file globs that trigger eval reruns in watch mode. This maps onto Vitest's `test.forceRerunTriggers` option (globs resolved relative to the directory where you run Evalite). See [Watching Additional Files](/api/cli#watching-additional-files).
 
 ## Important Configuration Options
 

packages/evalite-tests/tests/fixtures/config-watchfiles/evalite.config.ts

@@ -0,0 +1,5 @@
+import { defineConfig } from "evalite/config";
+
+export default defineConfig({
+  forceRerunTriggers: ["src/**/*.ts", "data/**/*.json"],
+});

packages/evalite-tests/tests/fixtures/config-watchfiles/test.eval.ts

@@ -0,0 +1,7 @@
+import { evalite } from "evalite";
+
+evalite("WatchFiles Config Test", {
+  data: () => [{ input: "hello", expected: "hello" }],
+  task: async (input) => input,
+  scorers: [],
+});

packages/evalite-tests/tests/test-utils.ts

@@ -1,11 +1,12 @@
 import { randomUUID } from "crypto";
+import type { Evalite } from "evalite";
+import { DB_LOCATION } from "evalite/backend-only-constants";
+import { createInMemoryStorage } from "evalite/in-memory-storage";
+import { runEvalite } from "evalite/runner";
 import { cpSync, rmSync } from "fs";
 import path from "path";
 import { Writable } from "stream";
 import stripAnsi from "strip-ansi";
-import type { Evalite } from "evalite";
-import { runEvalite } from "evalite/runner";
-import { createInMemoryStorage } from "evalite/in-memory-storage";
 import type { Vitest } from "vitest/node";
 
 const FIXTURES_DIR = path.join(import.meta.dirname, "./fixtures");
@@ -29,7 +30,7 @@ export const loadFixture = async (
 
   const captured = captureStdout();
 
-  let vitestInstance: Vitest | undefined = undefined;
+  let vitestInstance: Vitest | undefined;
 
   return {
     dir: dirPath,
@@ -69,6 +70,7 @@ export const loadFixture = async (
        * Enable cache for AI SDK model outputs.
        */
       cacheEnabled?: boolean;
+      forceRerunTriggers?: string[];
     }) => {
       const result = await runEvalite({
         ...opts,

packages/evalite-tests/tests/watch-files.test.ts

@@ -0,0 +1,57 @@
+import { expect, it } from "vitest";
+import { getSuitesAsRecordViaStorage, loadFixture } from "./test-utils.js";
+
+it("forceRerunTriggers in evalite.config.ts should configure Vitest forceRerunTriggers", async () => {
+  await using fixture = await loadFixture("config-watchfiles");
+
+  const vitest = await fixture.run({
+    mode: "run-once-and-exit",
+  });
+
+  // Verify the forceRerunTriggers includes our configured triggers
+  const forceRerunTriggers = vitest.config.forceRerunTriggers;
+
+  expect(forceRerunTriggers).toContain("src/**/*.ts");
+  expect(forceRerunTriggers).toContain("data/**/*.json");
+
+  const suites = await getSuitesAsRecordViaStorage(fixture.storage);
+
+  // Should complete successfully
+  expect(suites["WatchFiles Config Test"]).toHaveLength(1);
+  expect(suites["WatchFiles Config Test"]?.[0]?.status).toBe("success");
+});
+
+it("forceRerunTriggers passed to runEvalite should override evalite.config.ts", async () => {
+  await using fixture = await loadFixture("config-watchfiles");
+
+  // Override the config's forceRerunTriggers with different values
+  const vitest = await fixture.run({
+    mode: "run-once-and-exit",
+    forceRerunTriggers: ["custom/**/*.md"],
+  });
+
+  const forceRerunTriggers = vitest.config.forceRerunTriggers;
+
+  // Should contain the override value
+  expect(forceRerunTriggers).toContain("custom/**/*.md");
+
+  // Should NOT contain the config file values since we overrode them
+  expect(forceRerunTriggers).not.toContain("src/**/*.ts");
+  expect(forceRerunTriggers).not.toContain("data/**/*.json");
+});
+
+it("empty forceRerunTriggers array should not add any extra triggers", async () => {
+  await using fixture = await loadFixture("config-watchfiles");
+
+  // Override with empty array - should result in only Vitest defaults
+  const vitest = await fixture.run({
+    mode: "run-once-and-exit",
+    forceRerunTriggers: [],
+  });
+
+  const forceRerunTriggers = vitest.config.forceRerunTriggers;
+
+  // Should NOT contain the config file values since we overrode with empty array
+  expect(forceRerunTriggers).not.toContain("src/**/*.ts");
+  expect(forceRerunTriggers).not.toContain("data/**/*.json");
+});