Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ jobs:
pnpm --filter @maschina/notifications build
pnpm --filter @maschina/validation build
pnpm --filter @maschina/email build
pnpm --filter @maschina/webhooks build
pnpm --filter @maschina/search build
pnpm --filter @maschina/compliance build

- run: pnpm typecheck

Expand Down Expand Up @@ -135,6 +138,9 @@ jobs:
pnpm --filter @maschina/notifications build
pnpm --filter @maschina/validation build
pnpm --filter @maschina/email build
pnpm --filter @maschina/webhooks build
pnpm --filter @maschina/search build
pnpm --filter @maschina/compliance build

- name: Run migrations
run: pnpm db:migrate
Expand Down
95 changes: 69 additions & 26 deletions packages/model/src/catalog.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,52 @@ import {
getAllowedModels,
getModel,
getModelMultiplier,
inferProvider,
resolveModel,
validateModelAccess,
} from "./catalog.js";

describe("getModel", () => {
it("returns the model def for a known ID", () => {
const m = getModel("claude-haiku-4-5-20251001");
it("returns the model def for a known Claude ID", () => {
const m = getModel("claude-haiku-4-5");
expect(m).toBeDefined();
expect(m?.provider).toBe("anthropic");
expect(m?.multiplier).toBe(1);
});

it("returns the model def for a known OpenAI ID", () => {
const m = getModel("gpt-5");
expect(m).toBeDefined();
expect(m?.provider).toBe("openai");
expect(m?.multiplier).toBe(8);
});

it("returns undefined for an unknown ID", () => {
expect(getModel("gpt-99")).toBeUndefined();
});
});

describe("getModelMultiplier", () => {
it("returns 1 for haiku", () => expect(getModelMultiplier("claude-haiku-4-5-20251001")).toBe(1));
it("returns 1 for haiku", () => expect(getModelMultiplier("claude-haiku-4-5")).toBe(1));
it("returns 3 for sonnet", () => expect(getModelMultiplier("claude-sonnet-4-6")).toBe(3));
it("returns 15 for opus", () => expect(getModelMultiplier("claude-opus-4-6")).toBe(15));
it("returns 0 for ollama models", () => expect(getModelMultiplier("ollama/llama3.2")).toBe(0));
it("returns 1 for unknown model (safe default)", () =>
expect(getModelMultiplier("unknown")).toBe(1));
it("returns 1 for gpt-5-mini", () => expect(getModelMultiplier("gpt-5-mini")).toBe(1));
it("returns 8 for gpt-5", () => expect(getModelMultiplier("gpt-5")).toBe(8));
it("returns 20 for o3", () => expect(getModelMultiplier("o3")).toBe(20));
it("returns 2 for unknown model (passthrough rate)", () =>
expect(getModelMultiplier("unknown-future-model")).toBe(2));
});

describe("inferProvider", () => {
it("infers anthropic for claude- prefix", () =>
expect(inferProvider("claude-sonnet-4-6")).toBe("anthropic"));
it("infers openai for gpt- prefix", () => expect(inferProvider("gpt-5")).toBe("openai"));
it("infers openai for o3 prefix", () => expect(inferProvider("o3-pro")).toBe("openai"));
it("infers openai for o4 prefix", () => expect(inferProvider("o4-mini")).toBe("openai"));
it("infers ollama for ollama/ prefix", () =>
expect(inferProvider("ollama/llama3.2")).toBe("ollama"));
it("returns null for unknown prefix", () => expect(inferProvider("gemini-pro")).toBeNull());
});

describe("getAllowedModels", () => {
Expand All @@ -36,42 +58,40 @@ describe("getAllowedModels", () => {
expect(allowed.every((m) => m.isLocal)).toBe(true);
});

it("m1 tier can use haiku and ollama", () => {
it("m1 tier can use haiku, gpt-5-mini, o4-mini, and ollama", () => {
const ids = getAllowedModels("m1").map((m) => m.id);
expect(ids).toContain("claude-haiku-4-5-20251001");
expect(ids).toContain("claude-haiku-4-5");
expect(ids).toContain("gpt-5-mini");
expect(ids).toContain("o4-mini");
expect(ids).toContain("ollama/llama3.2");
expect(ids).not.toContain("claude-sonnet-4-6");
expect(ids).not.toContain("claude-opus-4-6");
});

it("m5 tier can use haiku and sonnet but not opus", () => {
it("m5 tier can use sonnet and gpt-5 but not opus or o3", () => {
const ids = getAllowedModels("m5").map((m) => m.id);
expect(ids).toContain("claude-haiku-4-5-20251001");
expect(ids).toContain("claude-sonnet-4-6");
expect(ids).toContain("gpt-5");
expect(ids).not.toContain("claude-opus-4-6");
expect(ids).not.toContain("o3");
});

it("m10 tier can use all models", () => {
it("m10 tier can use all models including opus and o3", () => {
const ids = getAllowedModels("m10").map((m) => m.id);
expect(ids).toContain("claude-haiku-4-5-20251001");
expect(ids).toContain("claude-sonnet-4-6");
expect(ids).toContain("claude-opus-4-6");
});

it("internal tier can use all models", () => {
const ids = getAllowedModels("internal").map((m) => m.id);
expect(ids).toContain("claude-opus-4-6");
expect(ids).toContain("o3");
expect(ids).toContain("o3-pro");
expect(ids).toContain("gpt-5.4-pro");
});
});

describe("validateModelAccess", () => {
it("allows access tier to use ollama", () => {
const result = validateModelAccess("access", "ollama/llama3.2");
expect(result.allowed).toBe(true);
expect(validateModelAccess("access", "ollama/llama3.2").allowed).toBe(true);
});

it("denies access tier from using haiku", () => {
const result = validateModelAccess("access", "claude-haiku-4-5-20251001");
const result = validateModelAccess("access", "claude-haiku-4-5");
expect(result.allowed).toBe(false);
expect(result.reason).toMatch(/m1/);
});
Expand All @@ -89,30 +109,53 @@ describe("validateModelAccess", () => {
});

it("allows m10 tier to use opus", () => {
const result = validateModelAccess("m10", "claude-opus-4-6");
expect(validateModelAccess("m10", "claude-opus-4-6").allowed).toBe(true);
});

it("allows m1 tier to use gpt-5-mini", () => {
expect(validateModelAccess("m1", "gpt-5-mini").allowed).toBe(true);
});

it("allows m5 tier to use gpt-5", () => {
expect(validateModelAccess("m5", "gpt-5").allowed).toBe(true);
});

it("allows m1+ passthrough for unknown claude model", () => {
const result = validateModelAccess("m1", "claude-future-model-9");
expect(result.allowed).toBe(true);
expect(result.passthrough).toBe(true);
});

it("allows m1+ passthrough for unknown gpt model", () => {
const result = validateModelAccess("m1", "gpt-6");
expect(result.allowed).toBe(true);
expect(result.passthrough).toBe(true);
});

it("denies access tier passthrough", () => {
const result = validateModelAccess("access", "gpt-6");
expect(result.allowed).toBe(false);
});

it("denies unknown model with clear error", () => {
const result = validateModelAccess("enterprise", "gpt-99");
it("denies unknown prefix with no inferred provider", () => {
const result = validateModelAccess("enterprise", "gemini-pro");
expect(result.allowed).toBe(false);
expect(result.reason).toMatch(/Unknown model/);
});
});

describe("resolveModel", () => {
it("returns the requested model if allowed", () => {
expect(resolveModel("m5", "claude-haiku-4-5-20251001")).toBe("claude-haiku-4-5-20251001");
expect(resolveModel("m5", "claude-haiku-4-5")).toBe("claude-haiku-4-5");
});

it("falls back to tier default if requested model is denied", () => {
// m1 requesting opus → should fall back to m1 default
expect(resolveModel("m1", "claude-opus-4-6")).toBe(DEFAULT_MODEL.m1);
});

it("returns tier default when no model is requested", () => {
expect(resolveModel("access")).toBe("ollama/llama3.2");
expect(resolveModel("m1")).toBe("claude-haiku-4-5-20251001");
expect(resolveModel("m1")).toBe("claude-haiku-4-5");
expect(resolveModel("m5")).toBe("claude-sonnet-4-6");
expect(resolveModel("m10")).toBe("claude-opus-4-6");
});
Expand Down
Loading
Loading