feat(webhooks): outbound webhook delivery (#32)

* feat(model-routing): per-tier model access gates and billing multipliers

* fix(ci): update lockfile and fix runtime test package stubs

* fix(ci): build @maschina/model before typecheck and vitest jobs

* fix(model): add @maschina/tsconfig devDependency

* chore(ci): upgrade codeql to v4, add continue-on-error

Author: RustMunkey

.github/workflows/codeql.yml

@@ -27,15 +27,16 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v4
         with:
           languages: ${{ matrix.language }}
           queries: security-extended
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
+        uses: github/codeql-action/autobuild@v4
 
       - name: Perform analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v4
+        continue-on-error: true
         with:
           category: "/language:${{ matrix.language }}"

packages/db/src/schema/pg/enums.ts

@@ -116,6 +116,29 @@ export const orderStatusEnum = pgEnum("order_status", [
   "disputed",
 ]);
 
+// ─── Nodes / Compute Network ──────────────────────────────────────────────────
+export const nodeStatusEnum = pgEnum("node_status", [
+  "pending", // registered, awaiting first heartbeat / approval
+  "active", // online and accepting work
+  "suspended", // temporarily suspended (policy violation, poor performance)
+  "offline", // heartbeat timeout — was active, now unreachable
+  "banned", // permanently removed from the network
+]);
+
+// Tier determines verification level, task routing priority, and trust model:
+// micro      — RPi, SBCs, watches (data relay, tiny quantized models only)
+// edge       — Mac Minis, consumer desktops, GPU workstations
+// standard   — mid-range servers, general compute (stake + reputation model)
+// verified   — TEE-attested nodes (AMD SEV / Intel SGX) — premium routing
+// datacenter — enterprise server farms, data centers, GPU clusters
+export const nodeTierEnum = pgEnum("node_tier", [
+  "micro",
+  "edge",
+  "standard",
+  "verified",
+  "datacenter",
+]);
+
 // ─── Compliance ───────────────────────────────────────────────────────────────
 export const consentTypeEnum = pgEnum("consent_type", [
   "terms_of_service",

packages/db/src/schema/pg/index.ts

@@ -18,3 +18,4 @@ export * from "./notifications.js";
 export * from "./connectors.js";
 export * from "./marketplace.js";
 export * from "./misc.js";
+export * from "./nodes.js";

packages/db/src/schema/pg/nodes.ts

@@ -0,0 +1,176 @@
+import {
+  boolean,
+  index,
+  integer,
+  jsonb,
+  numeric,
+  pgTable,
+  text,
+  timestamp,
+  uniqueIndex,
+  uuid,
+} from "drizzle-orm/pg-core";
+import { nodeStatusEnum, nodeTierEnum } from "./enums.js";
+import { users } from "./users.js";
+
+// ─── Nodes ────────────────────────────────────────────────────────────────────
+// Registered compute nodes in the Maschina network. Every node runs the
+// services/runtime software and receives work from the daemon's EXECUTE phase.
+// The daemon currently routes to one internal runtime — this table is the
+// foundation for routing to any registered node.
+
+export const nodes = pgTable(
+  "nodes",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+
+    // Owner — the user or org that registered and operates this node
+    userId: uuid("user_id")
+      .notNull()
+      .references(() => users.id, { onDelete: "cascade" }),
+    orgId: uuid("org_id"),
+
+    name: text("name").notNull(),
+    description: text("description"),
+
+    status: nodeStatusEnum("status").notNull().default("pending"),
+    tier: nodeTierEnum("tier").notNull().default("standard"),
+
+    // Software version running on this node (semver)
+    version: text("version"),
+
+    // Geographic region for latency-aware routing
+    // e.g. "us-east", "eu-west", "ap-southeast"
+    region: text("region"),
+
+    // Internal URL the daemon routes to for this node (e.g. http://1.2.3.4:8001)
+    // Null for Maschina-operated nodes (resolved via internal DNS)
+    internalUrl: text("internal_url"),
+
+    // Economic security — staked USDC as collateral against misbehaviour
+    // Slashing reduces this. Zero stake = micro/edge tier only.
+    stakedUsdc: numeric("staked_usdc", { precision: 18, scale: 6 }).notNull().default("0"),
+
+    // Rolling reputation score (0–100). Updated by daemon ANALYZE phase.
+    reputationScore: numeric("reputation_score", { precision: 5, scale: 2 })
+      .notNull()
+      .default("50"),
+
+    // Lifetime counters — used for reputation calculation
+    totalTasksCompleted: integer("total_tasks_completed").notNull().default(0),
+    totalTasksFailed: integer("total_tasks_failed").notNull().default(0),
+    totalTasksTimedOut: integer("total_tasks_timed_out").notNull().default(0),
+
+    // Last time this node sent a heartbeat
+    lastHeartbeatAt: timestamp("last_heartbeat_at", { withTimezone: true }),
+
+    // TEE attestation — set when a verified-tier node submits attestation proof
+    teeAttested: boolean("tee_attested").notNull().default(false),
+    teeAttestedAt: timestamp("tee_attested_at", { withTimezone: true }),
+    teeProvider: text("tee_provider"), // "amd_sev" | "intel_sgx"
+
+    createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+    updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
+    suspendedAt: timestamp("suspended_at", { withTimezone: true }),
+    bannedAt: timestamp("banned_at", { withTimezone: true }),
+  },
+  (t) => ({
+    userIdIdx: index("nodes_user_id_idx").on(t.userId),
+    statusIdx: index("nodes_status_idx").on(t.status),
+    tierIdx: index("nodes_tier_idx").on(t.tier),
+    regionIdx: index("nodes_region_idx").on(t.region),
+    // Daemon queries active nodes by tier + region for routing decisions
+    routingIdx: index("nodes_routing_idx").on(t.status, t.tier, t.region),
+  }),
+);
+
+// ─── Node Capabilities ────────────────────────────────────────────────────────
+// Hardware and software capabilities advertised by each node.
+// Updated on node registration and whenever the node reports a change.
+// The daemon uses this to match tasks with capable nodes.
+
+export const nodeCapabilities = pgTable(
+  "node_capabilities",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    nodeId: uuid("node_id")
+      .notNull()
+      .unique()
+      .references(() => nodes.id, { onDelete: "cascade" }),
+
+    // CPU
+    cpuCores: integer("cpu_cores"),
+    cpuModel: text("cpu_model"), // e.g. "Apple M4 Pro", "AMD EPYC 9654"
+    architecture: text("architecture"), // "amd64" | "arm64"
+
+    // Memory + Storage
+    ramGb: numeric("ram_gb", { precision: 8, scale: 2 }),
+    storageGb: numeric("storage_gb", { precision: 10, scale: 2 }),
+
+    // GPU — null if no GPU present
+    hasGpu: boolean("has_gpu").notNull().default(false),
+    gpuModel: text("gpu_model"), // e.g. "NVIDIA H100", "Apple M4 Pro GPU"
+    gpuVramGb: numeric("gpu_vram_gb", { precision: 8, scale: 2 }),
+    gpuCount: integer("gpu_count"),
+
+    // OS
+    osType: text("os_type"), // "linux" | "macos" | "windows"
+    osVersion: text("os_version"),
+
+    // Concurrency — how many tasks this node can run simultaneously
+    maxConcurrentTasks: integer("max_concurrent_tasks").notNull().default(1),
+
+    // Network
+    networkBandwidthMbps: integer("network_bandwidth_mbps"),
+
+    // Model support — array of model IDs this node can serve
+    // e.g. ["ollama/llama3.2", "claude-haiku-4-5"]
+    // Anthropic/OpenAI models are available to all nodes with valid API keys.
+    // Ollama models depend on what's pulled locally.
+    supportedModels: jsonb("supported_models").notNull().default([]),
+
+    updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
+  },
+  (t) => ({
+    nodeIdIdx: uniqueIndex("node_capabilities_node_id_idx").on(t.nodeId),
+    hasGpuIdx: index("node_capabilities_has_gpu_idx").on(t.hasGpu),
+  }),
+);
+
+// ─── Node Heartbeats ─────────────────────────────────────────────────────────
+// Rolling health log. Nodes ping every N seconds. The daemon marks a node
+// offline if no heartbeat is received within the timeout window.
+// Kept for short-term trend analysis — old rows are pruned by retention policy.
+
+export const nodeHeartbeats = pgTable(
+  "node_heartbeats",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    nodeId: uuid("node_id")
+      .notNull()
+      .references(() => nodes.id, { onDelete: "cascade" }),
+
+    // Snapshot of resource utilisation at heartbeat time
+    cpuUsagePct: numeric("cpu_usage_pct", { precision: 5, scale: 2 }),
+    ramUsagePct: numeric("ram_usage_pct", { precision: 5, scale: 2 }),
+    activeTaskCount: integer("active_task_count").notNull().default(0),
+
+    // Derived health signal — set by the heartbeat handler
+    // "online" = healthy, "degraded" = high load or partial failure, "offline" = unreachable
+    healthStatus: text("health_status").notNull().default("online"),
+
+    recordedAt: timestamp("recorded_at", { withTimezone: true }).notNull().defaultNow(),
+  },
+  (t) => ({
+    nodeIdIdx: index("node_heartbeats_node_id_idx").on(t.nodeId),
+    // Most recent heartbeat per node is the common query
+    nodeRecordedIdx: index("node_heartbeats_node_recorded_idx").on(t.nodeId, t.recordedAt),
+  }),
+);
+
+export type Node = typeof nodes.$inferSelect;
+export type NewNode = typeof nodes.$inferInsert;
+export type NodeCapabilities = typeof nodeCapabilities.$inferSelect;
+export type NewNodeCapabilities = typeof nodeCapabilities.$inferInsert;
+export type NodeHeartbeat = typeof nodeHeartbeats.$inferSelect;
+export type NewNodeHeartbeat = typeof nodeHeartbeats.$inferInsert;

packages/db/src/schema/pg/relations.ts

@@ -13,6 +13,7 @@ import {
   reputationScores,
   walletAddresses,
 } from "./misc.js";
+import { nodeCapabilities, nodeHeartbeats, nodes } from "./nodes.js";
 import { notifications } from "./notifications.js";
 import { organizations } from "./organizations.js";
 import { plans } from "./plans.js";
@@ -41,6 +42,7 @@ export const usersRelations = relations(users, ({ one, many }) => ({
   wallets: many(walletAddresses),
   files: many(files),
   reputation: many(reputationScores),
+  nodes: many(nodes),
 }));
 
 export const userPasswordsRelations = relations(userPasswords, ({ one }) => ({
@@ -162,3 +164,22 @@ export const featureFlagOverridesRelations = relations(featureFlagOverrides, ({
 export const reputationScoresRelations = relations(reputationScores, ({ one }) => ({
   user: one(users, { fields: [reputationScores.userId], references: [users.id] }),
 }));
+
+// ─── Nodes ────────────────────────────────────────────────────────────────────
+
+export const nodesRelations = relations(nodes, ({ one, many }) => ({
+  user: one(users, { fields: [nodes.userId], references: [users.id] }),
+  capabilities: one(nodeCapabilities, {
+    fields: [nodes.id],
+    references: [nodeCapabilities.nodeId],
+  }),
+  heartbeats: many(nodeHeartbeats),
+}));
+
+export const nodeCapabilitiesRelations = relations(nodeCapabilities, ({ one }) => ({
+  node: one(nodes, { fields: [nodeCapabilities.nodeId], references: [nodes.id] }),
+}));
+
+export const nodeHeartbeatsRelations = relations(nodeHeartbeats, ({ one }) => ({
+  node: one(nodes, { fields: [nodeHeartbeats.nodeId], references: [nodes.id] }),
+}));

packages/jobs/src/dispatch.ts

@@ -88,3 +88,35 @@ export async function dispatchPruneSessions(): Promise<string> {
 export async function dispatchPruneTokens(): Promise<string> {
   return dispatch({ type: "maintenance.prune_tokens" });
 }
+
+// ─── Webhook dispatch (published to Python worker) ───────────────────────────
+// These jobs are consumed by services/worker (Python), not the daemon.
+// Subject: maschina.jobs.worker.webhook_dispatch
+// NOTE: not routed through jobSubject() — Python worker expects a flat subject.
+
+export async function dispatchWebhookJob(opts: {
+  deliveryId: string;
+  webhookId: string;
+  event: string;
+  payload: Record<string, unknown>;
+  attempt?: number;
+}): Promise<void> {
+  const js = await getJs();
+  const subject = "maschina.jobs.worker.webhook_dispatch";
+
+  const envelope = {
+    id: randomUUID(),
+    timestamp: new Date().toISOString(),
+    version: 1,
+    subject,
+    data: {
+      delivery_id: opts.deliveryId,
+      webhook_id: opts.webhookId,
+      event: opts.event,
+      payload: opts.payload,
+      attempt: opts.attempt ?? 1,
+    },
+  };
+
+  await js.publish(subject, new TextEncoder().encode(JSON.stringify(envelope)));
+}

packages/webhooks/src/deliver.ts

@@ -0,0 +1,88 @@
+import type { WebhookEventType, WebhookPayload } from "./events.js";
+import { HEADER, signPayload } from "./sign.js";
+
+export interface DeliveryResult {
+  success: boolean;
+  status: number | null;
+  body: string | null;
+  attempt: number;
+  durationMs: number;
+}
+
+// Exponential backoff delays (ms) per attempt: 10s, 30s, 90s, 5m, 15m
+const BACKOFF_MS = [10_000, 30_000, 90_000, 300_000, 900_000];
+export const MAX_ATTEMPTS = 5;
+
+/**
+ * Deliver a single webhook event to the given URL.
+ * Does not retry — the caller (worker) controls the retry loop.
+ */
+export async function deliverOnce(
+  url: string,
+  secret: string,
+  payload: WebhookPayload,
+  attempt: number,
+): Promise<DeliveryResult> {
+  const body = JSON.stringify(payload);
+  const signature = signPayload(body, secret);
+  const start = Date.now();
+
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "User-Agent": "Maschina-Webhook/1.0",
+        [HEADER]: signature,
+        "X-Maschina-Event": payload.type,
+        "X-Maschina-Delivery": payload.id,
+      },
+      body,
+      signal: AbortSignal.timeout(10_000), // 10s timeout per attempt
+    });
+
+    const resBody = await res.text().catch(() => null);
+
+    return {
+      success: res.ok,
+      status: res.status,
+      body: resBody?.slice(0, 500) ?? null, // cap stored response size
+      attempt,
+      durationMs: Date.now() - start,
+    };
+  } catch (err) {
+    return {
+      success: false,
+      status: null,
+      body: err instanceof Error ? err.message : "Unknown error",
+      attempt,
+      durationMs: Date.now() - start,
+    };
+  }
+}
+
+/**
+ * Returns the delay in ms before the next retry attempt.
+ * Returns null if no more retries should be attempted.
+ */
+export function nextRetryDelay(attempt: number): number | null {
+  if (attempt >= MAX_ATTEMPTS) return null;
+  return BACKOFF_MS[attempt - 1] ?? BACKOFF_MS[BACKOFF_MS.length - 1];
+}
+
+/**
+ * Build a typed webhook payload ready to dispatch.
+ */
+export function buildPayload<T extends WebhookPayload>(
+  type: WebhookEventType,
+  data: T["data"],
+  deliveryId: string,
+): WebhookPayload {
+  return {
+    id: deliveryId,
+    type,
+    created_at: new Date().toISOString(),
+    api_version: "2026-03-13",
+    data,
+  } as WebhookPayload;
+}