From 4538898de70faa37fab031fdd44292633da2aa8e Mon Sep 17 00:00:00 2001 From: M Waleed Kadous Date: Tue, 24 Feb 2026 11:23:01 -0800 Subject: [PATCH 1/2] Fix #541: Track and display per-project agent time in analytics Add Phase 1 (consultation-based estimate) of agent time tracking: - MetricsDB.agentTimeByProtocol() aggregates consultation durations by project, then averages per protocol - Analytics server merges agent time into ProtocolStats alongside wall clock time, with graceful fallback on failure - Dashboard shows agent time next to wall clock in Activity section 112 LOC net diff across 5 files. --- .../src/components/AnalyticsView.tsx | 3 +- packages/codev/dashboard/src/lib/api.ts | 1 + .../agent-farm/__tests__/analytics.test.ts | 61 ++++++++++++++++++- .../codev/src/agent-farm/servers/analytics.ts | 21 +++++++ .../codev/src/commands/consult/metrics.ts | 26 ++++++++ 5 files changed, 108 insertions(+), 4 deletions(-) diff --git a/packages/codev/dashboard/src/components/AnalyticsView.tsx b/packages/codev/dashboard/src/components/AnalyticsView.tsx index 38b342b7..82e39014 100644 --- a/packages/codev/dashboard/src/components/AnalyticsView.tsx +++ b/packages/codev/dashboard/src/components/AnalyticsView.tsx @@ -110,6 +110,7 @@ function ActivitySection({ activity, errors }: { activity: AnalyticsResponse['ac name: proto.toUpperCase(), count: stats.count, avgWallClock: stats.avgWallClockHours, + avgAgentTime: stats.avgAgentTimeHours, })) .sort((a, b) => b.count - a.count); @@ -120,7 +121,7 @@ function ActivitySection({ activity, errors }: { activity: AnalyticsResponse['ac

Projects by Protocol

{protocolData.map(d => ( - + ))} diff --git a/packages/codev/dashboard/src/lib/api.ts b/packages/codev/dashboard/src/lib/api.ts index 14f3bf0d..4d01a3aa 100644 --- a/packages/codev/dashboard/src/lib/api.ts +++ b/packages/codev/dashboard/src/lib/api.ts @@ -146,6 +146,7 @@ export interface OverviewData { export interface ProtocolStats { count: number; avgWallClockHours: number | null; + avgAgentTimeHours: number | null; } export interface AnalyticsResponse { diff --git a/packages/codev/src/agent-farm/__tests__/analytics.test.ts b/packages/codev/src/agent-farm/__tests__/analytics.test.ts index 1d6b31ba..25a58034 100644 --- a/packages/codev/src/agent-farm/__tests__/analytics.test.ts +++ b/packages/codev/src/agent-farm/__tests__/analytics.test.ts @@ -14,6 +14,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; const execFileMock = vi.hoisted(() => vi.fn()); const mockSummary = vi.hoisted(() => vi.fn()); +const mockAgentTimeByProtocol = vi.hoisted(() => vi.fn()); const mockClose = vi.hoisted(() => vi.fn()); // Mock child_process + util (for GitHub CLI calls in github.ts) @@ -28,6 +29,7 @@ vi.mock('node:util', () => ({ vi.mock('../../commands/consult/metrics.js', () => ({ MetricsDB: class MockMetricsDB { summary = mockSummary; + agentTimeByProtocol = mockAgentTimeByProtocol; close = mockClose; }, })); @@ -218,6 +220,7 @@ describe('computeAnalytics', () => { clearAnalyticsCache(); vi.clearAllMocks(); mockSummary.mockReturnValue(defaultSummary()); + mockAgentTimeByProtocol.mockReturnValue([]); }); it('assembles full statistics from all data sources', async () => { @@ -240,9 +243,9 @@ describe('computeAnalytics', () => { expect(result.activity.issuesClosed).toBe(2); expect(result.activity.medianTimeToCloseBugsHours).toBeCloseTo(84); // 3.5 days for bug only (single item) expect(result.activity).not.toHaveProperty('activeBuilders'); - // Protocol breakdown now includes count + avgWallClockHours (no "on it" → falls back to PR times) - expect(result.activity.projectsByProtocol.spir).toEqual({ count: 1, avgWallClockHours: expect.closeTo(36) }); - expect(result.activity.projectsByProtocol.aspir).toEqual({ count: 1, avgWallClockHours: expect.closeTo(24) }); + // Protocol breakdown now includes count + avgWallClockHours + avgAgentTimeHours + expect(result.activity.projectsByProtocol.spir).toEqual({ count: 1, avgWallClockHours: expect.closeTo(36), avgAgentTimeHours: null }); + expect(result.activity.projectsByProtocol.aspir).toEqual({ count: 1, avgWallClockHours: expect.closeTo(24), avgAgentTimeHours: null }); // Removed fields expect(result.activity).not.toHaveProperty('projectsCompleted'); expect(result.activity).not.toHaveProperty('bugsFixed'); @@ -487,6 +490,58 @@ describe('computeAnalytics', () => { expect(result.activity.projectsByProtocol).toEqual({}); }); + // --- Agent time per protocol (#541) --- + + it('includes avgAgentTimeHours from MetricsDB consultation durations', async () => { + mockAgentTimeByProtocol.mockReturnValue([ + { protocol: 'spir', avgAgentTimeSeconds: 2700, projectCount: 5 }, // 45 min + { protocol: 'bugfix', avgAgentTimeSeconds: 720, projectCount: 10 }, // 12 min + ]); + mockGhOutput({ + mergedPRs: JSON.stringify([ + { number: 1, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-11T00:00:00Z', body: 'Fixes #42', headRefName: 'builder/spir-42-feature' }, + { number: 2, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-10T12:00:00Z', body: 'Fixes #100', headRefName: 'builder/bugfix-100-fix' }, + ]), + closedIssues: '[]', + }); + + const result = await computeAnalytics('/tmp/workspace', '7'); + expect(result.activity.projectsByProtocol.spir?.avgAgentTimeHours).toBeCloseTo(0.75); // 2700/3600 + expect(result.activity.projectsByProtocol.bugfix?.avgAgentTimeHours).toBeCloseTo(0.2); // 720/3600 + }); + + it('returns null avgAgentTimeHours when no consultation data for that protocol', async () => { + mockAgentTimeByProtocol.mockReturnValue([ + { protocol: 'spir', avgAgentTimeSeconds: 1800, projectCount: 3 }, + ]); + mockGhOutput({ + mergedPRs: JSON.stringify([ + { number: 1, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-11T00:00:00Z', body: 'Fixes #42', headRefName: 'builder/spir-42-feature' }, + { number: 2, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-10T12:00:00Z', body: 'Fixes #100', headRefName: 'builder/bugfix-100-fix' }, + ]), + closedIssues: '[]', + }); + + const result = await computeAnalytics('/tmp/workspace', '7'); + expect(result.activity.projectsByProtocol.spir?.avgAgentTimeHours).toBeCloseTo(0.5); // 1800/3600 + expect(result.activity.projectsByProtocol.bugfix?.avgAgentTimeHours).toBeNull(); + }); + + it('handles agentTimeByProtocol failure gracefully', async () => { + mockAgentTimeByProtocol.mockImplementation(() => { throw new Error('DB locked'); }); + mockGhOutput({ + mergedPRs: JSON.stringify([ + { number: 1, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-11T00:00:00Z', body: 'Fixes #42', headRefName: 'builder/spir-42-feature' }, + ]), + closedIssues: '[]', + }); + + const result = await computeAnalytics('/tmp/workspace', '7'); + // Should still return protocol stats, just with null agent time + expect(result.activity.projectsByProtocol.spir?.count).toBe(1); + expect(result.activity.projectsByProtocol.spir?.avgAgentTimeHours).toBeNull(); + }); + // --- Caching --- it('returns cached result on second call within TTL', async () => { diff --git a/packages/codev/src/agent-farm/servers/analytics.ts b/packages/codev/src/agent-farm/servers/analytics.ts index 52ba3a29..3889d881 100644 --- a/packages/codev/src/agent-farm/servers/analytics.ts +++ b/packages/codev/src/agent-farm/servers/analytics.ts @@ -25,6 +25,7 @@ import { MetricsDB } from '../../commands/consult/metrics.js'; export interface ProtocolStats { count: number; avgWallClockHours: number | null; + avgAgentTimeHours: number | null; } export interface AnalyticsResponse { @@ -269,6 +270,7 @@ export function protocolFromBranch(branch: string): string | null { async function computeProjectsByProtocol( mergedPRs: MergedPR[], cwd: string, + agentTimeByProtocol?: Map, ): Promise> { // Group PRs by protocol and collect linked issue numbers const byProtocol = new Map(); @@ -319,11 +321,13 @@ async function computeProjectsByProtocol( wallClockHours.push(ms / (1000 * 60 * 60)); } + const avgAgentSec = agentTimeByProtocol?.get(protocol); result[protocol] = { count: prs.length, avgWallClockHours: wallClockHours.length > 0 ? wallClockHours.reduce((a, b) => a + b, 0) / wallClockHours.length : null, + avgAgentTimeHours: avgAgentSec != null ? avgAgentSec / 3600 : null, }; } return result; @@ -394,10 +398,27 @@ export async function computeAnalytics( }; } + // Agent time by protocol from consultation metrics + let agentTimeByProtocol: Map | undefined; + try { + const db = new MetricsDB(); + try { + const agentFilters: { days?: number; workspace: string } = { workspace: workspaceRoot }; + if (days) agentFilters.days = days; + const agentTimeRows = db.agentTimeByProtocol(agentFilters); + agentTimeByProtocol = new Map(agentTimeRows.map(r => [r.protocol, r.avgAgentTimeSeconds])); + } finally { + db.close(); + } + } catch { + // Agent time is best-effort; don't fail if MetricsDB is unavailable + } + // Protocol breakdown with avg wall clock times (from PR branch names + "on it" timestamps) const projectsByProtocol = await computeProjectsByProtocol( githubMetrics.mergedPRList, workspaceRoot, + agentTimeByProtocol, ); const result: AnalyticsResponse = { diff --git a/packages/codev/src/commands/consult/metrics.ts b/packages/codev/src/commands/consult/metrics.ts index abb8af67..b01e6837 100644 --- a/packages/codev/src/commands/consult/metrics.ts +++ b/packages/codev/src/commands/consult/metrics.ts @@ -326,6 +326,32 @@ export class MetricsDB { }; } + agentTimeByProtocol(filters: StatsFilters): Array<{ protocol: string; avgAgentTimeSeconds: number; projectCount: number }> { + const { where, params } = buildWhereClause(filters); + const extraCondition = where + ? 'AND project_id IS NOT NULL AND protocol IS NOT NULL' + : 'WHERE project_id IS NOT NULL AND protocol IS NOT NULL'; + + const rows = this.db.prepare(` + SELECT + protocol, + AVG(project_total) as avg_agent_time_seconds, + COUNT(*) as project_count + FROM ( + SELECT protocol, project_id, SUM(duration_seconds) as project_total + FROM consultation_metrics ${where} ${extraCondition} + GROUP BY protocol, project_id + ) + GROUP BY protocol + `).all(params) as Array<{ protocol: string; avg_agent_time_seconds: number; project_count: number }>; + + return rows.map(r => ({ + protocol: r.protocol, + avgAgentTimeSeconds: r.avg_agent_time_seconds, + projectCount: r.project_count, + })); + } + costByProject(filters: StatsFilters): Array<{ projectId: string; totalCost: number }> { const { where, params } = buildWhereClause(filters); const extraCondition = where From 990bc21733702ac59f91c7e4d2d0e743a88f50ac Mon Sep 17 00:00:00 2001 From: M Waleed Kadous Date: Tue, 24 Feb 2026 11:28:03 -0800 Subject: [PATCH 2/2] Fix #541: Update dashboard test mock with avgAgentTimeHours field Address CMAP review feedback: dashboard analytics test fixture was missing the new avgAgentTimeHours property on ProtocolStats. --- packages/codev/dashboard/__tests__/analytics.test.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/codev/dashboard/__tests__/analytics.test.tsx b/packages/codev/dashboard/__tests__/analytics.test.tsx index ce1c2657..65fa3618 100644 --- a/packages/codev/dashboard/__tests__/analytics.test.tsx +++ b/packages/codev/dashboard/__tests__/analytics.test.tsx @@ -31,9 +31,9 @@ function makeStats(overrides: Partial = {}): AnalyticsRespons issuesClosed: 6, medianTimeToCloseBugsHours: 1.2, projectsByProtocol: { - spir: { count: 3, avgWallClockHours: 48.2 }, - bugfix: { count: 2, avgWallClockHours: 1.5 }, - aspir: { count: 1, avgWallClockHours: 24.0 }, + spir: { count: 3, avgWallClockHours: 48.2, avgAgentTimeHours: 0.75 }, + bugfix: { count: 2, avgWallClockHours: 1.5, avgAgentTimeHours: 0.2 }, + aspir: { count: 1, avgWallClockHours: 24.0, avgAgentTimeHours: null }, }, }, consultation: {