Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions packages/codev/dashboard/__tests__/analytics.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ function makeStats(overrides: Partial<AnalyticsResponse> = {}): AnalyticsRespons
issuesClosed: 6,
medianTimeToCloseBugsHours: 1.2,
projectsByProtocol: {
spir: { count: 3, avgWallClockHours: 48.2 },
bugfix: { count: 2, avgWallClockHours: 1.5 },
aspir: { count: 1, avgWallClockHours: 24.0 },
spir: { count: 3, avgWallClockHours: 48.2, avgAgentTimeHours: 0.75 },
bugfix: { count: 2, avgWallClockHours: 1.5, avgAgentTimeHours: 0.2 },
aspir: { count: 1, avgWallClockHours: 24.0, avgAgentTimeHours: null },
},
},
consultation: {
Expand Down
3 changes: 2 additions & 1 deletion packages/codev/dashboard/src/components/AnalyticsView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ function ActivitySection({ activity, errors }: { activity: AnalyticsResponse['ac
name: proto.toUpperCase(),
count: stats.count,
avgWallClock: stats.avgWallClockHours,
avgAgentTime: stats.avgAgentTimeHours,
}))
.sort((a, b) => b.count - a.count);

Expand All @@ -120,7 +121,7 @@ function ActivitySection({ activity, errors }: { activity: AnalyticsResponse['ac
<h4 className="analytics-sub-title">Projects by Protocol</h4>
<MetricGrid>
{protocolData.map(d => (
<Metric key={d.name} label={d.name} value={`${d.count} (avg ${fmtWallClock(d.avgWallClock)})`} />
<Metric key={d.name} label={d.name} value={`${d.count} (wall ${fmtWallClock(d.avgWallClock)}${d.avgAgentTime != null ? `, agent ${fmtWallClock(d.avgAgentTime)}` : ''})`} />
))}
</MetricGrid>
<MiniBarChart data={protocolData} dataKey="count" nameKey="name" />
Expand Down
1 change: 1 addition & 0 deletions packages/codev/dashboard/src/lib/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ export interface OverviewData {
export interface ProtocolStats {
count: number;
avgWallClockHours: number | null;
avgAgentTimeHours: number | null;
}

export interface AnalyticsResponse {
Expand Down
61 changes: 58 additions & 3 deletions packages/codev/src/agent-farm/__tests__/analytics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';

const execFileMock = vi.hoisted(() => vi.fn());
const mockSummary = vi.hoisted(() => vi.fn());
const mockAgentTimeByProtocol = vi.hoisted(() => vi.fn());
const mockClose = vi.hoisted(() => vi.fn());

// Mock child_process + util (for GitHub CLI calls in github.ts)
Expand All @@ -28,6 +29,7 @@ vi.mock('node:util', () => ({
vi.mock('../../commands/consult/metrics.js', () => ({
MetricsDB: class MockMetricsDB {
summary = mockSummary;
agentTimeByProtocol = mockAgentTimeByProtocol;
close = mockClose;
},
}));
Expand Down Expand Up @@ -218,6 +220,7 @@ describe('computeAnalytics', () => {
clearAnalyticsCache();
vi.clearAllMocks();
mockSummary.mockReturnValue(defaultSummary());
mockAgentTimeByProtocol.mockReturnValue([]);
});

it('assembles full statistics from all data sources', async () => {
Expand All @@ -240,9 +243,9 @@ describe('computeAnalytics', () => {
expect(result.activity.issuesClosed).toBe(2);
expect(result.activity.medianTimeToCloseBugsHours).toBeCloseTo(84); // 3.5 days for bug only (single item)
expect(result.activity).not.toHaveProperty('activeBuilders');
// Protocol breakdown now includes count + avgWallClockHours (no "on it" → falls back to PR times)
expect(result.activity.projectsByProtocol.spir).toEqual({ count: 1, avgWallClockHours: expect.closeTo(36) });
expect(result.activity.projectsByProtocol.aspir).toEqual({ count: 1, avgWallClockHours: expect.closeTo(24) });
// Protocol breakdown now includes count + avgWallClockHours + avgAgentTimeHours
expect(result.activity.projectsByProtocol.spir).toEqual({ count: 1, avgWallClockHours: expect.closeTo(36), avgAgentTimeHours: null });
expect(result.activity.projectsByProtocol.aspir).toEqual({ count: 1, avgWallClockHours: expect.closeTo(24), avgAgentTimeHours: null });
// Removed fields
expect(result.activity).not.toHaveProperty('projectsCompleted');
expect(result.activity).not.toHaveProperty('bugsFixed');
Expand Down Expand Up @@ -487,6 +490,58 @@ describe('computeAnalytics', () => {
expect(result.activity.projectsByProtocol).toEqual({});
});

// --- Agent time per protocol (#541) ---

it('includes avgAgentTimeHours from MetricsDB consultation durations', async () => {
mockAgentTimeByProtocol.mockReturnValue([
{ protocol: 'spir', avgAgentTimeSeconds: 2700, projectCount: 5 }, // 45 min
{ protocol: 'bugfix', avgAgentTimeSeconds: 720, projectCount: 10 }, // 12 min
]);
mockGhOutput({
mergedPRs: JSON.stringify([
{ number: 1, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-11T00:00:00Z', body: 'Fixes #42', headRefName: 'builder/spir-42-feature' },
{ number: 2, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-10T12:00:00Z', body: 'Fixes #100', headRefName: 'builder/bugfix-100-fix' },
]),
closedIssues: '[]',
});

const result = await computeAnalytics('/tmp/workspace', '7');
expect(result.activity.projectsByProtocol.spir?.avgAgentTimeHours).toBeCloseTo(0.75); // 2700/3600
expect(result.activity.projectsByProtocol.bugfix?.avgAgentTimeHours).toBeCloseTo(0.2); // 720/3600
});

it('returns null avgAgentTimeHours when no consultation data for that protocol', async () => {
mockAgentTimeByProtocol.mockReturnValue([
{ protocol: 'spir', avgAgentTimeSeconds: 1800, projectCount: 3 },
]);
mockGhOutput({
mergedPRs: JSON.stringify([
{ number: 1, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-11T00:00:00Z', body: 'Fixes #42', headRefName: 'builder/spir-42-feature' },
{ number: 2, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-10T12:00:00Z', body: 'Fixes #100', headRefName: 'builder/bugfix-100-fix' },
]),
closedIssues: '[]',
});

const result = await computeAnalytics('/tmp/workspace', '7');
expect(result.activity.projectsByProtocol.spir?.avgAgentTimeHours).toBeCloseTo(0.5); // 1800/3600
expect(result.activity.projectsByProtocol.bugfix?.avgAgentTimeHours).toBeNull();
});

it('handles agentTimeByProtocol failure gracefully', async () => {
mockAgentTimeByProtocol.mockImplementation(() => { throw new Error('DB locked'); });
mockGhOutput({
mergedPRs: JSON.stringify([
{ number: 1, title: 'PR', createdAt: '2026-02-10T00:00:00Z', mergedAt: '2026-02-11T00:00:00Z', body: 'Fixes #42', headRefName: 'builder/spir-42-feature' },
]),
closedIssues: '[]',
});

const result = await computeAnalytics('/tmp/workspace', '7');
// Should still return protocol stats, just with null agent time
expect(result.activity.projectsByProtocol.spir?.count).toBe(1);
expect(result.activity.projectsByProtocol.spir?.avgAgentTimeHours).toBeNull();
});

// --- Caching ---

it('returns cached result on second call within TTL', async () => {
Expand Down
21 changes: 21 additions & 0 deletions packages/codev/src/agent-farm/servers/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import { MetricsDB } from '../../commands/consult/metrics.js';
export interface ProtocolStats {
count: number;
avgWallClockHours: number | null;
avgAgentTimeHours: number | null;
}

export interface AnalyticsResponse {
Expand Down Expand Up @@ -269,6 +270,7 @@ export function protocolFromBranch(branch: string): string | null {
async function computeProjectsByProtocol(
mergedPRs: MergedPR[],
cwd: string,
agentTimeByProtocol?: Map<string, number>,
): Promise<Record<string, ProtocolStats>> {
// Group PRs by protocol and collect linked issue numbers
const byProtocol = new Map<string, MergedPR[]>();
Expand Down Expand Up @@ -319,11 +321,13 @@ async function computeProjectsByProtocol(
wallClockHours.push(ms / (1000 * 60 * 60));
}

const avgAgentSec = agentTimeByProtocol?.get(protocol);
result[protocol] = {
count: prs.length,
avgWallClockHours: wallClockHours.length > 0
? wallClockHours.reduce((a, b) => a + b, 0) / wallClockHours.length
: null,
avgAgentTimeHours: avgAgentSec != null ? avgAgentSec / 3600 : null,
};
}
return result;
Expand Down Expand Up @@ -394,10 +398,27 @@ export async function computeAnalytics(
};
}

// Agent time by protocol from consultation metrics
let agentTimeByProtocol: Map<string, number> | undefined;
try {
const db = new MetricsDB();
try {
const agentFilters: { days?: number; workspace: string } = { workspace: workspaceRoot };
if (days) agentFilters.days = days;
const agentTimeRows = db.agentTimeByProtocol(agentFilters);
agentTimeByProtocol = new Map(agentTimeRows.map(r => [r.protocol, r.avgAgentTimeSeconds]));
} finally {
db.close();
}
} catch {
// Agent time is best-effort; don't fail if MetricsDB is unavailable
}

// Protocol breakdown with avg wall clock times (from PR branch names + "on it" timestamps)
const projectsByProtocol = await computeProjectsByProtocol(
githubMetrics.mergedPRList,
workspaceRoot,
agentTimeByProtocol,
);

const result: AnalyticsResponse = {
Expand Down
26 changes: 26 additions & 0 deletions packages/codev/src/commands/consult/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,32 @@ export class MetricsDB {
};
}

agentTimeByProtocol(filters: StatsFilters): Array<{ protocol: string; avgAgentTimeSeconds: number; projectCount: number }> {
const { where, params } = buildWhereClause(filters);
const extraCondition = where
? 'AND project_id IS NOT NULL AND protocol IS NOT NULL'
: 'WHERE project_id IS NOT NULL AND protocol IS NOT NULL';

const rows = this.db.prepare(`
SELECT
protocol,
AVG(project_total) as avg_agent_time_seconds,
COUNT(*) as project_count
FROM (
SELECT protocol, project_id, SUM(duration_seconds) as project_total
FROM consultation_metrics ${where} ${extraCondition}
GROUP BY protocol, project_id
)
GROUP BY protocol
`).all(params) as Array<{ protocol: string; avg_agent_time_seconds: number; project_count: number }>;

return rows.map(r => ({
protocol: r.protocol,
avgAgentTimeSeconds: r.avg_agent_time_seconds,
projectCount: r.project_count,
}));
}

costByProject(filters: StatsFilters): Array<{ projectId: string; totalCost: number }> {
const { where, params } = buildWhereClause(filters);
const extraCondition = where
Expand Down