diff --git a/ROADMAP.md b/ROADMAP.md index 58db654..e0d5caa 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -102,7 +102,7 @@ As usage grows, the platform needs stronger derived data pipelines, performance - [ ] [P1] Cursor-specific workflow fingerprinting and session archetype mapping - [x] [P1] Session archetype detection: debugging, feature delivery, refactor, migration, docs, investigation - [ ] [P1] Delegation graph capture for multi-agent and subagent workflows -- [ ] [P2] Exemplar session library for high-value workflows and onboarding examples +- [x] [P2] Exemplar session library for high-value workflows and onboarding examples - [ ] [P2] Prompt, skill, and template reuse analytics by workflow and outcome ## Friction & Bottleneck Analysis @@ -144,7 +144,7 @@ As usage grows, the platform needs stronger derived data pipelines, performance - [x] Onboarding velocity scoring - [x] Onboarding recommendations - [x] Shared behavior pattern discovery with approach comparison -- [ ] [P1] Bright spot detection: explicitly surface high performers and cross-pollinate their patterns +- [x] [P1] Bright spot detection: explicitly surface high performers and cross-pollinate their patterns - [ ] [P1] Exemplar-session-to-learning-path pipeline - [ ] [P1] Team skill gap mapping by workflow, tool category, and project context - [ ] [P2] Coaching program measurement: which onboarding or training changes improved outcomes @@ -206,7 +206,8 @@ As usage grows, the platform needs stronger derived data pipelines, performance - [x] Budget tracking with burn-rate alerts and projected overrun warnings - [x] Cost per successful outcome metric - [ ] [P1] Break-even analysis for API vs seat-based pricing with per-engineer recommendations -- [ ] [P1] Cost per workflow archetype and cost per engineering outcome +- [x] [P1] Cost per workflow archetype and cost per engineering outcome +- [x] [P1] Workflow compare mode for archetype and fingerprint performance - [ ] [P1] Model-choice opportunity scoring for overspend reduction - [ ] [P2] Budget policy simulation by team, project, and billing model @@ -222,6 +223,11 @@ As usage grows, the platform needs stronger derived data pipelines, performance - [ ] [P2] Weekly manager review packs that combine quality, friction, growth, and cost - [ ] [P2] Recommendation narratives that explain why a workflow is likely to help +## Website & Positioning + +- [ ] [P1] Reposition the website around workflow intelligence for agentic engineering +- [ ] [P1] Showcase workflow cost, quality, compare mode, and exemplar sessions as the core proof points + ## Interventions & Experimentation - [x] [P0] Recommendation-to-intervention workflow with owner, status, due date, and linked evidence diff --git a/frontend/src/components/growth/__tests__/exemplar-session-library.test.tsx b/frontend/src/components/growth/__tests__/exemplar-session-library.test.tsx new file mode 100644 index 0000000..695fb7a --- /dev/null +++ b/frontend/src/components/growth/__tests__/exemplar-session-library.test.tsx @@ -0,0 +1,67 @@ +import { render, screen } from "@testing-library/react" +import { MemoryRouter } from "react-router-dom" + +import { ExemplarSessionLibrary } from "@/components/growth/exemplar-session-library" + +describe("ExemplarSessionLibrary", () => { + it("renders an empty state", () => { + render( + + + , + ) + + expect(screen.getByText("No exemplar sessions identified yet.")).toBeInTheDocument() + }) + + it("renders workflow-aware exemplar cards", () => { + render( + + edit -> execute -> fix", + summary: "Backed by 3 engineers across 4 sessions.", + why_selected: "Chosen as the fastest successful example for debugging on primer.", + session_id: "sess-1", + engineer_id: "eng-1", + engineer_name: "Alice Example", + project_name: "primer", + outcome: "success", + helpfulness: "very_helpful", + session_summary: "Resolved the failing auth flow after reproducing locally.", + duration_seconds: 180, + estimated_cost: 1.24, + tools_used: ["Read", "Edit", "Bash"], + workflow_archetype: "debugging", + workflow_fingerprint: "debugging: read -> edit -> execute -> fix", + workflow_steps: ["read", "edit", "execute", "fix"], + supporting_session_count: 4, + supporting_engineer_count: 3, + supporting_pattern_count: 2, + success_rate: 1, + linked_patterns: [ + { + cluster_id: "cluster-1", + cluster_type: "session_type", + cluster_label: "debugging on primer", + session_count: 4, + engineer_count: 3, + success_rate: 1, + }, + ], + }, + ]} + /> + , + ) + + expect(screen.getAllByText("debugging: read -> edit -> execute -> fix")).toHaveLength(2) + expect(screen.getByText("Why This Exemplar")).toBeInTheDocument() + expect(screen.getByText("Workflow Evidence")).toBeInTheDocument() + expect(screen.getByText("$1.24")).toBeInTheDocument() + expect(screen.getByText("debugging on primer")).toBeInTheDocument() + expect(screen.getByText("Open exemplar")).toHaveAttribute("href", "/sessions/sess-1") + }) +}) diff --git a/frontend/src/components/growth/exemplar-session-library.tsx b/frontend/src/components/growth/exemplar-session-library.tsx new file mode 100644 index 0000000..f9cd3ce --- /dev/null +++ b/frontend/src/components/growth/exemplar-session-library.tsx @@ -0,0 +1,152 @@ +import { Link } from "react-router-dom" + +import { Badge } from "@/components/ui/badge" +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" +import { formatCost, formatDuration, formatLabel, formatPercent, titleize } from "@/lib/utils" +import type { ExemplarSession } from "@/types/api" + +interface ExemplarSessionLibraryProps { + exemplars: ExemplarSession[] +} + +export function ExemplarSessionLibrary({ exemplars }: ExemplarSessionLibraryProps) { + if (exemplars.length === 0) { + return ( +
+ No exemplar sessions identified yet. +
+ ) + } + + return ( +
+ {exemplars.map((exemplar) => ( + + +
+
+ {exemplar.title} +

{exemplar.summary}

+
+
+ {exemplar.workflow_archetype && ( + {formatLabel(exemplar.workflow_archetype)} + )} + {exemplar.project_name && ( + {exemplar.project_name} + )} +
+
+ + +
+
+

Support

+

{exemplar.supporting_engineer_count}

+

+ {exemplar.supporting_session_count} sessions +

+
+
+

Success

+

{formatPercent(exemplar.success_rate)}

+
+
+

Duration

+

{formatDuration(exemplar.duration_seconds)}

+
+
+

Cost

+

+ {exemplar.estimated_cost != null ? formatCost(exemplar.estimated_cost) : "-"} +

+
+
+ +
+

+ Why This Exemplar +

+

{exemplar.why_selected}

+
+ + {exemplar.session_summary && ( +
+

+ Session Summary +

+

{exemplar.session_summary}

+
+ )} + +
+
+

+ Workflow Evidence +

+
+ {exemplar.workflow_fingerprint ? ( + {exemplar.workflow_fingerprint} + ) : ( + No workflow fingerprint yet + )} + {exemplar.workflow_steps.map((step) => ( + + {titleize(step)} + + ))} +
+
+ +
+

+ Exemplar Tools +

+
+ {exemplar.tools_used.length === 0 ? ( + No tool telemetry + ) : ( + exemplar.tools_used.map((tool) => ( + + {tool} + + )) + )} +
+
+
+ +
+

+ Linked Patterns +

+
+ {exemplar.linked_patterns.map((pattern) => ( + + {pattern.cluster_label} + + ))} +
+
+ +
+
+

{exemplar.engineer_name}

+

+ {exemplar.outcome ? formatLabel(exemplar.outcome) : "Outcome unknown"} + {exemplar.helpfulness ? ` • ${formatLabel(exemplar.helpfulness)}` : ""} +

+
+ + Open exemplar + +
+ + + ))} +
+ ) +} diff --git a/frontend/src/pages/__tests__/growth.test.tsx b/frontend/src/pages/__tests__/growth.test.tsx index ffd1e2a..474b0c4 100644 --- a/frontend/src/pages/__tests__/growth.test.tsx +++ b/frontend/src/pages/__tests__/growth.test.tsx @@ -37,6 +37,9 @@ vi.mock("@/components/growth/pattern-summary", () => ({ vi.mock("@/components/growth/bright-spot-cards", () => ({ BrightSpotCards: () =>
bright spots
, })) +vi.mock("@/components/growth/exemplar-session-library", () => ({ + ExemplarSessionLibrary: () =>
exemplar session library
, +})) vi.mock("@/components/growth/shared-pattern-card", () => ({ SharedPatternCards: () =>
shared patterns
, })) @@ -166,6 +169,29 @@ describe("GrowthPage", () => { ).toBeInTheDocument() }) + it("shows exemplar library content on the patterns tab", () => { + mockUsePatternSharing.mockReturnValue({ + data: { + patterns: [], + bright_spots: [], + exemplar_sessions: [], + total_clusters_found: 0, + sessions_analyzed: 0, + }, + isLoading: false, + } as unknown as ReturnType) + mockUseEngineerProfile.mockReturnValue({ + data: null, + isLoading: false, + } as unknown as ReturnType) + + renderPage() + fireEvent.click(screen.getByRole("button", { name: "Patterns" })) + + expect(screen.getByText("Exemplar Session Library")).toBeInTheDocument() + expect(screen.getByText("exemplar session library")).toBeInTheDocument() + }) + it("shows an engineer chooser for API-key admins without a selected context", () => { mockGetApiKey.mockReturnValue("test-key") mockUseAuth.mockReturnValue({ user: null } as ReturnType) diff --git a/frontend/src/pages/growth.tsx b/frontend/src/pages/growth.tsx index 0a957c4..f083a43 100644 --- a/frontend/src/pages/growth.tsx +++ b/frontend/src/pages/growth.tsx @@ -19,6 +19,7 @@ import { VelocityChart } from "@/components/growth/velocity-chart" import { OnboardingRecommendations } from "@/components/growth/onboarding-recommendations" import { PatternSummary } from "@/components/growth/pattern-summary" import { BrightSpotCards } from "@/components/growth/bright-spot-cards" +import { ExemplarSessionLibrary } from "@/components/growth/exemplar-session-library" import { SharedPatternCards } from "@/components/growth/shared-pattern-card" import { SkillInventorySummary } from "@/components/insights/skill-inventory-summary" import { CoverageSummary } from "@/components/growth/coverage-summary" @@ -69,6 +70,15 @@ function PatternsTab({ teamId, startDate, endDate }: TabProps) {
+
+
+

Exemplar Session Library

+

+ Browse concrete sessions worth copying, with workflow, support, and cost context. +

+
+ +
) diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 1182be1..5df5042 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -828,9 +828,44 @@ export interface BrightSpot { exemplar_tools: string[] } +export interface ExemplarPatternReference { + cluster_id: string + cluster_type: string + cluster_label: string + session_count: number + engineer_count: number + success_rate: number | null +} + +export interface ExemplarSession { + exemplar_id: string + title: string + summary: string + why_selected: string + session_id: string + engineer_id: string + engineer_name: string + project_name: string | null + outcome: string | null + helpfulness: string | null + session_summary: string | null + duration_seconds: number | null + estimated_cost: number | null + tools_used: string[] + workflow_archetype: string | null + workflow_fingerprint: string | null + workflow_steps: string[] + supporting_session_count: number + supporting_engineer_count: number + supporting_pattern_count: number + success_rate: number | null + linked_patterns: ExemplarPatternReference[] +} + export interface PatternSharingResponse { patterns: SharedPattern[] bright_spots: BrightSpot[] + exemplar_sessions: ExemplarSession[] total_clusters_found: number sessions_analyzed: number } diff --git a/src/primer/common/schemas.py b/src/primer/common/schemas.py index 782c229..5fc816e 100644 --- a/src/primer/common/schemas.py +++ b/src/primer/common/schemas.py @@ -1166,9 +1166,44 @@ class BrightSpot(BaseModel): exemplar_tools: list[str] +class ExemplarPatternReference(BaseModel): + cluster_id: str + cluster_type: str + cluster_label: str + session_count: int + engineer_count: int + success_rate: float | None + + +class ExemplarSession(BaseModel): + exemplar_id: str + title: str + summary: str + why_selected: str + session_id: str + engineer_id: str + engineer_name: str + project_name: str | None = None + outcome: str | None = None + helpfulness: str | None = None + session_summary: str | None = None + duration_seconds: float | None + estimated_cost: float | None = None + tools_used: list[str] + workflow_archetype: str | None = None + workflow_fingerprint: str | None = None + workflow_steps: list[str] = [] + supporting_session_count: int + supporting_engineer_count: int + supporting_pattern_count: int + success_rate: float | None + linked_patterns: list[ExemplarPatternReference] + + class PatternSharingResponse(BaseModel): patterns: list[SharedPattern] bright_spots: list[BrightSpot] = [] + exemplar_sessions: list[ExemplarSession] = [] total_clusters_found: int sessions_analyzed: int diff --git a/src/primer/server/services/insights_service.py b/src/primer/server/services/insights_service.py index 05afc69..bfbb365 100644 --- a/src/primer/server/services/insights_service.py +++ b/src/primer/server/services/insights_service.py @@ -8,8 +8,15 @@ from sqlalchemy.orm import Session from primer.common.facet_taxonomy import canonical_outcome, is_success_outcome -from primer.common.models import Engineer, SessionFacets, ToolUsage +from primer.common.models import ( + Engineer, + ModelUsage, + SessionFacets, + SessionWorkflowProfile, + ToolUsage, +) from primer.common.models import Session as SessionModel +from primer.common.pricing import estimate_cost from primer.common.schemas import ( BrightSpot, CohortMetrics, @@ -19,6 +26,8 @@ EngineerLearningPath, EngineerRampup, EngineerSkillProfile, + ExemplarPatternReference, + ExemplarSession, LearningPathsResponse, LearningRecommendation, NewHireProgress, @@ -35,9 +44,7 @@ TimeToTeamAverageResponse, WeeklySuccessPoint, ) -from primer.server.services.analytics_service import ( - base_session_query, -) +from primer.server.services.analytics_service import base_session_query def _shannon_entropy(counts: dict[str, int]) -> float: @@ -883,6 +890,7 @@ def get_pattern_sharing( SessionFacets.session_type, SessionFacets.outcome, SessionFacets.agent_helpfulness, + SessionFacets.brief_summary, SessionFacets.goal_categories, ) .filter(SessionFacets.session_id.in_(session_ids_q)) @@ -904,6 +912,47 @@ def get_pattern_sharing( session_tools[tu.session_id].append(tu.tool_name) session_tool_count[tu.session_id] += tu.call_count + session_workflows = { + row.session_id: { + "archetype": row.archetype, + "label": row.label, + "steps": list(row.steps or []), + } + for row in ( + db.query( + SessionWorkflowProfile.session_id, + SessionWorkflowProfile.archetype, + SessionWorkflowProfile.label, + SessionWorkflowProfile.steps, + ) + .filter(SessionWorkflowProfile.session_id.in_(session_ids_q)) + .all() + ) + } + + session_costs: dict[str, float] = defaultdict(float) + model_rows = ( + db.query( + ModelUsage.session_id, + ModelUsage.model_name, + func.sum(ModelUsage.input_tokens).label("input_tokens"), + func.sum(ModelUsage.output_tokens).label("output_tokens"), + func.sum(ModelUsage.cache_read_tokens).label("cache_read_tokens"), + func.sum(ModelUsage.cache_creation_tokens).label("cache_creation_tokens"), + ) + .filter(ModelUsage.session_id.in_(session_ids_q)) + .group_by(ModelUsage.session_id, ModelUsage.model_name) + .all() + ) + for row in model_rows: + session_costs[row.session_id] += estimate_cost( + row.model_name, + row.input_tokens or 0, + row.output_tokens or 0, + row.cache_read_tokens or 0, + row.cache_creation_tokens or 0, + ) + def _make_approach(sid: str) -> EngineerApproach: s = session_map[sid] eid = session_to_engineer[sid] @@ -1027,6 +1076,14 @@ def _build_pattern( return PatternSharingResponse( patterns=patterns, bright_spots=_derive_bright_spots(patterns), + exemplar_sessions=_derive_exemplar_sessions( + patterns, + session_map=session_map, + session_facets=session_facets, + session_tools=session_tools, + session_workflows=session_workflows, + session_costs=session_costs, + ), total_clusters_found=len(patterns), sessions_analyzed=len(sessions), ) @@ -1081,6 +1138,143 @@ def _derive_bright_spots(patterns: list[SharedPattern], limit: int = 3) -> list[ return bright_spots +def _derive_exemplar_sessions( + patterns: list[SharedPattern], + *, + session_map: dict[str, SessionModel], + session_facets: dict[str, object], + session_tools: dict[str, list[str]], + session_workflows: dict[str, dict[str, object]], + session_costs: dict[str, float], + limit: int = 6, +) -> list[ExemplarSession]: + cluster_type_order = { + "session_type": 0, + "goal_category": 1, + "project": 2, + } + buckets: dict[str, dict[str, object]] = {} + + for pattern in patterns: + exemplar = pattern.best_approach + if exemplar is None: + continue + + bucket = buckets.setdefault( + exemplar.session_id, + { + "approach": exemplar, + "linked_patterns": [], + "supporting_session_count": 0, + "supporting_engineer_count": 0, + "success_rate": None, + }, + ) + linked_patterns = bucket["linked_patterns"] + assert isinstance(linked_patterns, list) + linked_patterns.append( + ExemplarPatternReference( + cluster_id=pattern.cluster_id, + cluster_type=pattern.cluster_type, + cluster_label=pattern.cluster_label, + session_count=pattern.session_count, + engineer_count=pattern.engineer_count, + success_rate=pattern.success_rate, + ) + ) + bucket["supporting_session_count"] = max( + int(bucket["supporting_session_count"]), + pattern.session_count, + ) + bucket["supporting_engineer_count"] = max( + int(bucket["supporting_engineer_count"]), + pattern.engineer_count, + ) + current_success_rate = bucket["success_rate"] + if current_success_rate is None or ( + pattern.success_rate is not None and pattern.success_rate > current_success_rate + ): + bucket["success_rate"] = pattern.success_rate + + exemplars: list[ExemplarSession] = [] + for session_id, bucket in buckets.items(): + approach = bucket["approach"] + assert isinstance(approach, EngineerApproach) + session = session_map[session_id] + facet = session_facets.get(session_id) + workflow = session_workflows.get(session_id, {}) + linked_patterns = bucket["linked_patterns"] + assert isinstance(linked_patterns, list) + linked_patterns.sort( + key=lambda item: ( + cluster_type_order.get(item.cluster_type, 99), + -item.engineer_count, + -item.session_count, + -(item.success_rate or 0.0), + item.cluster_label, + ) + ) + primary_pattern = linked_patterns[0] + related_pattern_count = max(len(linked_patterns) - 1, 0) + linked_pattern_phrase = ( + " It also anchors " + f"{related_pattern_count} related pattern" + f"{'' if related_pattern_count == 1 else 's'}." + if related_pattern_count > 0 + else "" + ) + why_selected = ( + f"Chosen as the fastest successful example for {primary_pattern.cluster_label}." + f"{linked_pattern_phrase}" + ) + + title = str(workflow.get("label") or primary_pattern.cluster_label) + summary = ( + f"Backed by {bucket['supporting_engineer_count']} engineers across " + f"{bucket['supporting_session_count']} sessions." + ) + session_summary = getattr(facet, "brief_summary", None) or session.summary + exemplars.append( + ExemplarSession( + exemplar_id=f"exemplar:{session_id}", + title=title, + summary=summary, + why_selected=why_selected, + session_id=session_id, + engineer_id=approach.engineer_id, + engineer_name=approach.name, + project_name=session.project_name, + outcome=approach.outcome, + helpfulness=getattr(facet, "agent_helpfulness", None), + session_summary=session_summary, + duration_seconds=session.duration_seconds, + estimated_cost=( + round(session_costs[session_id], 4) if session_id in session_costs else None + ), + tools_used=sorted(set(session_tools.get(session_id, []))), + workflow_archetype=workflow.get("archetype"), + workflow_fingerprint=workflow.get("label"), + workflow_steps=list(workflow.get("steps") or []), + supporting_session_count=int(bucket["supporting_session_count"]), + supporting_engineer_count=int(bucket["supporting_engineer_count"]), + supporting_pattern_count=len(linked_patterns), + success_rate=bucket["success_rate"], + linked_patterns=linked_patterns[:3], + ) + ) + + exemplars.sort( + key=lambda exemplar: ( + -(exemplar.success_rate or 0.0), + -exemplar.supporting_engineer_count, + -exemplar.supporting_session_count, + exemplar.estimated_cost if exemplar.estimated_cost is not None else float("inf"), + exemplar.title, + ) + ) + return exemplars[:limit] + + # --------------------------------------------------------------------------- # Onboarding Acceleration # --------------------------------------------------------------------------- diff --git a/tests/test_growth.py b/tests/test_growth.py index 54147f5..32de455 100644 --- a/tests/test_growth.py +++ b/tests/test_growth.py @@ -4,7 +4,15 @@ import bcrypt -from primer.common.models import Engineer, Session, SessionFacets, Team, ToolUsage +from primer.common.models import ( + Engineer, + ModelUsage, + Session, + SessionFacets, + SessionWorkflowProfile, + Team, + ToolUsage, +) def _create_session(db_session, engineer, **kwargs): @@ -320,6 +328,83 @@ def test_bright_spots_surface_exemplar_sessions_for_strong_patterns( assert bright_spot["exemplar_session_id"] == session_ids[1] assert bright_spot["exemplar_tools"] == ["Edit", "Read"] + def test_pattern_sharing_includes_workflow_aware_exemplar_library_entries( + self, client, db_session, admin_headers + ): + team, engineers = _create_team_engineers(db_session, 3) + now = datetime.now(UTC) + + durations = [180.0, 60.0, 120.0] + session_ids = [] + for engineer, duration in zip(engineers, durations, strict=True): + session = _create_session( + db_session, + engineer, + started_at=now - timedelta(hours=1), + project_name="primer", + duration_seconds=duration, + summary="Debugged the auth flow and verified the fix.", + ) + session_ids.append(session.id) + db_session.add( + SessionFacets( + session_id=session.id, + session_type="debugging", + outcome="success", + brief_summary="Reproduced and fixed the auth failure.", + goal_categories=["auth"], + agent_helpfulness="very_helpful", + ) + ) + db_session.add(ToolUsage(session_id=session.id, tool_name="Read", call_count=5)) + db_session.add(ToolUsage(session_id=session.id, tool_name="Edit", call_count=2)) + + db_session.add( + SessionWorkflowProfile( + session_id=session_ids[1], + fingerprint_id="debugging::read+edit+execute+fix", + label="debugging: read -> edit -> execute -> fix", + steps=["read", "edit", "execute", "fix"], + archetype="debugging", + archetype_source="session_type", + top_tools=["Read", "Edit"], + ) + ) + db_session.add( + ModelUsage( + session_id=session_ids[1], + model_name="claude-sonnet-4", + input_tokens=1000, + output_tokens=500, + cache_read_tokens=0, + cache_creation_tokens=0, + ) + ) + db_session.flush() + + response = client.get( + f"/api/v1/analytics/pattern-sharing?team_id={team.id}", + headers=admin_headers, + ) + assert response.status_code == 200 + data = response.json() + + exemplar = next( + item for item in data["exemplar_sessions"] if item["session_id"] == session_ids[1] + ) + assert exemplar["title"] == "debugging: read -> edit -> execute -> fix" + assert exemplar["workflow_archetype"] == "debugging" + assert exemplar["workflow_fingerprint"] == "debugging: read -> edit -> execute -> fix" + assert exemplar["workflow_steps"] == ["read", "edit", "execute", "fix"] + assert exemplar["supporting_engineer_count"] == 3 + assert exemplar["supporting_session_count"] == 3 + assert exemplar["supporting_pattern_count"] == 2 + assert exemplar["linked_patterns"][0]["cluster_label"] == "debugging on primer" + assert exemplar["linked_patterns"][1]["cluster_label"] == "auth" + assert exemplar["estimated_cost"] == 0.0105 + assert exemplar["tools_used"] == ["Edit", "Read"] + assert exemplar["session_summary"] == "Reproduced and fixed the auth failure." + def test_best_approach_treats_legacy_success_as_success( self, client, db_session, admin_headers ):