diff --git a/ROADMAP.md b/ROADMAP.md index e0d5caa..b8b79f1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -145,7 +145,7 @@ As usage grows, the platform needs stronger derived data pipelines, performance - [x] Onboarding recommendations - [x] Shared behavior pattern discovery with approach comparison - [x] [P1] Bright spot detection: explicitly surface high performers and cross-pollinate their patterns -- [ ] [P1] Exemplar-session-to-learning-path pipeline +- [x] [P1] Exemplar-session-to-learning-path pipeline - [ ] [P1] Team skill gap mapping by workflow, tool category, and project context - [ ] [P2] Coaching program measurement: which onboarding or training changes improved outcomes diff --git a/frontend/src/components/growth/__tests__/learning-path-cards.test.tsx b/frontend/src/components/growth/__tests__/learning-path-cards.test.tsx new file mode 100644 index 0000000..6e1dc29 --- /dev/null +++ b/frontend/src/components/growth/__tests__/learning-path-cards.test.tsx @@ -0,0 +1,65 @@ +import { render, screen } from "@testing-library/react" +import { MemoryRouter } from "react-router-dom" + +import { LearningPathCards } from "@/components/growth/learning-path-cards" + +describe("LearningPathCards", () => { + it("renders an empty state", () => { + render( + + + , + ) + + expect(screen.getByText("No learning paths available.")).toBeInTheDocument() + }) + + it("renders exemplar study links for recommendations", () => { + render( + + edit -> execute -> fix", + engineer_name: "Bob Example", + project_name: "primer", + summary: "Resolved an auth regression.", + relevance_reason: "Shows Edit in a successful peer workflow.", + workflow_archetype: "debugging", + workflow_fingerprint: "debugging: read -> edit -> execute -> fix", + duration_seconds: 75, + estimated_cost: 0.5, + tools_used: ["Read", "Edit", "Bash"], + }, + ], + }, + ], + }, + ]} + /> + , + ) + + expect(screen.getByText("Study Exemplars")).toBeInTheDocument() + expect(screen.getByText("Bob Example • primer")).toBeInTheDocument() + expect(screen.getByText("Shows Edit in a successful peer workflow.")).toBeInTheDocument() + expect(screen.getByText("Debugging")).toBeInTheDocument() + expect(screen.getByRole("link")).toHaveAttribute("href", "/sessions/sess-1") + }) +}) diff --git a/frontend/src/components/growth/learning-path-cards.tsx b/frontend/src/components/growth/learning-path-cards.tsx index 6ff57c3..1c21e31 100644 --- a/frontend/src/components/growth/learning-path-cards.tsx +++ b/frontend/src/components/growth/learning-path-cards.tsx @@ -1,6 +1,8 @@ +import { Link } from "react-router-dom" + import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" import { Badge } from "@/components/ui/badge" -import { cn } from "@/lib/utils" +import { cn, formatCost, formatDuration, formatLabel } from "@/lib/utils" import type { EngineerLearningPath } from "@/types/api" const categoryColors: Record = { @@ -82,6 +84,49 @@ export function LearningPathCards({ paths }: LearningPathCardsProps) {

{rec.description}

+ {rec.exemplars.length > 0 && ( +
+

+ Study Exemplars +

+
+ {rec.exemplars.map((exemplar) => ( + +
+
+ {exemplar.title} + {exemplar.workflow_archetype && ( + + {formatLabel(exemplar.workflow_archetype)} + + )} +
+

+ {exemplar.engineer_name} + {exemplar.project_name ? ` • ${exemplar.project_name}` : ""} +

+

+ {exemplar.relevance_reason} +

+
+ {formatDuration(exemplar.duration_seconds)} + {exemplar.estimated_cost != null && ( + {formatCost(exemplar.estimated_cost)} + )} + {exemplar.tools_used.slice(0, 3).map((tool) => ( + {tool} + ))} +
+
+ + ))} +
+
+ )} ))} diff --git a/frontend/src/pages/__tests__/growth.test.tsx b/frontend/src/pages/__tests__/growth.test.tsx index 474b0c4..e7af9ac 100644 --- a/frontend/src/pages/__tests__/growth.test.tsx +++ b/frontend/src/pages/__tests__/growth.test.tsx @@ -55,6 +55,9 @@ vi.mock("@/components/insights/team-skill-gaps", () => ({ vi.mock("@/components/growth/skill-universe-chart", () => ({ SkillUniverseChart: () =>
skill universe
, })) +vi.mock("@/components/growth/learning-path-cards", () => ({ + LearningPathCards: () =>
learning paths
, +})) vi.mock("@/components/insights/engineer-skill-table", () => ({ EngineerSkillTable: () =>
engineer skill table
, })) @@ -192,6 +195,46 @@ describe("GrowthPage", () => { expect(screen.getByText("exemplar session library")).toBeInTheDocument() }) + it("shows learning paths on the skills tab", () => { + mockUseSkillInventory.mockReturnValue({ + data: { + engineer_profiles: [], + team_skill_gaps: [], + total_engineers: 1, + total_session_types: 1, + total_tools_used: 1, + }, + isLoading: false, + } as unknown as ReturnType) + mockUseLearningPaths.mockReturnValue({ + data: { + engineer_paths: [ + { + engineer_id: "eng-1", + name: "Alice Example", + total_sessions: 4, + recommendations: [], + coverage_score: 0.5, + complexity_trend: "flat", + }, + ], + team_skill_universe: {}, + sessions_analyzed: 4, + }, + isLoading: false, + } as unknown as ReturnType) + mockUseEngineerProfile.mockReturnValue({ + data: null, + isLoading: false, + } as unknown as ReturnType) + + renderPage() + fireEvent.click(screen.getByRole("button", { name: "Skills" })) + + expect(screen.getByText("Learning Paths")).toBeInTheDocument() + expect(screen.getByText("learning paths")).toBeInTheDocument() + }) + it("shows an engineer chooser for API-key admins without a selected context", () => { mockGetApiKey.mockReturnValue("test-key") mockUseAuth.mockReturnValue({ user: null } as ReturnType) diff --git a/frontend/src/pages/growth.tsx b/frontend/src/pages/growth.tsx index f083a43..119f5bb 100644 --- a/frontend/src/pages/growth.tsx +++ b/frontend/src/pages/growth.tsx @@ -25,6 +25,7 @@ import { SkillInventorySummary } from "@/components/insights/skill-inventory-sum import { CoverageSummary } from "@/components/growth/coverage-summary" import { TeamSkillGaps } from "@/components/insights/team-skill-gaps" import { SkillUniverseChart } from "@/components/growth/skill-universe-chart" +import { LearningPathCards } from "@/components/growth/learning-path-cards" import { WorkflowPlaybookCards } from "@/components/growth/workflow-playbook-cards" import { EngineerSkillTable } from "@/components/insights/engineer-skill-table" import { Card, CardContent } from "@/components/ui/card" @@ -102,6 +103,17 @@ function SkillsTab({ teamId, startDate, endDate }: TabProps) { )} + {learning && learning.engineer_paths.length > 0 && ( +
+
+

Learning Paths

+

+ Personalized recommendations with exemplar peer sessions to study. +

+
+ +
+ )} {skills && } ) diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 5df5042..2f8046e 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -761,6 +761,20 @@ export interface SkillInventoryResponse { // --- Learning Paths --- +export interface LearningRecommendationExemplar { + session_id: string + title: string + engineer_name: string + project_name: string | null + summary: string | null + relevance_reason: string + workflow_archetype: string | null + workflow_fingerprint: string | null + duration_seconds: number | null + estimated_cost: number | null + tools_used: string[] +} + export interface LearningRecommendation { category: string skill_area: string @@ -768,6 +782,7 @@ export interface LearningRecommendation { description: string priority: string evidence: Record + exemplars: LearningRecommendationExemplar[] } export interface EngineerLearningPath { diff --git a/src/primer/common/schemas.py b/src/primer/common/schemas.py index 5fc816e..baddc43 100644 --- a/src/primer/common/schemas.py +++ b/src/primer/common/schemas.py @@ -1098,6 +1098,20 @@ class SkillInventoryResponse(BaseModel): # --- Learning Paths --- +class LearningRecommendationExemplar(BaseModel): + session_id: str + title: str + engineer_name: str + project_name: str | None = None + summary: str | None = None + relevance_reason: str + workflow_archetype: str | None = None + workflow_fingerprint: str | None = None + duration_seconds: float | None + estimated_cost: float | None = None + tools_used: list[str] + + class LearningRecommendation(BaseModel): category: str # "session_type_gap", "tool_gap", "complexity", "goal_gap" skill_area: str @@ -1105,6 +1119,7 @@ class LearningRecommendation(BaseModel): description: str priority: str # "high", "medium", "low" evidence: dict + exemplars: list[LearningRecommendationExemplar] = [] class EngineerLearningPath(BaseModel): diff --git a/src/primer/server/services/insights_service.py b/src/primer/server/services/insights_service.py index bfbb365..be340cf 100644 --- a/src/primer/server/services/insights_service.py +++ b/src/primer/server/services/insights_service.py @@ -30,6 +30,7 @@ ExemplarSession, LearningPathsResponse, LearningRecommendation, + LearningRecommendationExemplar, NewHireProgress, OnboardingAccelerationResponse, OnboardingRecommendation, @@ -646,11 +647,14 @@ def get_learning_paths( engineers = db.query(Engineer.id, Engineer.name).filter(Engineer.id.in_(engineer_ids)).all() eng_names = {eid: name for eid, name in engineers} - # Session types per session + # Facets (expanded columns for reuse in pattern/exemplar building) all_facets = ( db.query( SessionFacets.session_id, SessionFacets.session_type, + SessionFacets.outcome, + SessionFacets.agent_helpfulness, + SessionFacets.brief_summary, SessionFacets.goal_categories, ) .filter(SessionFacets.session_id.in_(session_ids_q)) @@ -658,7 +662,9 @@ def get_learning_paths( ) session_to_type: dict[str, str] = {} session_to_goals: dict[str, list[str]] = {} + session_facets: dict[str, object] = {} for f in all_facets: + session_facets[f.session_id] = f if f.session_type: session_to_type[f.session_id] = f.session_type if f.goal_categories: @@ -671,11 +677,13 @@ def get_learning_paths( if isinstance(cats, list): session_to_goals[f.session_id] = cats - # Build session_id → engineer_id + # Build session_id → engineer_id and session_map session_to_engineer: dict[str, str] = {} + session_map: dict[str, object] = {} for eid, sess_list in eng_sessions.items(): for s in sess_list: session_to_engineer[s.id] = eid + session_map[s.id] = s # Tool usages all_tools = ( @@ -684,7 +692,11 @@ def get_learning_paths( .all() ) eng_tool_names: dict[str, set[str]] = defaultdict(set) + session_tools: dict[str, list[str]] = defaultdict(list) + session_tool_count: dict[str, int] = defaultdict(int) for tu in all_tools: + session_tools[tu.session_id].append(tu.tool_name) + session_tool_count[tu.session_id] += tu.call_count eid = session_to_engineer.get(tu.session_id) if eid: eng_tool_names[eid].add(tu.tool_name) @@ -731,6 +743,58 @@ def get_learning_paths( team_skill_count = len(team_skill_universe) if team_skill_universe else 1 + # Workflow profiles and costs for exemplar derivation + session_workflows: dict[str, dict[str, object]] = { + row.session_id: { + "archetype": row.archetype, + "label": row.label, + "steps": list(row.steps or []), + } + for row in ( + db.query( + SessionWorkflowProfile.session_id, + SessionWorkflowProfile.archetype, + SessionWorkflowProfile.label, + SessionWorkflowProfile.steps, + ) + .filter(SessionWorkflowProfile.session_id.in_(session_ids_q)) + .all() + ) + } + session_costs: dict[str, float] = defaultdict(float) + for row in ( + db.query( + ModelUsage.session_id, + ModelUsage.model_name, + func.sum(ModelUsage.input_tokens).label("input_tokens"), + func.sum(ModelUsage.output_tokens).label("output_tokens"), + func.sum(ModelUsage.cache_read_tokens).label("cache_read_tokens"), + func.sum(ModelUsage.cache_creation_tokens).label("cache_creation_tokens"), + ) + .filter(ModelUsage.session_id.in_(session_ids_q)) + .group_by(ModelUsage.session_id, ModelUsage.model_name) + .all() + ): + session_costs[row.session_id] += estimate_cost( + row.model_name, + row.input_tokens or 0, + row.output_tokens or 0, + row.cache_read_tokens or 0, + row.cache_creation_tokens or 0, + ) + + _, _, exemplar_sessions = _build_patterns_and_exemplars( + sessions=sessions, + session_map=session_map, + session_to_engineer=session_to_engineer, + eng_names=eng_names, + session_facets=session_facets, + session_tools=session_tools, + session_tool_count=session_tool_count, + session_workflows=session_workflows, + session_costs=session_costs, + ) + paths: list[EngineerLearningPath] = [] for eid in engineer_ids: recs: list[LearningRecommendation] = [] @@ -754,6 +818,11 @@ def get_learning_paths( ), priority="high", evidence={"team_adoption": round(adoption, 2), "team_count": cnt}, + exemplars=_select_learning_recommendation_exemplars( + "session_type_gap", + stype, + exemplar_sessions, + ), ) ) @@ -773,6 +842,11 @@ def get_learning_paths( ), priority=priority, evidence={"team_adoption": round(adoption, 2), "team_count": cnt}, + exemplars=_select_learning_recommendation_exemplars( + "tool_gap", + tool, + exemplar_sessions, + ), ) ) @@ -791,6 +865,11 @@ def get_learning_paths( ), priority="medium", evidence={"team_adoption": round(adoption, 2), "team_count": cnt}, + exemplars=_select_learning_recommendation_exemplars( + "goal_gap", + cat, + exemplar_sessions, + ), ) ) @@ -828,6 +907,11 @@ def get_learning_paths( "first_avg": round(first_avg, 1), "recent_avg": round(recent_avg, 1), }, + exemplars=_select_learning_recommendation_exemplars( + "complexity", + "task complexity", + exemplar_sessions, + ), ) ) @@ -953,6 +1037,41 @@ def get_pattern_sharing( row.cache_creation_tokens or 0, ) + patterns, bright_spots, exemplar_sessions = _build_patterns_and_exemplars( + sessions=sessions, + session_map=session_map, + session_to_engineer=session_to_engineer, + eng_names=eng_names, + session_facets=session_facets, + session_tools=session_tools, + session_tool_count=session_tool_count, + session_workflows=session_workflows, + session_costs=session_costs, + ) + + return PatternSharingResponse( + patterns=patterns, + bright_spots=bright_spots, + exemplar_sessions=exemplar_sessions, + total_clusters_found=len(patterns), + sessions_analyzed=len(sessions), + ) + + +def _build_patterns_and_exemplars( + *, + sessions: list, + session_map: dict[str, object], + session_to_engineer: dict[str, str], + eng_names: dict[str, str], + session_facets: dict[str, object], + session_tools: dict[str, list[str]], + session_tool_count: dict[str, int], + session_workflows: dict[str, dict[str, object]], + session_costs: dict[str, float], +) -> tuple[list[SharedPattern], list[BrightSpot], list[ExemplarSession]]: + """Build pattern clusters, bright spots, and exemplar sessions from pre-loaded data.""" + def _make_approach(sid: str) -> EngineerApproach: s = session_map[sid] eid = session_to_engineer[sid] @@ -977,7 +1096,6 @@ def _build_pattern( approaches = [_make_approach(sid) for sid in sids] eng_set = {a.engineer_id for a in approaches} - # Best approach: successful + shortest duration successful = [ approach for approach in approaches @@ -992,7 +1110,6 @@ def _build_pattern( successes = sum(1 for outcome in outcomes if is_success_outcome(outcome)) sr = round(successes / len(outcomes), 3) if outcomes else None - # Insight insight_parts = [f"{len(eng_set)} engineers worked on {cluster_label}"] if best and avg_dur and avg_dur > 0: pct = round((1 - best.duration_seconds / avg_dur) * 100) @@ -1034,8 +1151,8 @@ def _build_pattern( # 2. Goal category clusters goal_groups: dict[str, list[str]] = defaultdict(list) - for f in all_facets: - cats = f.goal_categories + for f in session_facets.values(): + cats = getattr(f, "goal_categories", None) if cats: if isinstance(cats, str): try: @@ -1047,7 +1164,6 @@ def _build_pattern( goal_groups[cat].append(f.session_id) for cat, sids in goal_groups.items(): - # Filter to valid session ids valid_sids = [sid for sid in sids if sid in session_map] eng_set = {session_to_engineer[sid] for sid in valid_sids} if len(eng_set) >= 2 and len(valid_sids) >= 3: @@ -1059,7 +1175,6 @@ def _build_pattern( if s.project_name: project_groups[s.project_name].append(s.id) - # Track projects already covered by type+project clusters covered_projects: set[str] = set() for (_, proj), sids_tp in type_project_groups.items(): if len({session_to_engineer[sid] for sid in sids_tp}) >= 2: @@ -1070,13 +1185,12 @@ def _build_pattern( if len(eng_set) >= 2 and proj not in covered_projects: patterns.append(_build_pattern(f"project:{proj}", "project", proj, sids)) - # Sort by engineer_count desc patterns.sort(key=lambda p: p.engineer_count, reverse=True) - return PatternSharingResponse( - patterns=patterns, - bright_spots=_derive_bright_spots(patterns), - exemplar_sessions=_derive_exemplar_sessions( + return ( + patterns, + _derive_bright_spots(patterns), + _derive_exemplar_sessions( patterns, session_map=session_map, session_facets=session_facets, @@ -1084,8 +1198,6 @@ def _build_pattern( session_workflows=session_workflows, session_costs=session_costs, ), - total_clusters_found=len(patterns), - sessions_analyzed=len(sessions), ) @@ -1138,6 +1250,80 @@ def _derive_bright_spots(patterns: list[SharedPattern], limit: int = 3) -> list[ return bright_spots +def _select_learning_recommendation_exemplars( + category: str, + skill_area: str, + exemplars: list[ExemplarSession], + limit: int = 2, +) -> list[LearningRecommendationExemplar]: + matches: list[tuple[ExemplarSession, str]] = [] + + for exemplar in exemplars: + reason = _learning_exemplar_reason(category, skill_area, exemplar) + if reason is not None: + matches.append((exemplar, reason)) + + matches.sort( + key=lambda item: ( + -(item[0].success_rate or 0.0), + -item[0].supporting_engineer_count, + -item[0].supporting_session_count, + item[0].estimated_cost if item[0].estimated_cost is not None else float("inf"), + item[0].title, + ) + ) + + return [ + LearningRecommendationExemplar( + session_id=exemplar.session_id, + title=exemplar.title, + engineer_name=exemplar.engineer_name, + project_name=exemplar.project_name, + summary=exemplar.session_summary, + relevance_reason=reason, + workflow_archetype=exemplar.workflow_archetype, + workflow_fingerprint=exemplar.workflow_fingerprint, + duration_seconds=exemplar.duration_seconds, + estimated_cost=exemplar.estimated_cost, + tools_used=exemplar.tools_used, + ) + for exemplar, reason in matches[:limit] + ] + + +def _learning_exemplar_reason( + category: str, skill_area: str, exemplar: ExemplarSession +) -> str | None: + if category == "session_type_gap": + if any( + pattern.cluster_type == "session_type" + and pattern.cluster_label.startswith(f"{skill_area} on ") + for pattern in exemplar.linked_patterns + ): + return f"Strong peer example of a '{skill_area}' workflow." + return None + + if category == "goal_gap": + if any( + pattern.cluster_type == "goal_category" and pattern.cluster_label == skill_area + for pattern in exemplar.linked_patterns + ): + return f"Relevant exemplar for '{skill_area}' work." + return None + + if category == "tool_gap": + if skill_area in exemplar.tools_used: + return f"Shows {skill_area} in a successful peer workflow." + return None + + if category == "complexity": + if len(exemplar.workflow_steps) >= 3 or len(exemplar.tools_used) >= 3: + return "A stronger multi-step workflow to study when ramping complexity back up." + return None + + return None + + def _derive_exemplar_sessions( patterns: list[SharedPattern], *, diff --git a/tests/test_growth.py b/tests/test_growth.py index 32de455..3b874b5 100644 --- a/tests/test_growth.py +++ b/tests/test_growth.py @@ -130,6 +130,128 @@ def test_tool_gap_detected(self, client, db_session, admin_headers): assert len(tool_recs) >= 1 assert any("Grep" in r["skill_area"] for r in tool_recs) + def test_learning_recommendations_include_relevant_exemplar_sessions( + self, client, db_session, admin_headers + ): + team, engineers = _create_team_engineers(db_session, 3) + now = datetime.now(UTC) + + peer_session_ids = [] + for engineer, duration in zip(engineers[:2], [180.0, 75.0], strict=True): + session = _create_session( + db_session, + engineer, + started_at=now - timedelta(hours=1), + project_name="primer", + duration_seconds=duration, + summary="Debugged the auth flow with a lightweight fix loop.", + ) + peer_session_ids.append(session.id) + db_session.add( + SessionFacets( + session_id=session.id, + session_type="debugging", + outcome="success", + brief_summary="Reproduced and fixed an auth regression.", + goal_categories=["auth"], + ) + ) + db_session.add(ToolUsage(session_id=session.id, tool_name="Read", call_count=5)) + db_session.add(ToolUsage(session_id=session.id, tool_name="Edit", call_count=2)) + + db_session.add( + SessionWorkflowProfile( + session_id=peer_session_ids[1], + fingerprint_id="debugging::read+edit+execute+fix", + label="debugging: read -> edit -> execute -> fix", + steps=["read", "edit", "execute", "fix"], + archetype="debugging", + archetype_source="session_type", + top_tools=["Read", "Edit"], + ) + ) + db_session.add( + ModelUsage( + session_id=peer_session_ids[1], + model_name="claude-sonnet-4", + input_tokens=1000, + output_tokens=500, + cache_read_tokens=0, + cache_creation_tokens=0, + ) + ) + extra_auth_session = _create_session( + db_session, + engineers[0], + started_at=now - timedelta(minutes=50), + project_name="primer", + duration_seconds=210.0, + ) + db_session.add( + SessionFacets( + session_id=extra_auth_session.id, + session_type="debugging", + outcome="success", + goal_categories=["auth"], + ) + ) + db_session.add(ToolUsage(session_id=extra_auth_session.id, tool_name="Read", call_count=4)) + db_session.add(ToolUsage(session_id=extra_auth_session.id, tool_name="Edit", call_count=1)) + + learner_session = _create_session( + db_session, + engineers[2], + started_at=now - timedelta(minutes=30), + project_name="primer", + duration_seconds=120.0, + ) + db_session.add( + SessionFacets( + session_id=learner_session.id, + session_type="feature", + outcome="success", + goal_categories=["ui_component"], + ) + ) + db_session.add(ToolUsage(session_id=learner_session.id, tool_name="Read", call_count=3)) + db_session.flush() + + response = client.get( + f"/api/v1/analytics/learning-paths?team_id={team.id}", + headers=admin_headers, + ) + assert response.status_code == 200 + data = response.json() + + learner_path = next( + path for path in data["engineer_paths"] if path["engineer_id"] == engineers[2].id + ) + session_type_gap = next( + rec + for rec in learner_path["recommendations"] + if rec["category"] == "session_type_gap" and rec["skill_area"] == "debugging" + ) + assert session_type_gap["exemplars"][0]["session_id"] == peer_session_ids[1] + assert session_type_gap["exemplars"][0]["workflow_fingerprint"] == ( + "debugging: read -> edit -> execute -> fix" + ) + assert "debugging" in session_type_gap["exemplars"][0]["relevance_reason"] + + tool_gap = next( + rec + for rec in learner_path["recommendations"] + if rec["category"] == "tool_gap" and rec["skill_area"] == "Edit" + ) + assert tool_gap["exemplars"][0]["session_id"] == peer_session_ids[1] + assert tool_gap["exemplars"][0]["estimated_cost"] == 0.0105 + + goal_gap = next( + rec + for rec in learner_path["recommendations"] + if rec["category"] == "goal_gap" and rec["skill_area"] == "auth" + ) + assert goal_gap["exemplars"][0]["session_id"] == peer_session_ids[1] + def test_complexity_trend_increasing(self, client, db_session, admin_headers): """Rising tool_call_count + message_count over time → increasing trend.""" team, engineers = _create_team_engineers(db_session, 1)