From 0aae5b7058709c65a4fdd6ca5f2354c0bc14e95c Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 9 Apr 2026 16:15:30 -0400 Subject: [PATCH 1/8] progress --- src/sentry/conf/server.py | 2 +- src/sentry/tasks/seer/night_shift/__init__.py | 0 .../tasks/seer/night_shift/agentic_triage.py | 153 ++++++++++++++++++ .../{night_shift.py => night_shift/cron.py} | 92 +---------- src/sentry/tasks/seer/night_shift/models.py | 17 ++ .../tasks/seer/night_shift/simple_triage.py | 88 ++++++++++ tests/sentry/tasks/seer/test_night_shift.py | 30 ++-- 7 files changed, 280 insertions(+), 102 deletions(-) create mode 100644 src/sentry/tasks/seer/night_shift/__init__.py create mode 100644 src/sentry/tasks/seer/night_shift/agentic_triage.py rename src/sentry/tasks/seer/{night_shift.py => night_shift/cron.py} (61%) create mode 100644 src/sentry/tasks/seer/night_shift/models.py create mode 100644 src/sentry/tasks/seer/night_shift/simple_triage.py diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index a9a4944d73c758..b3949f05609eaf 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -991,7 +991,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: "sentry.tasks.seer.explorer_index", "sentry.tasks.seer.context_engine_index", "sentry.tasks.seer.lightweight_rca_cluster", - "sentry.tasks.seer.night_shift", + "sentry.tasks.seer.night_shift.cron", # Used for tests "sentry.taskworker.tasks.examples", ) diff --git a/src/sentry/tasks/seer/night_shift/__init__.py b/src/sentry/tasks/seer/night_shift/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/sentry/tasks/seer/night_shift/agentic_triage.py b/src/sentry/tasks/seer/night_shift/agentic_triage.py new file mode 100644 index 00000000000000..add2510166bad9 --- /dev/null +++ b/src/sentry/tasks/seer/night_shift/agentic_triage.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import logging +import textwrap +from collections.abc import Sequence + +import orjson +import pydantic + +from sentry.models.organization import Organization +from sentry.models.project import Project +from sentry.seer.signed_seer_api import LlmGenerateRequest, make_llm_generate_request +from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult +from sentry.tasks.seer.night_shift.simple_triage import ( + ScoredCandidate, + fixability_score_strategy, + priority_label, +) + +logger = logging.getLogger("sentry.tasks.seer.night_shift") + + +class _TriageVerdict(pydantic.BaseModel): + group_id: int + action: str + reason: str + + +class _TriageResponse(pydantic.BaseModel): + verdicts: list[_TriageVerdict] + + +def agentic_triage_strategy( + projects: Sequence[Project], + organization: Organization, +) -> list[TriageResult]: + """ + Select candidates via fixability scoring, then filter through an LLM + triage call that decides the action for each candidate. + """ + scored = fixability_score_strategy(projects) + if not scored: + return [] + + return _triage_candidates(scored, organization) + + +def _triage_candidates( + candidates: list[ScoredCandidate], + organization: Organization, +) -> list[TriageResult]: + """ + Call Seer LLM proxy to triage the candidate batch via a single LLM call. + Returns candidates the LLM didn't skip, with their assigned action. + """ + groups_by_id = {c.group.id: c.group for c in candidates} + + body = LlmGenerateRequest( + provider="gemini", + model="pro-preview", + referrer="night_shift.triage", + prompt=_build_triage_prompt(candidates), + system_prompt="", + temperature=0.0, + max_tokens=4096, + response_schema=_TriageResponse.schema(), + ) + + try: + response = make_llm_generate_request(body, timeout=60) + if response.status >= 400: + logger.error( + "night_shift.triage_request_failed", + extra={ + "organization_id": organization.id, + "status": response.status, + }, + ) + return [] + + data = orjson.loads(response.data) + content = data.get("content") + if not content: + logger.error( + "night_shift.triage_empty_response", + extra={"organization_id": organization.id}, + ) + return [] + + verdicts_data = orjson.loads(content) + except Exception: + logger.exception( + "night_shift.triage_request_error", + extra={"organization_id": organization.id}, + ) + return [] + + results: list[TriageResult] = [] + for verdict in verdicts_data.get("verdicts", []): + group = groups_by_id.get(verdict["group_id"]) + raw_action = verdict.get("action", "skip") + try: + action = TriageAction(raw_action) + except ValueError: + continue + if group is None: + continue + results.append(TriageResult(group=group, action=action)) + + logger.info( + "night_shift.triage_verdicts", + extra={ + "organization_id": organization.id, + "verdicts": {v["group_id"]: v["action"] for v in verdicts_data.get("verdicts", [])}, + }, + ) + + return results + + +def _build_triage_prompt( + candidates: list[ScoredCandidate], +) -> str: + candidates_block = "\n".join( + f"- group_id={c.group.id} | title={c.group.title or 'Unknown error'!r} " + f"| culprit={c.group.culprit or 'unknown'!r} " + f"| fixability={c.fixability:.2f} | times_seen={c.times_seen} " + f"| first_seen={c.group.first_seen.isoformat()} " + f"| priority={priority_label(c.group.priority) or 'unknown'}" + for c in candidates + ) + + return textwrap.dedent(f"""\ + You are a triage agent for Sentry's Night Shift system. Your job is to review + a batch of candidate issues and decide which ones are worth running automated + root-cause analysis and code fixes on. + + For each candidate, choose one action: + - "autofix": Run the full automated pipeline (root cause → solution → code changes). + Choose this for issues that look clearly fixable from their title/culprit and have + a high fixability score. + - "root_cause_only": Only run root-cause analysis, don't attempt a fix. + Choose this for issues that are worth investigating but may be too complex or + ambiguous to auto-fix confidently. + - "skip": Don't process this issue. + Choose this for issues that are vague, likely duplicates of each other in this + batch, or not worth spending compute on. + + Provide a brief reason for each decision. + + Candidates: + {candidates_block} + """) diff --git a/src/sentry/tasks/seer/night_shift.py b/src/sentry/tasks/seer/night_shift/cron.py similarity index 61% rename from src/sentry/tasks/seer/night_shift.py rename to src/sentry/tasks/seer/night_shift/cron.py index fad1463296697f..0c3af3c95b66b1 100644 --- a/src/sentry/tasks/seer/night_shift.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -2,23 +2,19 @@ import logging from collections.abc import Sequence -from dataclasses import dataclass from datetime import timedelta import sentry_sdk -from django.db.models import F from sentry import features, options from sentry.constants import ObjectStatus -from sentry.models.group import Group, GroupStatus from sentry.models.organization import Organization, OrganizationStatus from sentry.models.project import Project from sentry.seer.autofix.constants import AutofixAutomationTuningSettings -from sentry.seer.autofix.utils import is_issue_category_eligible from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.base import instrumented_task +from sentry.tasks.seer.night_shift.simple_triage import fixability_score_strategy from sentry.taskworker.namespaces import seer_tasks -from sentry.types.group import PriorityLevel from sentry.utils.iterators import chunked from sentry.utils.query import RangeQuerySetWrapper @@ -26,13 +22,6 @@ NIGHT_SHIFT_DISPATCH_STEP_SECONDS = 37 NIGHT_SHIFT_SPREAD_DURATION = timedelta(hours=4) -NIGHT_SHIFT_MAX_CANDIDATES = 10 -NIGHT_SHIFT_ISSUE_FETCH_LIMIT = 100 - -# Weights for candidate scoring. Set to 0 to disable a signal. -WEIGHT_FIXABILITY = 1.0 -WEIGHT_SEVERITY = 0.0 -WEIGHT_TIMES_SEEN = 0.0 FEATURE_NAMES = [ "organizations:seer-night-shift", @@ -64,9 +53,6 @@ def schedule_night_shift() -> None: 100, ): for org in _get_eligible_orgs_from_batch(org_batch): - if bool(org.get_option("sentry:hide_ai_features")): - continue - delay = (batch_index * NIGHT_SHIFT_DISPATCH_STEP_SECONDS) % spread_seconds run_night_shift_for_org.apply_async( @@ -81,28 +67,6 @@ def schedule_night_shift() -> None: ) -@dataclass -class _ScoredCandidate: - """A candidate issue with raw signals for ranking.""" - - group_id: int - project_id: int - fixability: float - times_seen: int - severity: float - - @property - def score(self) -> float: - return ( - WEIGHT_FIXABILITY * self.fixability - + WEIGHT_SEVERITY * self.severity - + WEIGHT_TIMES_SEEN * min(self.times_seen / 1000.0, 1.0) - ) - - def __lt__(self, other: _ScoredCandidate) -> bool: - return self.score < other.score - - @instrumented_task( name="sentry.tasks.seer.night_shift.run_night_shift_for_org", namespace=seer_tasks, @@ -134,7 +98,7 @@ def run_night_shift_for_org(organization_id: int) -> None: ) return - top_candidates = _fixability_score_strategy(eligible_projects) + candidates = fixability_score_strategy(eligible_projects) logger.info( "night_shift.candidates_selected", @@ -142,17 +106,13 @@ def run_night_shift_for_org(organization_id: int) -> None: "organization_id": organization_id, "organization_slug": organization.slug, "num_eligible_projects": len(eligible_projects), - "num_candidates": len(top_candidates), + "num_candidates": len(candidates), "candidates": [ { - "group_id": c.group_id, - "project_id": c.project_id, - "score": c.score, - "fixability": c.fixability, - "severity": c.severity, - "times_seen": c.times_seen, + "group_id": c.group.id, + "action": c.action, } - for c in top_candidates + for c in candidates ], }, ) @@ -165,7 +125,7 @@ def _get_eligible_orgs_from_batch( Check feature flags for a batch of orgs using batch_has_for_organizations. Returns orgs that have all required feature flags enabled. """ - eligible = list(orgs) + eligible = [org for org in orgs if not org.get_option("sentry:hide_ai_features")] for feature_name in FEATURE_NAMES: batch_result = features.batch_has_for_organizations(feature_name, eligible) @@ -197,41 +157,3 @@ def _get_eligible_projects(organization: Organization) -> list[Project]: for p in projects if p.get_option("sentry:autofix_automation_tuning") != AutofixAutomationTuningSettings.OFF ] - - -def _fixability_score_strategy( - projects: Sequence[Project], -) -> list[_ScoredCandidate]: - """ - Rank issues by existing fixability score with times_seen as tiebreaker. - Simple baseline — doesn't require any additional LLM calls. - """ - all_candidates: list[_ScoredCandidate] = [] - - for project_id_batch in chunked(projects, 100): - groups = Group.objects.filter( - project_id__in=[p.id for p in project_id_batch], - status=GroupStatus.UNRESOLVED, - seer_autofix_last_triggered__isnull=True, - seer_explorer_autofix_last_triggered__isnull=True, - ).order_by( - F("seer_fixability_score").desc(nulls_last=True), - F("times_seen").desc(), - )[:NIGHT_SHIFT_ISSUE_FETCH_LIMIT] - - for group in groups: - if not is_issue_category_eligible(group): - continue - - all_candidates.append( - _ScoredCandidate( - group_id=group.id, - project_id=group.project_id, - fixability=group.seer_fixability_score or 0.0, - times_seen=group.times_seen, - severity=(group.priority or 0) / PriorityLevel.HIGH, - ) - ) - - all_candidates.sort(reverse=True) - return all_candidates[:NIGHT_SHIFT_MAX_CANDIDATES] diff --git a/src/sentry/tasks/seer/night_shift/models.py b/src/sentry/tasks/seer/night_shift/models.py new file mode 100644 index 00000000000000..dc5941c4d0e87a --- /dev/null +++ b/src/sentry/tasks/seer/night_shift/models.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import enum +from dataclasses import dataclass + +from sentry.models.group import Group + + +class TriageAction(enum.StrEnum): + AUTOFIX = "autofix" + ROOT_CAUSE_ONLY = "root_cause_only" + + +@dataclass +class TriageResult: + group: Group + action: TriageAction = TriageAction.AUTOFIX diff --git a/src/sentry/tasks/seer/night_shift/simple_triage.py b/src/sentry/tasks/seer/night_shift/simple_triage.py new file mode 100644 index 00000000000000..97afd66a884c51 --- /dev/null +++ b/src/sentry/tasks/seer/night_shift/simple_triage.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence +from dataclasses import dataclass + +from django.db.models import F + +from sentry.models.group import Group, GroupStatus +from sentry.models.project import Project +from sentry.seer.autofix.utils import is_issue_category_eligible +from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult +from sentry.types.group import PriorityLevel +from sentry.utils.iterators import chunked + +logger = logging.getLogger("sentry.tasks.seer.night_shift") + +NIGHT_SHIFT_MAX_CANDIDATES = 10 +NIGHT_SHIFT_ISSUE_FETCH_LIMIT = 100 + +# Weights for candidate scoring. Set to 0 to disable a signal. +WEIGHT_FIXABILITY = 1.0 +WEIGHT_SEVERITY = 0.0 +WEIGHT_TIMES_SEEN = 0.0 + + +@dataclass +class ScoredCandidate(TriageResult): + """A candidate issue with raw signals for ranking.""" + + fixability: float = 0.0 + times_seen: int = 0 + severity: float = 0.0 + action: TriageAction = TriageAction.AUTOFIX + + @property + def score(self) -> float: + return ( + WEIGHT_FIXABILITY * self.fixability + + WEIGHT_SEVERITY * self.severity + + WEIGHT_TIMES_SEEN * min(self.times_seen / 1000.0, 1.0) + ) + + def __lt__(self, other: ScoredCandidate) -> bool: + return self.score < other.score + + +def fixability_score_strategy( + projects: Sequence[Project], +) -> list[ScoredCandidate]: + """ + Rank issues by existing fixability score with times_seen as tiebreaker. + Simple baseline — doesn't require any additional LLM calls. + """ + all_candidates: list[ScoredCandidate] = [] + + for project_id_batch in chunked(projects, 100): + groups = Group.objects.filter( + project_id__in=[p.id for p in project_id_batch], + status=GroupStatus.UNRESOLVED, + seer_autofix_last_triggered__isnull=True, + seer_explorer_autofix_last_triggered__isnull=True, + ).order_by( + F("seer_fixability_score").desc(nulls_last=True), + F("times_seen").desc(), + )[:NIGHT_SHIFT_ISSUE_FETCH_LIMIT] + + for group in groups: + if not is_issue_category_eligible(group): + continue + + all_candidates.append( + ScoredCandidate( + group=group, + fixability=group.seer_fixability_score or 0.0, + times_seen=group.times_seen, + severity=(group.priority or 0) / PriorityLevel.HIGH, + ) + ) + + all_candidates.sort(reverse=True) + return all_candidates[:NIGHT_SHIFT_MAX_CANDIDATES] + + +def priority_label(priority: int | None) -> str | None: + if priority is None: + return None + return PriorityLevel(priority).name.lower() diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index d014767612e653..d6d631d231d7b7 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -5,12 +5,12 @@ from sentry.models.group import GroupStatus from sentry.seer.autofix.constants import AutofixAutomationTuningSettings from sentry.seer.models.project_repository import SeerProjectRepository -from sentry.tasks.seer.night_shift import ( - _fixability_score_strategy, +from sentry.tasks.seer.night_shift.cron import ( _get_eligible_projects, run_night_shift_for_org, schedule_night_shift, ) +from sentry.tasks.seer.night_shift.simple_triage import fixability_score_strategy from sentry.testutils.cases import TestCase from sentry.testutils.pytest.fixtures import django_db_all @@ -20,7 +20,7 @@ class TestScheduleNightShift(TestCase): def test_disabled_by_option(self) -> None: with ( self.options({"seer.night_shift.enable": False}), - patch("sentry.tasks.seer.night_shift.run_night_shift_for_org") as mock_worker, + patch("sentry.tasks.seer.night_shift.cron.run_night_shift_for_org") as mock_worker, ): schedule_night_shift() mock_worker.apply_async.assert_not_called() @@ -36,7 +36,7 @@ def test_dispatches_eligible_orgs(self) -> None: "organizations:gen-ai-features": [org.slug], } ), - patch("sentry.tasks.seer.night_shift.run_night_shift_for_org") as mock_worker, + patch("sentry.tasks.seer.night_shift.cron.run_night_shift_for_org") as mock_worker, ): schedule_night_shift() mock_worker.apply_async.assert_called_once() @@ -47,7 +47,7 @@ def test_skips_ineligible_orgs(self) -> None: with ( self.options({"seer.night_shift.enable": True}), - patch("sentry.tasks.seer.night_shift.run_night_shift_for_org") as mock_worker, + patch("sentry.tasks.seer.night_shift.cron.run_night_shift_for_org") as mock_worker, ): schedule_night_shift() mock_worker.apply_async.assert_not_called() @@ -64,7 +64,7 @@ def test_skips_orgs_with_hidden_ai(self) -> None: "organizations:gen-ai-features": [org.slug], } ), - patch("sentry.tasks.seer.night_shift.run_night_shift_for_org") as mock_worker, + patch("sentry.tasks.seer.night_shift.cron.run_night_shift_for_org") as mock_worker, ): schedule_night_shift() mock_worker.apply_async.assert_not_called() @@ -105,7 +105,7 @@ def _make_eligible(self, project): SeerProjectRepository.objects.create(project=project, repository=repo) def test_nonexistent_org(self) -> None: - with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: + with patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger: run_night_shift_for_org(999999999) mock_logger.info.assert_not_called() @@ -113,7 +113,7 @@ def test_no_eligible_projects(self) -> None: org = self.create_organization() self.create_project(organization=org) - with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: + with patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger: run_night_shift_for_org(org.id) mock_logger.info.assert_called_once() assert mock_logger.info.call_args.args[0] == "night_shift.no_eligible_projects" @@ -143,7 +143,7 @@ def test_selects_candidates_and_skips_triggered(self) -> None: seer_autofix_last_triggered=timezone.now(), ) - with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: + with patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger: run_night_shift_for_org(org.id) call_extra = mock_logger.info.call_args.kwargs["extra"] @@ -170,19 +170,17 @@ def test_global_ranking_across_projects(self) -> None: seer_fixability_score=0.95, ) - with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: + with patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger: run_night_shift_for_org(org.id) candidates = mock_logger.info.call_args.kwargs["extra"]["candidates"] assert candidates[0]["group_id"] == high_group.id - assert candidates[0]["project_id"] == project_b.id assert candidates[1]["group_id"] == low_group.id - assert candidates[1]["project_id"] == project_a.id @django_db_all class TestFixabilityScoreStrategy(TestCase): - @patch("sentry.tasks.seer.night_shift.NIGHT_SHIFT_ISSUE_FETCH_LIMIT", 3) + @patch("sentry.tasks.seer.night_shift.simple_triage.NIGHT_SHIFT_ISSUE_FETCH_LIMIT", 3) def test_ranks_and_captures_signals(self) -> None: project = self.create_project() high = self.create_group( @@ -208,10 +206,10 @@ def test_ranks_and_captures_signals(self) -> None: times_seen=100, ) - result = _fixability_score_strategy([project]) + result = fixability_score_strategy([project]) - assert result[0].group_id == high.id + assert result[0].group.id == high.id assert result[0].fixability == 0.9 assert result[0].times_seen == 5 assert result[0].severity == 1.0 - assert result[1].group_id == low.id + assert result[1].group.id == low.id From 5f432735f586431c1c86a545d1f488e0a093cbb9 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 9 Apr 2026 16:45:25 -0400 Subject: [PATCH 2/8] switch to search backend --- .../tasks/seer/night_shift/simple_triage.py | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/src/sentry/tasks/seer/night_shift/simple_triage.py b/src/sentry/tasks/seer/night_shift/simple_triage.py index 97afd66a884c51..cc7b42ae93f7e0 100644 --- a/src/sentry/tasks/seer/night_shift/simple_triage.py +++ b/src/sentry/tasks/seer/night_shift/simple_triage.py @@ -4,14 +4,12 @@ from collections.abc import Sequence from dataclasses import dataclass -from django.db.models import F - -from sentry.models.group import Group, GroupStatus +from sentry import search +from sentry.api.event_search import SearchFilter, SearchKey, SearchValue from sentry.models.project import Project from sentry.seer.autofix.utils import is_issue_category_eligible from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult from sentry.types.group import PriorityLevel -from sentry.utils.iterators import chunked logger = logging.getLogger("sentry.tasks.seer.night_shift") @@ -49,37 +47,36 @@ def fixability_score_strategy( projects: Sequence[Project], ) -> list[ScoredCandidate]: """ - Rank issues by existing fixability score with times_seen as tiebreaker. - Simple baseline — doesn't require any additional LLM calls. + Fetch top recommended unresolved issues that haven't been triaged by Seer yet, + then re-rank by fixability score. Doesn't require any additional LLM calls. """ - all_candidates: list[ScoredCandidate] = [] - - for project_id_batch in chunked(projects, 100): - groups = Group.objects.filter( - project_id__in=[p.id for p in project_id_batch], - status=GroupStatus.UNRESOLVED, - seer_autofix_last_triggered__isnull=True, - seer_explorer_autofix_last_triggered__isnull=True, - ).order_by( - F("seer_fixability_score").desc(nulls_last=True), - F("times_seen").desc(), - )[:NIGHT_SHIFT_ISSUE_FETCH_LIMIT] - - for group in groups: - if not is_issue_category_eligible(group): - continue - - all_candidates.append( - ScoredCandidate( - group=group, - fixability=group.seer_fixability_score or 0.0, - times_seen=group.times_seen, - severity=(group.priority or 0) / PriorityLevel.HIGH, - ) + result = search.backend.query( + projects=projects, + sort_by="recommended", + limit=NIGHT_SHIFT_ISSUE_FETCH_LIMIT, + search_filters=[ + SearchFilter(SearchKey("status"), "=", SearchValue("unresolved")), + SearchFilter(SearchKey("issue.seer_last_run"), "=", SearchValue("")), + ], + referrer="seer.night_shift.fixability_score_strategy", + ) + + candidates: list[ScoredCandidate] = [] + for group in result.results: + if not is_issue_category_eligible(group): + continue + + candidates.append( + ScoredCandidate( + group=group, + fixability=group.seer_fixability_score or 0.0, + times_seen=group.times_seen, + severity=(group.priority or 0) / PriorityLevel.HIGH, ) + ) - all_candidates.sort(reverse=True) - return all_candidates[:NIGHT_SHIFT_MAX_CANDIDATES] + candidates.sort(reverse=True) + return candidates[:NIGHT_SHIFT_MAX_CANDIDATES] def priority_label(priority: int | None) -> str | None: From 808e2bff9d78838999c35cbaee5a9f0da18ab05f Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 9 Apr 2026 17:33:32 -0400 Subject: [PATCH 3/8] fix --- .../tasks/seer/night_shift/simple_triage.py | 3 +- tests/sentry/tasks/seer/test_night_shift.py | 92 ++++++++++--------- 2 files changed, 49 insertions(+), 46 deletions(-) diff --git a/src/sentry/tasks/seer/night_shift/simple_triage.py b/src/sentry/tasks/seer/night_shift/simple_triage.py index cc7b42ae93f7e0..d70f669e17a0b8 100644 --- a/src/sentry/tasks/seer/night_shift/simple_triage.py +++ b/src/sentry/tasks/seer/night_shift/simple_triage.py @@ -6,6 +6,7 @@ from sentry import search from sentry.api.event_search import SearchFilter, SearchKey, SearchValue +from sentry.models.group import GroupStatus from sentry.models.project import Project from sentry.seer.autofix.utils import is_issue_category_eligible from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult @@ -55,7 +56,7 @@ def fixability_score_strategy( sort_by="recommended", limit=NIGHT_SHIFT_ISSUE_FETCH_LIMIT, search_filters=[ - SearchFilter(SearchKey("status"), "=", SearchValue("unresolved")), + SearchFilter(SearchKey("status"), "=", SearchValue([GroupStatus.UNRESOLVED])), SearchFilter(SearchKey("issue.seer_last_run"), "=", SearchValue("")), ], referrer="seer.night_shift.fixability_score_strategy", diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index d6d631d231d7b7..5969f3fdd3aa08 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -2,7 +2,7 @@ from django.utils import timezone -from sentry.models.group import GroupStatus +from sentry.models.group import Group from sentry.seer.autofix.constants import AutofixAutomationTuningSettings from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.seer.night_shift.cron import ( @@ -11,7 +11,8 @@ schedule_night_shift, ) from sentry.tasks.seer.night_shift.simple_triage import fixability_score_strategy -from sentry.testutils.cases import TestCase +from sentry.testutils.cases import SnubaTestCase, TestCase +from sentry.testutils.helpers.datetime import before_now from sentry.testutils.pytest.fixtures import django_db_all @@ -96,7 +97,7 @@ def test_filters_by_automation_and_repos(self) -> None: @django_db_all -class TestRunNightShiftForOrg(TestCase): +class TestRunNightShiftForOrg(TestCase, SnubaTestCase): def _make_eligible(self, project): project.update_option( "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM @@ -104,6 +105,18 @@ def _make_eligible(self, project): repo = self.create_repo(project=project, provider="github") SeerProjectRepository.objects.create(project=project, repository=repo) + def _store_event_and_update_group(self, project, fingerprint, **group_attrs): + event = self.store_event( + data={ + "fingerprint": [fingerprint], + "timestamp": before_now(hours=1).isoformat(), + "environment": "production", + }, + project_id=project.id, + ) + Group.objects.filter(id=event.group_id).update(**group_attrs) + return Group.objects.get(id=event.group_id) + def test_nonexistent_org(self) -> None: with patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger: run_night_shift_for_org(999999999) @@ -123,22 +136,16 @@ def test_selects_candidates_and_skips_triggered(self) -> None: project = self.create_project(organization=org) self._make_eligible(project) - high_fix = self.create_group( - project=project, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=0.9, - times_seen=5, + high_fix = self._store_event_and_update_group( + project, "high-fix", seer_fixability_score=0.9, times_seen=5 ) - low_fix = self.create_group( - project=project, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=0.2, - times_seen=100, + low_fix = self._store_event_and_update_group( + project, "low-fix", seer_fixability_score=0.2, times_seen=100 ) # Already triggered — should be excluded - self.create_group( - project=project, - status=GroupStatus.UNRESOLVED, + self._store_event_and_update_group( + project, + "triggered", seer_fixability_score=0.95, seer_autofix_last_triggered=timezone.now(), ) @@ -159,15 +166,11 @@ def test_global_ranking_across_projects(self) -> None: self._make_eligible(project_a) self._make_eligible(project_b) - low_group = self.create_group( - project=project_a, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=0.3, + low_group = self._store_event_and_update_group( + project_a, "low-group", seer_fixability_score=0.3 ) - high_group = self.create_group( - project=project_b, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=0.95, + high_group = self._store_event_and_update_group( + project_b, "high-group", seer_fixability_score=0.95 ) with patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger: @@ -179,31 +182,30 @@ def test_global_ranking_across_projects(self) -> None: @django_db_all -class TestFixabilityScoreStrategy(TestCase): - @patch("sentry.tasks.seer.night_shift.simple_triage.NIGHT_SHIFT_ISSUE_FETCH_LIMIT", 3) +class TestFixabilityScoreStrategy(TestCase, SnubaTestCase): + def _store_event_and_update_group(self, project, fingerprint, **group_attrs): + event = self.store_event( + data={ + "fingerprint": [fingerprint], + "timestamp": before_now(hours=1).isoformat(), + "environment": "production", + }, + project_id=project.id, + ) + Group.objects.filter(id=event.group_id).update(**group_attrs) + return Group.objects.get(id=event.group_id) + def test_ranks_and_captures_signals(self) -> None: project = self.create_project() - high = self.create_group( - project=project, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=0.9, - times_seen=5, - priority=75, + high = self._store_event_and_update_group( + project, "high", seer_fixability_score=0.9, times_seen=5, priority=75 ) - low = self.create_group( - project=project, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=0.2, - times_seen=500, + low = self._store_event_and_update_group( + project, "low", seer_fixability_score=0.2, times_seen=500 ) - # NULL-scored issues should sort after scored ones even with a tight DB limit. - # Without nulls_last these would fill the limit and exclude scored issues. - for _ in range(3): - self.create_group( - project=project, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=None, - times_seen=100, + for i in range(3): + self._store_event_and_update_group( + project, f"null-{i}", seer_fixability_score=None, times_seen=100 ) result = fixability_score_strategy([project]) From 2a17e8a2e30c6fb5cf384a201d6a3ed299af5783 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 9 Apr 2026 17:56:47 -0400 Subject: [PATCH 4/8] add script --- bin/seer/trigger-night-shift | 25 +++++++++++++++++++ src/sentry/snuba/referrer.py | 1 + .../tasks/seer/night_shift/simple_triage.py | 3 ++- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100755 bin/seer/trigger-night-shift diff --git a/bin/seer/trigger-night-shift b/bin/seer/trigger-night-shift new file mode 100755 index 00000000000000..3c146f8b23ff14 --- /dev/null +++ b/bin/seer/trigger-night-shift @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +from sentry.runner import configure + +configure() + +import argparse +import sys + +from sentry.tasks.seer.night_shift.cron import run_night_shift_for_org + + +def main(org_id: int) -> None: + sys.stdout.write(f"> Running night shift for organization {org_id}...\n") + run_night_shift_for_org(org_id) + sys.stdout.write("> Done.\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Trigger night shift for an organization.") + parser.add_argument( + "org_id", nargs="?", default=1, type=int, help="Organization ID (default: 1)" + ) + args = parser.parse_args() + main(args.org_id) diff --git a/src/sentry/snuba/referrer.py b/src/sentry/snuba/referrer.py index 65cff749297787..93bda3ba687119 100644 --- a/src/sentry/snuba/referrer.py +++ b/src/sentry/snuba/referrer.py @@ -778,6 +778,7 @@ class Referrer(StrEnum): SEARCH_SAMPLE = "search_sample" SEARCH = "search" SEARCH_GROUP_INDEX = "search.group_index" + SEER_NIGHT_SHIFT_FIXABILITY_SCORE_STRATEGY = "seer.night_shift.fixability_score_strategy" SEARCH_GROUP_INDEX_SAMPLE = "search.group_index_sample" SEARCH_SNUBA_GROUP_ATTRIBUTES_SEARCH_QUERY = "search.snuba.group_attributes_search.query" SEARCH_SNUBA_GROUP_ATTRIBUTES_SEARCH_HITS = "search.snuba.group_attributes_search.hits" diff --git a/src/sentry/tasks/seer/night_shift/simple_triage.py b/src/sentry/tasks/seer/night_shift/simple_triage.py index d70f669e17a0b8..0ea58aedf9c0e2 100644 --- a/src/sentry/tasks/seer/night_shift/simple_triage.py +++ b/src/sentry/tasks/seer/night_shift/simple_triage.py @@ -9,6 +9,7 @@ from sentry.models.group import GroupStatus from sentry.models.project import Project from sentry.seer.autofix.utils import is_issue_category_eligible +from sentry.snuba.referrer import Referrer from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult from sentry.types.group import PriorityLevel @@ -59,7 +60,7 @@ def fixability_score_strategy( SearchFilter(SearchKey("status"), "=", SearchValue([GroupStatus.UNRESOLVED])), SearchFilter(SearchKey("issue.seer_last_run"), "=", SearchValue("")), ], - referrer="seer.night_shift.fixability_score_strategy", + referrer=Referrer.SEER_NIGHT_SHIFT_FIXABILITY_SCORE_STRATEGY.value, ) candidates: list[ScoredCandidate] = [] From a67440f7b2e39cc06bae7bf9bddc73fea6b66ec2 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 9 Apr 2026 18:03:20 -0400 Subject: [PATCH 5/8] fix types --- .../tasks/seer/night_shift/agentic_triage.py | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/sentry/tasks/seer/night_shift/agentic_triage.py b/src/sentry/tasks/seer/night_shift/agentic_triage.py index add2510166bad9..c1921c011acaf3 100644 --- a/src/sentry/tasks/seer/night_shift/agentic_triage.py +++ b/src/sentry/tasks/seer/night_shift/agentic_triage.py @@ -22,7 +22,7 @@ class _TriageVerdict(pydantic.BaseModel): group_id: int - action: str + action: TriageAction reason: str @@ -87,7 +87,7 @@ def _triage_candidates( ) return [] - verdicts_data = orjson.loads(content) + triage_response: _TriageResponse = _TriageResponse.model_validate_json(content) except Exception: logger.exception( "night_shift.triage_request_error", @@ -95,23 +95,17 @@ def _triage_candidates( ) return [] - results: list[TriageResult] = [] - for verdict in verdicts_data.get("verdicts", []): - group = groups_by_id.get(verdict["group_id"]) - raw_action = verdict.get("action", "skip") - try: - action = TriageAction(raw_action) - except ValueError: - continue - if group is None: - continue - results.append(TriageResult(group=group, action=action)) + results = [ + TriageResult(group=groups_by_id[v.group_id], action=v.action) + for v in triage_response.verdicts + if v.group_id in groups_by_id + ] logger.info( "night_shift.triage_verdicts", extra={ "organization_id": organization.id, - "verdicts": {v["group_id"]: v["action"] for v in verdicts_data.get("verdicts", [])}, + "verdicts": {v.group_id: v.action for v in triage_response.verdicts}, }, ) From d6d6616d4e4d41844303befb3a07c8bc3762db63 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 9 Apr 2026 18:18:23 -0400 Subject: [PATCH 6/8] fix --- src/sentry/tasks/seer/night_shift/simple_triage.py | 8 ++++++++ src/sentry/testutils/cases.py | 5 ++++- tests/sentry/tasks/seer/test_night_shift.py | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/sentry/tasks/seer/night_shift/simple_triage.py b/src/sentry/tasks/seer/night_shift/simple_triage.py index 0ea58aedf9c0e2..6562c9c140180d 100644 --- a/src/sentry/tasks/seer/night_shift/simple_triage.py +++ b/src/sentry/tasks/seer/night_shift/simple_triage.py @@ -63,6 +63,14 @@ def fixability_score_strategy( referrer=Referrer.SEER_NIGHT_SHIFT_FIXABILITY_SCORE_STRATEGY.value, ) + logger.info( + "night_shift.search_results", + extra={ + "num_projects": len(projects), + "num_results": len(result.results), + }, + ) + candidates: list[ScoredCandidate] = [] for group in result.results: if not is_issue_category_eligible(group): diff --git a/src/sentry/testutils/cases.py b/src/sentry/testutils/cases.py index ed64d3cda455ae..256a84435a574e 100644 --- a/src/sentry/testutils/cases.py +++ b/src/sentry/testutils/cases.py @@ -1050,13 +1050,16 @@ class SnubaTestCase(BaseTestCase): # We need Django to flush all databases. databases: set[str] | str = "__all__" + reset_snuba_data: bool = True def setUp(self): super().setUp() self.init_snuba() @pytest.fixture(autouse=True) - def initialize(self, reset_snuba, call_snuba): + def initialize(self, request, call_snuba): + if self.reset_snuba_data: + request.getfixturevalue("reset_snuba") self.call_snuba = call_snuba def create_project(self, **kwargs) -> Project: diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index 5969f3fdd3aa08..a7a3a314668960 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -98,6 +98,8 @@ def test_filters_by_automation_and_repos(self) -> None: @django_db_all class TestRunNightShiftForOrg(TestCase, SnubaTestCase): + reset_snuba_data = False + def _make_eligible(self, project): project.update_option( "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM @@ -183,6 +185,8 @@ def test_global_ranking_across_projects(self) -> None: @django_db_all class TestFixabilityScoreStrategy(TestCase, SnubaTestCase): + reset_snuba_data = False + def _store_event_and_update_group(self, project, fingerprint, **group_attrs): event = self.store_event( data={ From 77f6767b2a6091035002d75d24278de2bc0842f2 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 9 Apr 2026 22:29:43 -0400 Subject: [PATCH 7/8] fix --- .../tasks/seer/night_shift/agentic_triage.py | 6 +++- src/sentry/tasks/seer/night_shift/cron.py | 4 +-- src/sentry/tasks/seer/night_shift/models.py | 1 + tests/sentry/tasks/seer/test_night_shift.py | 28 +++++++++++++++++-- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/sentry/tasks/seer/night_shift/agentic_triage.py b/src/sentry/tasks/seer/night_shift/agentic_triage.py index c1921c011acaf3..cc0a74c901afae 100644 --- a/src/sentry/tasks/seer/night_shift/agentic_triage.py +++ b/src/sentry/tasks/seer/night_shift/agentic_triage.py @@ -29,6 +29,10 @@ class _TriageVerdict(pydantic.BaseModel): class _TriageResponse(pydantic.BaseModel): verdicts: list[_TriageVerdict] + @pydantic.validator("verdicts") + def filter_skips(cls, v: list[_TriageVerdict]) -> list[_TriageVerdict]: + return [verdict for verdict in v if verdict.action != TriageAction.SKIP] + def agentic_triage_strategy( projects: Sequence[Project], @@ -87,7 +91,7 @@ def _triage_candidates( ) return [] - triage_response: _TriageResponse = _TriageResponse.model_validate_json(content) + triage_response = _TriageResponse.parse_raw(content) except Exception: logger.exception( "night_shift.triage_request_error", diff --git a/src/sentry/tasks/seer/night_shift/cron.py b/src/sentry/tasks/seer/night_shift/cron.py index 0c3af3c95b66b1..f7d1d279e97451 100644 --- a/src/sentry/tasks/seer/night_shift/cron.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -13,7 +13,7 @@ from sentry.seer.autofix.constants import AutofixAutomationTuningSettings from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.base import instrumented_task -from sentry.tasks.seer.night_shift.simple_triage import fixability_score_strategy +from sentry.tasks.seer.night_shift.agentic_triage import agentic_triage_strategy from sentry.taskworker.namespaces import seer_tasks from sentry.utils.iterators import chunked from sentry.utils.query import RangeQuerySetWrapper @@ -98,7 +98,7 @@ def run_night_shift_for_org(organization_id: int) -> None: ) return - candidates = fixability_score_strategy(eligible_projects) + candidates = agentic_triage_strategy(eligible_projects, organization) logger.info( "night_shift.candidates_selected", diff --git a/src/sentry/tasks/seer/night_shift/models.py b/src/sentry/tasks/seer/night_shift/models.py index dc5941c4d0e87a..6380bb3819c3f7 100644 --- a/src/sentry/tasks/seer/night_shift/models.py +++ b/src/sentry/tasks/seer/night_shift/models.py @@ -9,6 +9,7 @@ class TriageAction(enum.StrEnum): AUTOFIX = "autofix" ROOT_CAUSE_ONLY = "root_cause_only" + SKIP = "skip" @dataclass diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index a7a3a314668960..4bb0d4b51fec43 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch from django.utils import timezone @@ -14,6 +14,16 @@ from sentry.testutils.cases import SnubaTestCase, TestCase from sentry.testutils.helpers.datetime import before_now from sentry.testutils.pytest.fixtures import django_db_all +from sentry.utils import json + + +def _mock_llm_response(group_ids: list[int], action: str = "autofix") -> MagicMock: + verdicts = [{"group_id": gid, "action": action, "reason": "test"} for gid in group_ids] + content = json.dumps({"verdicts": verdicts}) + response = MagicMock() + response.status = 200 + response.data = json.dumps({"content": content}).encode() + return response @django_db_all @@ -152,7 +162,13 @@ def test_selects_candidates_and_skips_triggered(self) -> None: seer_autofix_last_triggered=timezone.now(), ) - with patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger: + with ( + patch( + "sentry.tasks.seer.night_shift.agentic_triage.make_llm_generate_request", + return_value=_mock_llm_response([high_fix.id, low_fix.id]), + ), + patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger, + ): run_night_shift_for_org(org.id) call_extra = mock_logger.info.call_args.kwargs["extra"] @@ -175,7 +191,13 @@ def test_global_ranking_across_projects(self) -> None: project_b, "high-group", seer_fixability_score=0.95 ) - with patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger: + with ( + patch( + "sentry.tasks.seer.night_shift.agentic_triage.make_llm_generate_request", + return_value=_mock_llm_response([high_group.id, low_group.id]), + ), + patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger, + ): run_night_shift_for_org(org.id) candidates = mock_logger.info.call_args.kwargs["extra"]["candidates"] From 7d9b2a5c6437cbbbd890e3d86a53bbcd9fc4b0f7 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Thu, 9 Apr 2026 22:45:27 -0400 Subject: [PATCH 8/8] fix --- src/sentry/tasks/seer/night_shift/simple_triage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sentry/tasks/seer/night_shift/simple_triage.py b/src/sentry/tasks/seer/night_shift/simple_triage.py index 6562c9c140180d..86ec27c731f6ab 100644 --- a/src/sentry/tasks/seer/night_shift/simple_triage.py +++ b/src/sentry/tasks/seer/night_shift/simple_triage.py @@ -92,4 +92,7 @@ def fixability_score_strategy( def priority_label(priority: int | None) -> str | None: if priority is None: return None - return PriorityLevel(priority).name.lower() + try: + return PriorityLevel(priority).name.lower() + except ValueError: + return None