ref(seer): Refactor night shift into modules and use search backend (#112635)

Refactors night shift from a single file into a package with separate
modules for cron scheduling, simple triage, agentic triage, and shared
models.

Switches `fixability_score_strategy` from a direct ORM query to using
`search.backend.query()` with the `recommended` sort. This gives us the
same ranking algorithm used in the issues list UI (recency, spike
detection, severity, user impact, event volume) as a pre-filter, then
re-ranks by fixability score in-memory.

Other changes:
- Adds `reset_snuba_data` flag to `SnubaTestCase` so tests can opt out
of dropping ClickHouse data between runs
- Adds `bin/seer/trigger-night-shift` script for local testing
- Registers a new Snuba referrer for the search query
- Uses pydantic `model_validate_json` for LLM response parsing in
agentic triage

Author: trevor-e

bin/seer/trigger-night-shift

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+from sentry.runner import configure
+
+configure()
+
+import argparse
+import sys
+
+from sentry.tasks.seer.night_shift.cron import run_night_shift_for_org
+
+
+def main(org_id: int) -> None:
+    sys.stdout.write(f"> Running night shift for organization {org_id}...\n")
+    run_night_shift_for_org(org_id)
+    sys.stdout.write("> Done.\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Trigger night shift for an organization.")
+    parser.add_argument(
+        "org_id", nargs="?", default=1, type=int, help="Organization ID (default: 1)"
+    )
+    args = parser.parse_args()
+    main(args.org_id)

src/sentry/conf/server.py

@@ -992,7 +992,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "sentry.tasks.seer.explorer_index",
     "sentry.tasks.seer.context_engine_index",
     "sentry.tasks.seer.lightweight_rca_cluster",
-    "sentry.tasks.seer.night_shift",
+    "sentry.tasks.seer.night_shift.cron",
     # Used for tests
     "sentry.taskworker.tasks.examples",
 )

src/sentry/snuba/referrer.py

@@ -778,6 +778,7 @@ class Referrer(StrEnum):
     SEARCH_SAMPLE = "search_sample"
     SEARCH = "search"
     SEARCH_GROUP_INDEX = "search.group_index"
+    SEER_NIGHT_SHIFT_FIXABILITY_SCORE_STRATEGY = "seer.night_shift.fixability_score_strategy"
     SEARCH_GROUP_INDEX_SAMPLE = "search.group_index_sample"
     SEARCH_SNUBA_GROUP_ATTRIBUTES_SEARCH_QUERY = "search.snuba.group_attributes_search.query"
     SEARCH_SNUBA_GROUP_ATTRIBUTES_SEARCH_HITS = "search.snuba.group_attributes_search.hits"

src/sentry/tasks/seer/night_shift/__init__.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import logging
+import textwrap
+from collections.abc import Sequence
+
+import orjson
+import pydantic
+
+from sentry.models.organization import Organization
+from sentry.models.project import Project
+from sentry.seer.signed_seer_api import LlmGenerateRequest, make_llm_generate_request
+from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult
+from sentry.tasks.seer.night_shift.simple_triage import (
+    ScoredCandidate,
+    fixability_score_strategy,
+    priority_label,
+)
+
+logger = logging.getLogger("sentry.tasks.seer.night_shift")
+
+
+class _TriageVerdict(pydantic.BaseModel):
+    group_id: int
+    action: TriageAction
+    reason: str
+
+
+class _TriageResponse(pydantic.BaseModel):
+    verdicts: list[_TriageVerdict]
+
+    @pydantic.validator("verdicts")
+    def filter_skips(cls, v: list[_TriageVerdict]) -> list[_TriageVerdict]:
+        return [verdict for verdict in v if verdict.action != TriageAction.SKIP]
+
+
+def agentic_triage_strategy(
+    projects: Sequence[Project],
+    organization: Organization,
+) -> list[TriageResult]:
+    """
+    Select candidates via fixability scoring, then filter through an LLM
+    triage call that decides the action for each candidate.
+    """
+    scored = fixability_score_strategy(projects)
+    if not scored:
+        return []
+
+    return _triage_candidates(scored, organization)
+
+
+def _triage_candidates(
+    candidates: list[ScoredCandidate],
+    organization: Organization,
+) -> list[TriageResult]:
+    """
+    Call Seer LLM proxy to triage the candidate batch via a single LLM call.
+    Returns candidates the LLM didn't skip, with their assigned action.
+    """
+    groups_by_id = {c.group.id: c.group for c in candidates}
+
+    body = LlmGenerateRequest(
+        provider="gemini",
+        model="pro-preview",
+        referrer="night_shift.triage",
+        prompt=_build_triage_prompt(candidates),
+        system_prompt="",
+        temperature=0.0,
+        max_tokens=4096,
+        response_schema=_TriageResponse.schema(),
+    )
+
+    try:
+        response = make_llm_generate_request(body, timeout=60)
+        if response.status >= 400:
+            logger.error(
+                "night_shift.triage_request_failed",
+                extra={
+                    "organization_id": organization.id,
+                    "status": response.status,
+                },
+            )
+            return []
+
+        data = orjson.loads(response.data)
+        content = data.get("content")
+        if not content:
+            logger.error(
+                "night_shift.triage_empty_response",
+                extra={"organization_id": organization.id},
+            )
+            return []
+
+        triage_response = _TriageResponse.parse_raw(content)
+    except Exception:
+        logger.exception(
+            "night_shift.triage_request_error",
+            extra={"organization_id": organization.id},
+        )
+        return []
+
+    results = [
+        TriageResult(group=groups_by_id[v.group_id], action=v.action)
+        for v in triage_response.verdicts
+        if v.group_id in groups_by_id
+    ]
+
+    logger.info(
+        "night_shift.triage_verdicts",
+        extra={
+            "organization_id": organization.id,
+            "verdicts": {v.group_id: v.action for v in triage_response.verdicts},
+        },
+    )
+
+    return results
+
+
+def _build_triage_prompt(
+    candidates: list[ScoredCandidate],
+) -> str:
+    candidates_block = "\n".join(
+        f"- group_id={c.group.id} | title={c.group.title or 'Unknown error'!r} "
+        f"| culprit={c.group.culprit or 'unknown'!r} "
+        f"| fixability={c.fixability:.2f} | times_seen={c.times_seen} "
+        f"| first_seen={c.group.first_seen.isoformat()} "
+        f"| priority={priority_label(c.group.priority) or 'unknown'}"
+        for c in candidates
+    )
+
+    return textwrap.dedent(f"""\
+        You are a triage agent for Sentry's Night Shift system. Your job is to review
+        a batch of candidate issues and decide which ones are worth running automated
+        root-cause analysis and code fixes on.
+
+        For each candidate, choose one action:
+        - "autofix": Run the full automated pipeline (root cause → solution → code changes).
+          Choose this for issues that look clearly fixable from their title/culprit and have
+          a high fixability score.
+        - "root_cause_only": Only run root-cause analysis, don't attempt a fix.
+          Choose this for issues that are worth investigating but may be too complex or
+          ambiguous to auto-fix confidently.
+        - "skip": Don't process this issue.
+          Choose this for issues that are vague, likely duplicates of each other in this
+          batch, or not worth spending compute on.
+
+        Provide a brief reason for each decision.
+
+        Candidates:
+        {candidates_block}
+    """)

src/sentry/tasks/seer/night_shift/agentic_triage.py

@@ -2,37 +2,26 @@
 
 import logging
 from collections.abc import Sequence
-from dataclasses import dataclass
 from datetime import timedelta
 
 import sentry_sdk
-from django.db.models import F
 
 from sentry import features, options
 from sentry.constants import ObjectStatus
-from sentry.models.group import Group, GroupStatus
 from sentry.models.organization import Organization, OrganizationStatus
 from sentry.models.project import Project
 from sentry.seer.autofix.constants import AutofixAutomationTuningSettings
-from sentry.seer.autofix.utils import is_issue_category_eligible
 from sentry.seer.models.project_repository import SeerProjectRepository
 from sentry.tasks.base import instrumented_task
+from sentry.tasks.seer.night_shift.agentic_triage import agentic_triage_strategy
 from sentry.taskworker.namespaces import seer_tasks
-from sentry.types.group import PriorityLevel
 from sentry.utils.iterators import chunked
 from sentry.utils.query import RangeQuerySetWrapper
 
 logger = logging.getLogger("sentry.tasks.seer.night_shift")
 
 NIGHT_SHIFT_DISPATCH_STEP_SECONDS = 37
 NIGHT_SHIFT_SPREAD_DURATION = timedelta(hours=4)
-NIGHT_SHIFT_MAX_CANDIDATES = 10
-NIGHT_SHIFT_ISSUE_FETCH_LIMIT = 100
-
-# Weights for candidate scoring. Set to 0 to disable a signal.
-WEIGHT_FIXABILITY = 1.0
-WEIGHT_SEVERITY = 0.0
-WEIGHT_TIMES_SEEN = 0.0
 
 FEATURE_NAMES = [
     "organizations:seer-night-shift",
@@ -64,9 +53,6 @@ def schedule_night_shift() -> None:
         100,
     ):
         for org in _get_eligible_orgs_from_batch(org_batch):
-            if bool(org.get_option("sentry:hide_ai_features")):
-                continue
-
             delay = (batch_index * NIGHT_SHIFT_DISPATCH_STEP_SECONDS) % spread_seconds
 
             run_night_shift_for_org.apply_async(
@@ -81,28 +67,6 @@ def schedule_night_shift() -> None:
     )
 
 
-@dataclass
-class _ScoredCandidate:
-    """A candidate issue with raw signals for ranking."""
-
-    group_id: int
-    project_id: int
-    fixability: float
-    times_seen: int
-    severity: float
-
-    @property
-    def score(self) -> float:
-        return (
-            WEIGHT_FIXABILITY * self.fixability
-            + WEIGHT_SEVERITY * self.severity
-            + WEIGHT_TIMES_SEEN * min(self.times_seen / 1000.0, 1.0)
-        )
-
-    def __lt__(self, other: _ScoredCandidate) -> bool:
-        return self.score < other.score
-
-
 @instrumented_task(
     name="sentry.tasks.seer.night_shift.run_night_shift_for_org",
     namespace=seer_tasks,
@@ -134,25 +98,21 @@ def run_night_shift_for_org(organization_id: int) -> None:
         )
         return
 
-    top_candidates = _fixability_score_strategy(eligible_projects)
+    candidates = agentic_triage_strategy(eligible_projects, organization)
 
     logger.info(
         "night_shift.candidates_selected",
         extra={
             "organization_id": organization_id,
             "organization_slug": organization.slug,
             "num_eligible_projects": len(eligible_projects),
-            "num_candidates": len(top_candidates),
+            "num_candidates": len(candidates),
             "candidates": [
                 {
-                    "group_id": c.group_id,
-                    "project_id": c.project_id,
-                    "score": c.score,
-                    "fixability": c.fixability,
-                    "severity": c.severity,
-                    "times_seen": c.times_seen,
+                    "group_id": c.group.id,
+                    "action": c.action,
                 }
-                for c in top_candidates
+                for c in candidates
             ],
         },
     )
@@ -165,7 +125,7 @@ def _get_eligible_orgs_from_batch(
     Check feature flags for a batch of orgs using batch_has_for_organizations.
     Returns orgs that have all required feature flags enabled.
     """
-    eligible = list(orgs)
+    eligible = [org for org in orgs if not org.get_option("sentry:hide_ai_features")]
 
     for feature_name in FEATURE_NAMES:
         batch_result = features.batch_has_for_organizations(feature_name, eligible)
@@ -197,41 +157,3 @@ def _get_eligible_projects(organization: Organization) -> list[Project]:
         for p in projects
         if p.get_option("sentry:autofix_automation_tuning") != AutofixAutomationTuningSettings.OFF
     ]
-
-
-def _fixability_score_strategy(
-    projects: Sequence[Project],
-) -> list[_ScoredCandidate]:
-    """
-    Rank issues by existing fixability score with times_seen as tiebreaker.
-    Simple baseline — doesn't require any additional LLM calls.
-    """
-    all_candidates: list[_ScoredCandidate] = []
-
-    for project_id_batch in chunked(projects, 100):
-        groups = Group.objects.filter(
-            project_id__in=[p.id for p in project_id_batch],
-            status=GroupStatus.UNRESOLVED,
-            seer_autofix_last_triggered__isnull=True,
-            seer_explorer_autofix_last_triggered__isnull=True,
-        ).order_by(
-            F("seer_fixability_score").desc(nulls_last=True),
-            F("times_seen").desc(),
-        )[:NIGHT_SHIFT_ISSUE_FETCH_LIMIT]
-
-        for group in groups:
-            if not is_issue_category_eligible(group):
-                continue
-
-            all_candidates.append(
-                _ScoredCandidate(
-                    group_id=group.id,
-                    project_id=group.project_id,
-                    fixability=group.seer_fixability_score or 0.0,
-                    times_seen=group.times_seen,
-                    severity=(group.priority or 0) / PriorityLevel.HIGH,
-                )
-            )
-
-    all_candidates.sort(reverse=True)
-    return all_candidates[:NIGHT_SHIFT_MAX_CANDIDATES]

src/sentry/tasks/seer/night_shift.py src/sentry/tasks/seer/night_shift/cron.pysrc/sentry/tasks/seer/night_shift.py renamed to src/sentry/tasks/seer/night_shift/cron.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass
+
+from sentry.models.group import Group
+
+
+class TriageAction(enum.StrEnum):
+    AUTOFIX = "autofix"
+    ROOT_CAUSE_ONLY = "root_cause_only"
+    SKIP = "skip"
+
+
+@dataclass
+class TriageResult:
+    group: Group
+    action: TriageAction = TriageAction.AUTOFIX