From 81226481472aaa2dfad8f26ef6b43f14da6030eb Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Wed, 8 Apr 2026 16:30:12 -0400 Subject: [PATCH 1/5] feat(seer): Add candidate issue selection to night shift Implement the per-org worker task to query and rank candidate issues for night shift autofix. Issues are selected across eligible projects using a weighted scoring system with pluggable strategy functions. - Add _get_eligible_projects: filters to active projects with automation enabled and connected repos via SeerProjectRepository - Add _fixability_score_strategy: ranks unresolved issues by fixability score, severity, and times_seen with configurable weights - Add _ScoredCandidate dataclass with raw signals and computed score - Log ranked candidates for inspection before triggering autofix Co-Authored-By: Claude Opus 4.6 --- src/sentry/tasks/seer/night_shift.py | 118 ++++++++++- tests/sentry/tasks/seer/test_night_shift.py | 204 +++++++++++++++++++- 2 files changed, 315 insertions(+), 7 deletions(-) diff --git a/src/sentry/tasks/seer/night_shift.py b/src/sentry/tasks/seer/night_shift.py index 74407c0b4ba2fa..e9b901626d7b72 100644 --- a/src/sentry/tasks/seer/night_shift.py +++ b/src/sentry/tasks/seer/night_shift.py @@ -2,12 +2,19 @@ import logging from collections.abc import Sequence +from dataclasses import dataclass from datetime import timedelta import sentry_sdk from sentry import features, options +from sentry.constants import ObjectStatus +from sentry.models.group import Group, GroupStatus from sentry.models.organization import Organization, OrganizationStatus +from sentry.models.project import Project +from sentry.seer.autofix.constants import AutofixAutomationTuningSettings +from sentry.seer.autofix.utils import is_issue_category_eligible +from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import seer_tasks from sentry.utils.iterators import chunked @@ -17,6 +24,13 @@ NIGHT_SHIFT_DISPATCH_STEP_SECONDS = 37 NIGHT_SHIFT_SPREAD_DURATION = timedelta(hours=4) +NIGHT_SHIFT_MAX_CANDIDATES = 10 +NIGHT_SHIFT_ISSUES_PER_PROJECT = 50 + +# Weights for candidate scoring. Set to 0 to disable a signal. +WEIGHT_FIXABILITY = 1.0 +WEIGHT_SEVERITY = 0.0 +WEIGHT_TIMES_SEEN = 0.0 FEATURE_NAMES = [ "organizations:seer-night-shift", @@ -65,6 +79,28 @@ def schedule_night_shift() -> None: ) +@dataclass +class _ScoredCandidate: + """A candidate issue with raw signals for ranking.""" + + group_id: int + project_id: int + fixability: float + times_seen: int + severity: float + + @property + def score(self) -> float: + return ( + WEIGHT_FIXABILITY * self.fixability + + WEIGHT_SEVERITY * self.severity + + WEIGHT_TIMES_SEEN * min(self.times_seen / 1000.0, 1.0) + ) + + def __lt__(self, other: _ScoredCandidate) -> bool: + return self.score < other.score + + @instrumented_task( name="sentry.tasks.seer.night_shift.run_night_shift_for_org", namespace=seer_tasks, @@ -85,11 +121,37 @@ def run_night_shift_for_org(organization_id: int) -> None: } ) + eligible_projects = _get_eligible_projects(organization) + if not eligible_projects: + logger.info( + "night_shift.no_eligible_projects", + extra={ + "organization_id": organization_id, + "organization_slug": organization.slug, + }, + ) + return + + top_candidates = _fixability_score_strategy(eligible_projects) + logger.info( - "night_shift.org_dispatched", + "night_shift.candidates_selected", extra={ "organization_id": organization_id, "organization_slug": organization.slug, + "num_eligible_projects": len(eligible_projects), + "num_candidates": len(top_candidates), + "candidates": [ + { + "group_id": c.group_id, + "project_id": c.project_id, + "score": c.score, + "fixability": c.fixability, + "severity": c.severity, + "times_seen": c.times_seen, + } + for c in top_candidates + ], }, ) @@ -114,3 +176,57 @@ def _get_eligible_orgs_from_batch( return [] return eligible + + +def _get_eligible_projects(organization: Organization) -> list[Project]: + """Return active projects that have automation enabled and connected repos.""" + projects_with_repos = set( + SeerProjectRepository.objects.filter( + project__organization=organization, + project__status=ObjectStatus.ACTIVE, + ).values_list("project_id", flat=True) + ) + if not projects_with_repos: + return [] + + projects = Project.objects.filter(id__in=projects_with_repos) + return [ + p + for p in projects + if p.get_option("sentry:autofix_automation_tuning") != AutofixAutomationTuningSettings.OFF + ] + + +def _fixability_score_strategy( + projects: Sequence[Project], +) -> list[_ScoredCandidate]: + """ + Rank issues by existing fixability score with times_seen as tiebreaker. + Simple baseline — doesn't require any additional LLM calls. + """ + all_candidates: list[_ScoredCandidate] = [] + + for project in projects: + groups = Group.objects.filter( + project_id=project.id, + status=GroupStatus.UNRESOLVED, + seer_autofix_last_triggered__isnull=True, + seer_explorer_autofix_last_triggered__isnull=True, + ).order_by("-seer_fixability_score", "-times_seen")[:NIGHT_SHIFT_ISSUES_PER_PROJECT] + + for group in groups: + if not is_issue_category_eligible(group): + continue + + all_candidates.append( + _ScoredCandidate( + group_id=group.id, + project_id=project.id, + fixability=group.seer_fixability_score or 0.0, + times_seen=group.times_seen, + severity=(group.priority or 0) / 75.0, + ) + ) + + all_candidates.sort(reverse=True) + return all_candidates[:NIGHT_SHIFT_MAX_CANDIDATES] diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index b379f605b1f55a..c308b080dd37c3 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -1,6 +1,13 @@ from unittest.mock import patch +from django.utils import timezone + +from sentry.models.group import GroupStatus +from sentry.seer.autofix.constants import AutofixAutomationTuningSettings +from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.seer.night_shift import ( + _fixability_score_strategy, + _get_eligible_projects, run_night_shift_for_org, schedule_night_shift, ) @@ -64,16 +71,201 @@ def test_skips_orgs_with_hidden_ai(self) -> None: @django_db_all -class TestRunNightShiftForOrg(TestCase): - def test_logs_for_valid_org(self) -> None: +class TestGetEligibleProjects(TestCase): + def _connect_repo(self, project): + repo = self.create_repo(project=project, provider="github") + SeerProjectRepository.objects.create(project=project, repository=repo) + + def test_skips_projects_with_automation_off(self) -> None: org = self.create_organization() + project = self.create_project(organization=org) + project.update_option( + "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.OFF + ) + self._connect_repo(project) - with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: - run_night_shift_for_org(org.id) - mock_logger.info.assert_called_once() - assert mock_logger.info.call_args.args[0] == "night_shift.org_dispatched" + assert _get_eligible_projects(org) == [] + + def test_skips_projects_without_connected_repos(self) -> None: + org = self.create_organization() + self.create_project(organization=org) + assert _get_eligible_projects(org) == [] + + def test_returns_eligible_projects(self) -> None: + org = self.create_organization() + project = self.create_project(organization=org) + project.update_option( + "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM + ) + self._connect_repo(project) + + assert _get_eligible_projects(org) == [project] + + +@django_db_all +class TestRunNightShiftForOrg(TestCase): def test_nonexistent_org(self) -> None: with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: run_night_shift_for_org(999999999) mock_logger.info.assert_not_called() + + def test_no_eligible_projects(self) -> None: + org = self.create_organization() + + with ( + patch("sentry.tasks.seer.night_shift._get_eligible_projects", return_value=[]), + patch("sentry.tasks.seer.night_shift.logger") as mock_logger, + ): + run_night_shift_for_org(org.id) + mock_logger.info.assert_called_once() + assert mock_logger.info.call_args.args[0] == "night_shift.no_eligible_projects" + + def test_selects_candidates_by_fixability(self) -> None: + org = self.create_organization() + project = self.create_project(organization=org) + project.update_option( + "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM + ) + + # Create issues with different fixability scores + high_fix = self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.9, + times_seen=5, + ) + low_fix = self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.2, + times_seen=100, + ) + + with ( + patch( + "sentry.tasks.seer.night_shift._get_eligible_projects", + return_value=[project], + ), + patch("sentry.tasks.seer.night_shift.logger") as mock_logger, + ): + run_night_shift_for_org(org.id) + + call_extra = mock_logger.info.call_args.kwargs["extra"] + assert call_extra["num_candidates"] == 2 + candidates = call_extra["candidates"] + # Higher fixability should rank first + assert candidates[0]["group_id"] == high_fix.id + assert candidates[1]["group_id"] == low_fix.id + + def test_skips_already_triggered_issues(self) -> None: + org = self.create_organization() + project = self.create_project(organization=org) + + self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.9, + seer_autofix_last_triggered=timezone.now(), + ) + untriggered = self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.5, + ) + + with ( + patch( + "sentry.tasks.seer.night_shift._get_eligible_projects", + return_value=[project], + ), + patch("sentry.tasks.seer.night_shift.logger") as mock_logger, + ): + run_night_shift_for_org(org.id) + + call_extra = mock_logger.info.call_args.kwargs["extra"] + assert call_extra["num_candidates"] == 1 + assert call_extra["candidates"][0]["group_id"] == untriggered.id + + def test_global_ranking_across_projects(self) -> None: + org = self.create_organization() + project_a = self.create_project(organization=org) + project_b = self.create_project(organization=org) + + low_group = self.create_group( + project=project_a, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.3, + ) + high_group = self.create_group( + project=project_b, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.95, + ) + + with ( + patch( + "sentry.tasks.seer.night_shift._get_eligible_projects", + return_value=[project_a, project_b], + ), + patch("sentry.tasks.seer.night_shift.logger") as mock_logger, + ): + run_night_shift_for_org(org.id) + + candidates = mock_logger.info.call_args.kwargs["extra"]["candidates"] + assert candidates[0]["group_id"] == high_group.id + assert candidates[0]["project_id"] == project_b.id + assert candidates[1]["group_id"] == low_group.id + assert candidates[1]["project_id"] == project_a.id + + +@django_db_all +class TestFixabilityScoreStrategy(TestCase): + def test_ranks_by_fixability(self) -> None: + project = self.create_project() + high = self.create_group( + project=project, status=GroupStatus.UNRESOLVED, seer_fixability_score=0.9, times_seen=1 + ) + low = self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.2, + times_seen=500, + ) + + result = _fixability_score_strategy([project]) + + assert result[0].group_id == high.id + assert result[1].group_id == low.id + + def test_captures_raw_signals(self) -> None: + project = self.create_project() + self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.7, + times_seen=100, + priority=75, + ) + + result = _fixability_score_strategy([project]) + + assert len(result) == 1 + assert result[0].fixability == 0.7 + assert result[0].times_seen == 100 + assert result[0].severity == 1.0 + + def test_includes_issues_without_fixability_score(self) -> None: + project = self.create_project() + self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=None, + times_seen=50, + ) + + result = _fixability_score_strategy([project]) + + assert len(result) == 1 + assert result[0].fixability == 0.0 + assert result[0].times_seen == 50 From c4afe3fd999e4e09aa3c2529b5c924d92334c2a3 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Wed, 8 Apr 2026 16:46:07 -0400 Subject: [PATCH 2/5] limit query --- src/sentry/tasks/seer/night_shift.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sentry/tasks/seer/night_shift.py b/src/sentry/tasks/seer/night_shift.py index e9b901626d7b72..ceba836327007c 100644 --- a/src/sentry/tasks/seer/night_shift.py +++ b/src/sentry/tasks/seer/night_shift.py @@ -206,9 +206,9 @@ def _fixability_score_strategy( """ all_candidates: list[_ScoredCandidate] = [] - for project in projects: + for project_id_batch in chunked(projects, 100): groups = Group.objects.filter( - project_id=project.id, + project_id__in=[p.id for p in project_id_batch], status=GroupStatus.UNRESOLVED, seer_autofix_last_triggered__isnull=True, seer_explorer_autofix_last_triggered__isnull=True, @@ -221,7 +221,7 @@ def _fixability_score_strategy( all_candidates.append( _ScoredCandidate( group_id=group.id, - project_id=project.id, + project_id=group.project_id, fixability=group.seer_fixability_score or 0.0, times_seen=group.times_seen, severity=(group.priority or 0) / 75.0, From b96d174fb018cc69a08f0e2c5284e20ae35f1c74 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Wed, 8 Apr 2026 17:01:45 -0400 Subject: [PATCH 3/5] ref(seer): Address review feedback for night shift - Single query across all projects instead of N+1 per-project queries - Chunk project IDs in batches of 100 to avoid large IN clauses - Use PriorityLevel.HIGH instead of magic number 75 - Use SeerProjectRepository DB query instead of Seer API call Co-Authored-By: Claude Opus 4.6 --- src/sentry/tasks/seer/night_shift.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sentry/tasks/seer/night_shift.py b/src/sentry/tasks/seer/night_shift.py index ceba836327007c..0947853dbc5dd1 100644 --- a/src/sentry/tasks/seer/night_shift.py +++ b/src/sentry/tasks/seer/night_shift.py @@ -17,6 +17,7 @@ from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import seer_tasks +from sentry.types.group import PriorityLevel from sentry.utils.iterators import chunked from sentry.utils.query import RangeQuerySetWrapper @@ -224,7 +225,7 @@ def _fixability_score_strategy( project_id=group.project_id, fixability=group.seer_fixability_score or 0.0, times_seen=group.times_seen, - severity=(group.priority or 0) / 75.0, + severity=(group.priority or 0) / PriorityLevel.HIGH, ) ) From 4686afdf64120126289ae3cd012416379bfb522b Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Wed, 8 Apr 2026 17:21:35 -0400 Subject: [PATCH 4/5] test(seer): Remove _get_eligible_projects mocks from night shift tests Use real DB state (SeerProjectRepository rows + project options) instead of mocking _get_eligible_projects. Only logger and scheduler dispatch mocks remain. Co-Authored-By: Claude Opus 4.6 --- tests/sentry/tasks/seer/test_night_shift.py | 46 +++++++-------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index c308b080dd37c3..1d6ff6f4f4633f 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -105,6 +105,13 @@ def test_returns_eligible_projects(self) -> None: @django_db_all class TestRunNightShiftForOrg(TestCase): + def _make_eligible(self, project): + project.update_option( + "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM + ) + repo = self.create_repo(project=project, provider="github") + SeerProjectRepository.objects.create(project=project, repository=repo) + def test_nonexistent_org(self) -> None: with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: run_night_shift_for_org(999999999) @@ -112,11 +119,9 @@ def test_nonexistent_org(self) -> None: def test_no_eligible_projects(self) -> None: org = self.create_organization() + self.create_project(organization=org) - with ( - patch("sentry.tasks.seer.night_shift._get_eligible_projects", return_value=[]), - patch("sentry.tasks.seer.night_shift.logger") as mock_logger, - ): + with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: run_night_shift_for_org(org.id) mock_logger.info.assert_called_once() assert mock_logger.info.call_args.args[0] == "night_shift.no_eligible_projects" @@ -124,11 +129,8 @@ def test_no_eligible_projects(self) -> None: def test_selects_candidates_by_fixability(self) -> None: org = self.create_organization() project = self.create_project(organization=org) - project.update_option( - "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM - ) + self._make_eligible(project) - # Create issues with different fixability scores high_fix = self.create_group( project=project, status=GroupStatus.UNRESOLVED, @@ -142,25 +144,19 @@ def test_selects_candidates_by_fixability(self) -> None: times_seen=100, ) - with ( - patch( - "sentry.tasks.seer.night_shift._get_eligible_projects", - return_value=[project], - ), - patch("sentry.tasks.seer.night_shift.logger") as mock_logger, - ): + with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: run_night_shift_for_org(org.id) call_extra = mock_logger.info.call_args.kwargs["extra"] assert call_extra["num_candidates"] == 2 candidates = call_extra["candidates"] - # Higher fixability should rank first assert candidates[0]["group_id"] == high_fix.id assert candidates[1]["group_id"] == low_fix.id def test_skips_already_triggered_issues(self) -> None: org = self.create_organization() project = self.create_project(organization=org) + self._make_eligible(project) self.create_group( project=project, @@ -174,13 +170,7 @@ def test_skips_already_triggered_issues(self) -> None: seer_fixability_score=0.5, ) - with ( - patch( - "sentry.tasks.seer.night_shift._get_eligible_projects", - return_value=[project], - ), - patch("sentry.tasks.seer.night_shift.logger") as mock_logger, - ): + with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: run_night_shift_for_org(org.id) call_extra = mock_logger.info.call_args.kwargs["extra"] @@ -191,6 +181,8 @@ def test_global_ranking_across_projects(self) -> None: org = self.create_organization() project_a = self.create_project(organization=org) project_b = self.create_project(organization=org) + self._make_eligible(project_a) + self._make_eligible(project_b) low_group = self.create_group( project=project_a, @@ -203,13 +195,7 @@ def test_global_ranking_across_projects(self) -> None: seer_fixability_score=0.95, ) - with ( - patch( - "sentry.tasks.seer.night_shift._get_eligible_projects", - return_value=[project_a, project_b], - ), - patch("sentry.tasks.seer.night_shift.logger") as mock_logger, - ): + with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: run_night_shift_for_org(org.id) candidates = mock_logger.info.call_args.kwargs["extra"]["candidates"] From fd3708fe8d7d2ac359ac20a52e67f85928b4dc08 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Wed, 8 Apr 2026 17:36:33 -0400 Subject: [PATCH 5/5] ref(seer): Address night shift review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix NULL fixability scores sorting first in DESC ordering by using F().desc(nulls_last=True) - Rename NIGHT_SHIFT_ISSUES_PER_PROJECT to NIGHT_SHIFT_ISSUE_FETCH_LIMIT to reflect it's a per-batch limit, not per-project - Increase fetch limit from 50 to 100 - Consolidate tests: 15 → 10 with same coverage Co-Authored-By: Claude Opus 4.6 --- src/sentry/tasks/seer/night_shift.py | 8 +- tests/sentry/tasks/seer/test_night_shift.py | 122 +++++++------------- 2 files changed, 47 insertions(+), 83 deletions(-) diff --git a/src/sentry/tasks/seer/night_shift.py b/src/sentry/tasks/seer/night_shift.py index 0947853dbc5dd1..fad1463296697f 100644 --- a/src/sentry/tasks/seer/night_shift.py +++ b/src/sentry/tasks/seer/night_shift.py @@ -6,6 +6,7 @@ from datetime import timedelta import sentry_sdk +from django.db.models import F from sentry import features, options from sentry.constants import ObjectStatus @@ -26,7 +27,7 @@ NIGHT_SHIFT_DISPATCH_STEP_SECONDS = 37 NIGHT_SHIFT_SPREAD_DURATION = timedelta(hours=4) NIGHT_SHIFT_MAX_CANDIDATES = 10 -NIGHT_SHIFT_ISSUES_PER_PROJECT = 50 +NIGHT_SHIFT_ISSUE_FETCH_LIMIT = 100 # Weights for candidate scoring. Set to 0 to disable a signal. WEIGHT_FIXABILITY = 1.0 @@ -213,7 +214,10 @@ def _fixability_score_strategy( status=GroupStatus.UNRESOLVED, seer_autofix_last_triggered__isnull=True, seer_explorer_autofix_last_triggered__isnull=True, - ).order_by("-seer_fixability_score", "-times_seen")[:NIGHT_SHIFT_ISSUES_PER_PROJECT] + ).order_by( + F("seer_fixability_score").desc(nulls_last=True), + F("times_seen").desc(), + )[:NIGHT_SHIFT_ISSUE_FETCH_LIMIT] for group in groups: if not is_issue_category_eligible(group): diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index 1d6ff6f4f4633f..d014767612e653 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -72,35 +72,27 @@ def test_skips_orgs_with_hidden_ai(self) -> None: @django_db_all class TestGetEligibleProjects(TestCase): - def _connect_repo(self, project): - repo = self.create_repo(project=project, provider="github") - SeerProjectRepository.objects.create(project=project, repository=repo) - - def test_skips_projects_with_automation_off(self) -> None: + def test_filters_by_automation_and_repos(self) -> None: org = self.create_organization() - project = self.create_project(organization=org) - project.update_option( - "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.OFF + + # Eligible: automation on + connected repo + eligible = self.create_project(organization=org) + eligible.update_option( + "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM ) - self._connect_repo(project) + repo = self.create_repo(project=eligible, provider="github") + SeerProjectRepository.objects.create(project=eligible, repository=repo) - assert _get_eligible_projects(org) == [] + # Automation off (even with repo) + off = self.create_project(organization=org) + off.update_option("sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.OFF) + repo2 = self.create_repo(project=off, provider="github") + SeerProjectRepository.objects.create(project=off, repository=repo2) - def test_skips_projects_without_connected_repos(self) -> None: - org = self.create_organization() + # No connected repo self.create_project(organization=org) - assert _get_eligible_projects(org) == [] - - def test_returns_eligible_projects(self) -> None: - org = self.create_organization() - project = self.create_project(organization=org) - project.update_option( - "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM - ) - self._connect_repo(project) - - assert _get_eligible_projects(org) == [project] + assert _get_eligible_projects(org) == [eligible] @django_db_all @@ -126,7 +118,7 @@ def test_no_eligible_projects(self) -> None: mock_logger.info.assert_called_once() assert mock_logger.info.call_args.args[0] == "night_shift.no_eligible_projects" - def test_selects_candidates_by_fixability(self) -> None: + def test_selects_candidates_and_skips_triggered(self) -> None: org = self.create_organization() project = self.create_project(organization=org) self._make_eligible(project) @@ -143,39 +135,22 @@ def test_selects_candidates_by_fixability(self) -> None: seer_fixability_score=0.2, times_seen=100, ) - - with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: - run_night_shift_for_org(org.id) - - call_extra = mock_logger.info.call_args.kwargs["extra"] - assert call_extra["num_candidates"] == 2 - candidates = call_extra["candidates"] - assert candidates[0]["group_id"] == high_fix.id - assert candidates[1]["group_id"] == low_fix.id - - def test_skips_already_triggered_issues(self) -> None: - org = self.create_organization() - project = self.create_project(organization=org) - self._make_eligible(project) - + # Already triggered — should be excluded self.create_group( project=project, status=GroupStatus.UNRESOLVED, - seer_fixability_score=0.9, + seer_fixability_score=0.95, seer_autofix_last_triggered=timezone.now(), ) - untriggered = self.create_group( - project=project, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=0.5, - ) with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: run_night_shift_for_org(org.id) call_extra = mock_logger.info.call_args.kwargs["extra"] - assert call_extra["num_candidates"] == 1 - assert call_extra["candidates"][0]["group_id"] == untriggered.id + assert call_extra["num_candidates"] == 2 + candidates = call_extra["candidates"] + assert candidates[0]["group_id"] == high_fix.id + assert candidates[1]["group_id"] == low_fix.id def test_global_ranking_across_projects(self) -> None: org = self.create_organization() @@ -207,10 +182,15 @@ def test_global_ranking_across_projects(self) -> None: @django_db_all class TestFixabilityScoreStrategy(TestCase): - def test_ranks_by_fixability(self) -> None: + @patch("sentry.tasks.seer.night_shift.NIGHT_SHIFT_ISSUE_FETCH_LIMIT", 3) + def test_ranks_and_captures_signals(self) -> None: project = self.create_project() high = self.create_group( - project=project, status=GroupStatus.UNRESOLVED, seer_fixability_score=0.9, times_seen=1 + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.9, + times_seen=5, + priority=75, ) low = self.create_group( project=project, @@ -218,40 +198,20 @@ def test_ranks_by_fixability(self) -> None: seer_fixability_score=0.2, times_seen=500, ) + # NULL-scored issues should sort after scored ones even with a tight DB limit. + # Without nulls_last these would fill the limit and exclude scored issues. + for _ in range(3): + self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=None, + times_seen=100, + ) result = _fixability_score_strategy([project]) assert result[0].group_id == high.id - assert result[1].group_id == low.id - - def test_captures_raw_signals(self) -> None: - project = self.create_project() - self.create_group( - project=project, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=0.7, - times_seen=100, - priority=75, - ) - - result = _fixability_score_strategy([project]) - - assert len(result) == 1 - assert result[0].fixability == 0.7 - assert result[0].times_seen == 100 + assert result[0].fixability == 0.9 + assert result[0].times_seen == 5 assert result[0].severity == 1.0 - - def test_includes_issues_without_fixability_score(self) -> None: - project = self.create_project() - self.create_group( - project=project, - status=GroupStatus.UNRESOLVED, - seer_fixability_score=None, - times_seen=50, - ) - - result = _fixability_score_strategy([project]) - - assert len(result) == 1 - assert result[0].fixability == 0.0 - assert result[0].times_seen == 50 + assert result[1].group_id == low.id