diff --git a/oioioi/default_settings.py b/oioioi/default_settings.py index cf5eb407b..63bfdb9df 100755 --- a/oioioi/default_settings.py +++ b/oioioi/default_settings.py @@ -658,6 +658,13 @@ 'level': 'DEBUG', 'propagate': True, }, + # Errors in recalculation of rankings are rare, but not trivial to + # notice. + 'oioioi.rankings.models.recalculation': { + 'handlers': ['mail_admins'], + 'level': 'DEBUG', + 'propagate': True, + }, 'celery': { 'handlers': ['console', 'emit_notification'], 'level': 'DEBUG', @@ -774,8 +781,10 @@ } # Ranking +RANKINGSD_CONCURRENCY = 1 # Number of rankingsd instances to start. RANKINGSD_POLLING_INTERVAL = 0.5 # seconds RANKING_COOLDOWN_FACTOR = 2 # seconds +RANKING_ERROR_COOLDOWN = 300 # seconds; Don't overwhelm the admins' mailbox :). RANKING_MIN_COOLDOWN = 5 # seconds RANKING_MAX_COOLDOWN = 100 # seconds @@ -900,4 +909,4 @@ STATICFILES_DIRS = [ BASE_DIR / "dist_webpack", BASE_DIR / "node_modules" -] \ No newline at end of file +] diff --git a/oioioi/deployment/settings.py.template b/oioioi/deployment/settings.py.template index 0a4f91bb7..e79f8c662 100755 --- a/oioioi/deployment/settings.py.template +++ b/oioioi/deployment/settings.py.template @@ -495,6 +495,7 @@ ZEUS_INSTANCES = { # } # Ranking +# RANKINGSD_CONCURRENCY = 1 # Number of rankingsd instances to start. # RANKINGSD_POLLING_INTERVAL = 0.5 # seconds # RANKING_COOLDOWN_FACTOR = 2 # seconds # RANKING_MIN_COOLDOWN = 5 # seconds @@ -608,4 +609,4 @@ ALLOW_ONLY_GET_FOR_SU_CONTEST_ADMINS = True # # REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] += ( # 'oauth2_provider.contrib.rest_framework.OAuth2Authentication', -# ) \ No newline at end of file +# ) diff --git a/oioioi/deployment/supervisord.conf.template b/oioioi/deployment/supervisord.conf.template index fe7b9f375..ecd11a775 100644 --- a/oioioi/deployment/supervisord.conf.template +++ b/oioioi/deployment/supervisord.conf.template @@ -25,9 +25,11 @@ stderr_logfile={{ PROJECT_DIR }}/logs/runserver-err.log [program:rankingsd] command={{ PYTHON }} {{ PROJECT_DIR }}/manage.py rankingsd startretries=0 +numprocs={{ settings.RANKINGSD_CONCURRENCY }} redirect_stderr=false -stdout_logfile={{ PROJECT_DIR }}/logs/rankingsd.log -stderr_logfile={{ PROJECT_DIR }}/logs/rankingsd-err.log +process_name=rankingsd_%(process_num)d +stdout_logfile={{ PROJECT_DIR }}/logs/rankingsd_%(process_num)d.log +stderr_logfile={{ PROJECT_DIR }}/logs/rankingsd_%(process_num)d-err.log [program:mailnotifyd] command={{ PYTHON }} {{ PROJECT_DIR }}/manage.py mailnotifyd diff --git a/oioioi/pa/controllers.py b/oioioi/pa/controllers.py index 2fd7441fc..9bc7aaa70 100644 --- a/oioioi/pa/controllers.py +++ b/oioioi/pa/controllers.py @@ -251,6 +251,19 @@ def available_rankings(self, request): rankings.append((str(round.id), round.name)) return rankings + def partial_keys_for_probleminstance(self, pi): + partial_keys = [] + division = pi.paprobleminstancedata.division + if division == "NONE": + if pi.round is not None and pi.round.is_trial: + partial_keys.append(str(pi.round_id)) + else: + # This logic works for PADivCRankingController too. + partial_keys.append(A_PLUS_B_RANKING_KEY) + if division != "A": + partial_keys.append(B_RANKING_KEY) + return partial_keys + def _filter_pis_for_ranking(self, partial_key, queryset): if partial_key == A_PLUS_B_RANKING_KEY: return queryset.filter(paprobleminstancedata__division__in=["A", "B"]) diff --git a/oioioi/pa/tests.py b/oioioi/pa/tests.py index 6ff8b729e..7a0dc9d19 100644 --- a/oioioi/pa/tests.py +++ b/oioioi/pa/tests.py @@ -13,6 +13,7 @@ from oioioi.contests.models import ( Contest, ProblemInstance, + Round, Submission, UserResultForProblem, ) @@ -23,6 +24,7 @@ from oioioi.pa.score import PAScore, ScoreDistribution from oioioi.participants.models import Participant, TermsAcceptedPhrase from oioioi.problems.models import Problem +from oioioi.rankings.models import Ranking class TestPAScore(TestCase): @@ -178,6 +180,48 @@ def check_visibility(good_keys, response): response = self.client.get(self._ranking_url(3)) check_visibility(["NONE"], response) + def test_probleminstance_invalidation(self): + contest = Contest.objects.get() + + rc = contest.controller.ranking_controller() + trial_r = Round.objects.get(id=3) + self.assertTrue(trial_r.is_trial) + a_pi = ProblemInstance.objects.get(id=1) + b_pi = ProblemInstance.objects.get(id=3) + trial_pi = ProblemInstance.objects.get(id=5) + + def make_ranking_for_key(perm, partial_key): + key = perm + "#" + partial_key + return Ranking.objects.get_or_create(contest=contest, key=key, needs_recalculation=False)[0] + + ab_ranking = make_ranking_for_key("admin", A_PLUS_B_RANKING_KEY) + b_ranking = make_ranking_for_key("admin", B_RANKING_KEY) + trial_ranking = make_ranking_for_key("admin", str(trial_r.id)) + + rankings = (b_ranking, ab_ranking, trial_ranking) + + def uninvalidate(): + for r in rankings: + r.needs_recalculation = False + r.save() + self.assertTrue(r.is_up_to_date()) + + def test_pi_invalidation(pi, invalid_mask): + uninvalidate() + rc.invalidate_pi(pi) + i = 0 + for r in rankings: + r.refresh_from_db() + self.assertEqual(r.is_up_to_date(), 0 == invalid_mask[i]) + i += 1 + + test_pi_invalidation(a_pi, (0, 1, 0)) + test_pi_invalidation(b_pi, (1, 1, 0)) + test_pi_invalidation(trial_pi, (0, 0, 1)) + trial_pi.round = None + trial_pi.save() + test_pi_invalidation(trial_pi, (0, 0, 0)) + def test_no_zero_scores_in_ranking(self): self.assertTrue(self.client.login(username="test_user")) with fake_time(datetime(2013, 1, 1, 0, 0, tzinfo=UTC)): diff --git a/oioioi/rankings/controllers.py b/oioioi/rankings/controllers.py index 9f6b43cd2..1c6159e42 100644 --- a/oioioi/rankings/controllers.py +++ b/oioioi/rankings/controllers.py @@ -33,7 +33,7 @@ def ranking_controller(self): def update_user_results(self, user, problem_instance, *args, **kwargs): super().update_user_results(user, problem_instance, *args, **kwargs) - Ranking.invalidate_contest(problem_instance.round.contest) + self.ranking_controller().invalidate_pi(problem_instance) ContestController.mix_in(RankingMixinForContestController) @@ -55,26 +55,41 @@ class RankingController(RegisteredSubclassesBase, ObjectWithMixins): modules_with_subclasses = ["controllers"] abstract = True + PERMISSION_LEVELS = [ + "admin", + "observer", + "regular", + ] PERMISSION_CHECKERS = [ lambda request: "admin" if is_contest_basicadmin(request) else None, lambda request: "observer" if is_contest_observer(request) else None, lambda request: "regular", ] + def construct_full_key(self, perm_level, partial_key): + return perm_level + "#" + partial_key + def get_partial_key(self, key): """Extracts partial key from a full key.""" return key.split("#")[1] def replace_partial_key(self, key, new_partial): """Replaces partial key in a full key""" - return key.split("#")[0] + "#" + new_partial + return self.construct_full_key(self._key_permission(key), new_partial) def get_full_key(self, request, partial_key): """Returns a full key associated with request and partial_key""" for checker in self.PERMISSION_CHECKERS: res = checker(request) if res is not None: - return res + "#" + partial_key + return self.construct_full_key(res, partial_key) + + def construct_all_full_keys(self, partial_keys): + fulls = [] + for perm in self.PERMISSION_LEVELS: + for partial in partial_keys: + fulls.append(self.construct_full_key(perm, partial)) + return fulls def _key_permission(self, key): """Returns a permission level associated with given full key""" @@ -228,6 +243,23 @@ def available_rankings(self, request): return rankings[:1] return rankings + # Rankings with different partial key logic need must override this + # or invalidate_pi accordingly. As a last resort, the all rankings + # for the given contest may be invalidated. + def partial_keys_for_probleminstance(self, pi): + return [CONTEST_RANKING_KEY, str(pi.round_id)] + + def keys_for_probleminstance(self, pi): + return self.construct_all_full_keys(self.partial_keys_for_probleminstance(pi)) + + def invalidate_pi(self, pi): + Ranking.invalidate_queryset( + Ranking.objects.filter( + contest_id=pi.contest_id, + key__in=self.keys_for_probleminstance(pi), + ) + ) + def can_search_for_users(self): return True diff --git a/oioioi/rankings/models.py b/oioioi/rankings/models.py index dc8fe2174..0dedb5921 100644 --- a/oioioi/rankings/models.py +++ b/oioioi/rankings/models.py @@ -1,3 +1,4 @@ +import logging import pickle from datetime import timedelta # pylint: disable=E0611 @@ -140,7 +141,16 @@ def clamp(minimum, x, maximum): @transaction.atomic def choose_for_recalculation(): now = timezone.now() - r = Ranking.objects.filter(needs_recalculation=True, cooldown_date__lt=now).order_by("last_recalculation_date").select_for_update().first() + r = ( + Ranking.objects.filter( + needs_recalculation=True, + cooldown_date__lt=now, + recalc_in_progress=None, + ) + .order_by("last_recalculation_date") + .select_for_update() + .first() + ) if r is None: return None cooldown_duration = clamp( @@ -166,17 +176,21 @@ def save_pages(ranking, pages_list): @transaction.atomic -def save_recalc_results(recalc, date_before, date_after, serialized, pages_list): +def save_recalc_results(recalc, date_before, date_after, serialized, pages_list, cooldown_date): try: r = Ranking.objects.filter(recalc_in_progress=recalc).select_for_update().get() except Ranking.DoesNotExist: return - r.serialized_data = pickle.dumps(serialized) - save_pages(r, pages_list) + if serialized is not None: + assert pages_list is not None + r.serialized_data = pickle.dumps(serialized) + save_pages(r, pages_list) r.last_recalculation_date = date_before r.last_recalculation_duration = date_after - date_before old_recalc = r.recalc_in_progress r.recalc_in_progress = None + if cooldown_date is not None: + r.cooldown_date = cooldown_date r.save() old_recalc.delete() @@ -188,9 +202,19 @@ def recalculate(recalc): except Ranking.DoesNotExist: return ranking_controller = r.controller() - serialized, pages_list = ranking_controller.build_ranking(r.key) + try: + serialized, pages_list = ranking_controller.build_ranking(r.key) + cooldown_date = None + except Exception as e: + if getattr(settings, "MOCK_RANKINGSD", False): + raise + logger = logging.getLogger(__name__ + ".recalculation") + logger.exception("An error occurred while recalculating ranking", e) + cooldown_duration = timedelta(seconds=settings.RANKINGS_ERROR_COOLDOWN) + cooldown_date = timezone.now() + cooldown_duration + serialized, pages_list = (None, None) date_after = timezone.now() - save_recalc_results(recalc, date_before, date_after, serialized, pages_list) + save_recalc_results(recalc, date_before, date_after, serialized, pages_list, cooldown_date) class RankingMessage(PublicMessage): diff --git a/oioioi/rankings/templates/rankings/ranking_view.html b/oioioi/rankings/templates/rankings/ranking_view.html index c3aa18510..ffdca33ee 100644 --- a/oioioi/rankings/templates/rankings/ranking_view.html +++ b/oioioi/rankings/templates/rankings/ranking_view.html @@ -23,6 +23,11 @@ {% trans "Regenerate ranking" %} + + + {% trans "Regenerate all rankings" %} + {% endif %} {% if form and user.is_authenticated and not is_admin %} [a-z0-9_-]+)/invalidate_contest/$", + views.ranking_invalidate_view, + {"invalidate_whole_contest": True}, + name="ranking_invalidate_contest", + ), ] diff --git a/oioioi/rankings/views.py b/oioioi/rankings/views.py index 553034053..90d758919 100644 --- a/oioioi/rankings/views.py +++ b/oioioi/rankings/views.py @@ -127,11 +127,14 @@ def ranking_csv_view(request, key): @enforce_condition(contest_exists & is_contest_basicadmin) @require_POST -def ranking_invalidate_view(request, key): - rcontroller = request.contest.controller.ranking_controller() - full_key = rcontroller.get_full_key(request, key) - ranking = Ranking.objects.filter(key=full_key) - Ranking.invalidate_queryset(ranking) +def ranking_invalidate_view(request, key, invalidate_whole_contest=False): + if invalidate_whole_contest: + Ranking.invalidate_contest(request.contest) + else: + rcontroller = request.contest.controller.ranking_controller() + full_keys = rcontroller.construct_all_full_keys([key]) + rankings = Ranking.objects.filter(key__in=full_keys, contest=request.contest) + Ranking.invalidate_queryset(rankings) return redirect("ranking", key=key) diff --git a/oioioi/usergroups/controllers.py b/oioioi/usergroups/controllers.py index 27346fbc4..064e33b58 100644 --- a/oioioi/usergroups/controllers.py +++ b/oioioi/usergroups/controllers.py @@ -3,6 +3,7 @@ from oioioi.contests.utils import is_contest_basicadmin, is_contest_observer from oioioi.rankings.controllers import CONTEST_RANKING_KEY, DefaultRankingController +from oioioi.rankings.models import Ranking from oioioi.teachers.controllers import TeacherRegistrationController from oioioi.usergroups.models import UserGroup, UserGroupRanking @@ -60,6 +61,16 @@ def available_rankings(self, request): rankings.append((user_group_ranking_id(user_group.id), user_group.name)) return rankings + def partial_keys_for_probleminstance(self, pi): + partial_keys = super().partial_keys_for_probleminstance(pi) + keys = Ranking.objects.filter( + contest_id=pi.contest_id, + # Somewhat hacky. + key__contains=self.construct_full_key("", USER_GROUP_RANKING_PREFIX), + ).values_list("key", flat=True) + partial_keys.extend(self.get_partial_key(key) for key in keys) + return partial_keys + def filter_users_for_ranking(self, key, queryset): queryset = super().filter_users_for_ranking(key, queryset) partial_key = self.get_partial_key(key) diff --git a/oioioi/usergroups/tests.py b/oioioi/usergroups/tests.py index a7e5efc39..eb243b9dc 100644 --- a/oioioi/usergroups/tests.py +++ b/oioioi/usergroups/tests.py @@ -6,7 +6,7 @@ from django.urls.exceptions import NoReverseMatch from oioioi.base.tests import TestCase -from oioioi.contests.models import Contest +from oioioi.contests.models import Contest, ProblemInstance from oioioi.contests.tests.utils import make_user_contest_admin from oioioi.participants.models import Participant from oioioi.rankings.models import Ranking @@ -583,6 +583,38 @@ def test_ranking_view(self): response = self.client.get(url) self.assertEqual(response.status_code, 404) + def test_probleminstance_invalidation(self): + contest = Contest.objects.get(id="c") + + rc = contest.controller.ranking_controller() + pi = ProblemInstance.objects.filter(contest=contest).first() + + def make_ranking_for_key(perm, partial_key): + key = perm + "#" + partial_key + return Ranking.objects.get_or_create(contest=contest, key=key, needs_recalculation=False)[0] + + usergroup_ranking = make_ranking_for_key("admin", "g2001") + contest_ranking = make_ranking_for_key("admin", "c") + self.assertTrue(contest_ranking.is_up_to_date()) + self.assertTrue(usergroup_ranking.is_up_to_date()) + + round = pi.round + self.assertIsNotNone(round) + pi.round = None + pi.save() + rc.invalidate_pi(pi) + self.assertTrue(contest_ranking.is_up_to_date()) + self.assertTrue(usergroup_ranking.is_up_to_date()) + + pi.round = round + pi.save() + rc.invalidate_pi(pi) + contest_ranking.refresh_from_db() + usergroup_ranking.refresh_from_db() + self.assertFalse(contest_ranking.is_up_to_date()) + self.assertFalse(usergroup_ranking.is_up_to_date()) + + class RemoveUserGroupRankingsTest(TestCase): fixtures = [