Parallellize rankings' recalculation and optimize invalidation

oioioi/default_settings.py

@@ -658,6 +658,13 @@
             'level': 'DEBUG',
             'propagate': True,
         },
+        # Errors in recalculation of rankings are rare, but not trivial to
+        # notice.
+        'oioioi.rankings.models.recalculation': {
+            'handlers': ['mail_admins'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
         'celery': {
             'handlers': ['console', 'emit_notification'],
             'level': 'DEBUG',
@@ -774,8 +781,10 @@
 }
 
 # Ranking
+RANKINGSD_CONCURRENCY = 1 # Number of rankingsd instances to start.
 RANKINGSD_POLLING_INTERVAL = 0.5  # seconds
 RANKING_COOLDOWN_FACTOR = 2  # seconds
+RANKING_ERROR_COOLDOWN = 300  # seconds; Don't overwhelm the admins' mailbox :).
 RANKING_MIN_COOLDOWN = 5  # seconds
 RANKING_MAX_COOLDOWN = 100  # seconds
 
@@ -900,4 +909,4 @@
 STATICFILES_DIRS = [
     BASE_DIR / "dist_webpack",
     BASE_DIR / "node_modules"
-]
+]

oioioi/deployment/settings.py.template

@@ -495,6 +495,7 @@ ZEUS_INSTANCES = {
 # }
 
 # Ranking
+# RANKINGSD_CONCURRENCY = 1  # Number of rankingsd instances to start.
 # RANKINGSD_POLLING_INTERVAL = 0.5  # seconds
 # RANKING_COOLDOWN_FACTOR = 2  # seconds
 # RANKING_MIN_COOLDOWN = 5  # seconds
@@ -608,4 +609,4 @@ ALLOW_ONLY_GET_FOR_SU_CONTEST_ADMINS = True
 #
 # REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] += (
 #    'oauth2_provider.contrib.rest_framework.OAuth2Authentication',
-# )
+# )

oioioi/deployment/supervisord.conf.template

@@ -25,9 +25,11 @@ stderr_logfile={{ PROJECT_DIR }}/logs/runserver-err.log
 [program:rankingsd]
 command={{ PYTHON }} {{ PROJECT_DIR }}/manage.py rankingsd
 startretries=0
+numprocs={{ settings.RANKINGSD_CONCURRENCY }}
 redirect_stderr=false
-stdout_logfile={{ PROJECT_DIR }}/logs/rankingsd.log
-stderr_logfile={{ PROJECT_DIR }}/logs/rankingsd-err.log
+process_name=rankingsd_%(process_num)d
+stdout_logfile={{ PROJECT_DIR }}/logs/rankingsd_%(process_num)d.log
+stderr_logfile={{ PROJECT_DIR }}/logs/rankingsd_%(process_num)d-err.log
 
 [program:mailnotifyd]
 command={{ PYTHON }} {{ PROJECT_DIR }}/manage.py mailnotifyd

oioioi/pa/controllers.py

@@ -251,6 +251,19 @@ def available_rankings(self, request):
                 rankings.append((str(round.id), round.name))
         return rankings
 
+    def partial_keys_for_probleminstance(self, pi):
+        partial_keys = []
+        division = pi.paprobleminstancedata.division
+        if division == "NONE":
+            if pi.round is not None and pi.round.is_trial:
+                partial_keys.append(str(pi.round_id))
+        else:
+            # This logic works for PADivCRankingController too.
+            partial_keys.append(A_PLUS_B_RANKING_KEY)
+            if division != "A":
+                partial_keys.append(B_RANKING_KEY)
+        return partial_keys
+
     def _filter_pis_for_ranking(self, partial_key, queryset):
         if partial_key == A_PLUS_B_RANKING_KEY:
             return queryset.filter(paprobleminstancedata__division__in=["A", "B"])

oioioi/pa/tests.py

@@ -13,6 +13,7 @@
 from oioioi.contests.models import (
     Contest,
     ProblemInstance,
+    Round,
     Submission,
     UserResultForProblem,
 )
@@ -23,6 +24,7 @@
 from oioioi.pa.score import PAScore, ScoreDistribution
 from oioioi.participants.models import Participant, TermsAcceptedPhrase
 from oioioi.problems.models import Problem
+from oioioi.rankings.models import Ranking
 
 
 class TestPAScore(TestCase):
@@ -178,6 +180,48 @@ def check_visibility(good_keys, response):
             response = self.client.get(self._ranking_url(3))
             check_visibility(["NONE"], response)
 
+    def test_probleminstance_invalidation(self):
+        contest = Contest.objects.get()
+
+        rc = contest.controller.ranking_controller()
+        trial_r = Round.objects.get(id=3)
+        self.assertTrue(trial_r.is_trial)
+        a_pi = ProblemInstance.objects.get(id=1)
+        b_pi = ProblemInstance.objects.get(id=3)
+        trial_pi = ProblemInstance.objects.get(id=5)
+
+        def make_ranking_for_key(perm, partial_key):
+            key = perm + "#" + partial_key
+            return Ranking.objects.get_or_create(contest=contest, key=key, needs_recalculation=False)[0]
+
+        ab_ranking = make_ranking_for_key("admin", A_PLUS_B_RANKING_KEY)
+        b_ranking = make_ranking_for_key("admin", B_RANKING_KEY)
+        trial_ranking = make_ranking_for_key("admin", str(trial_r.id))
+
+        rankings = (b_ranking, ab_ranking, trial_ranking)
+
+        def uninvalidate():
+            for r in rankings:
+                r.needs_recalculation = False
+                r.save()
+                self.assertTrue(r.is_up_to_date())
+
+        def test_pi_invalidation(pi, invalid_mask):
+            uninvalidate()
+            rc.invalidate_pi(pi)
+            i = 0
+            for r in rankings:
+                r.refresh_from_db()
+                self.assertEqual(r.is_up_to_date(), 0 == invalid_mask[i])
+                i += 1
+
+        test_pi_invalidation(a_pi, (0, 1, 0))
+        test_pi_invalidation(b_pi, (1, 1, 0))
+        test_pi_invalidation(trial_pi, (0, 0, 1))
+        trial_pi.round = None
+        trial_pi.save()
+        test_pi_invalidation(trial_pi, (0, 0, 0))
+
     def test_no_zero_scores_in_ranking(self):
         self.assertTrue(self.client.login(username="test_user"))
         with fake_time(datetime(2013, 1, 1, 0, 0, tzinfo=UTC)):

oioioi/rankings/controllers.py

@@ -33,7 +33,7 @@ def ranking_controller(self):
 
     def update_user_results(self, user, problem_instance, *args, **kwargs):
         super().update_user_results(user, problem_instance, *args, **kwargs)
-        Ranking.invalidate_contest(problem_instance.round.contest)
+        self.ranking_controller().invalidate_pi(problem_instance)
 
 
 ContestController.mix_in(RankingMixinForContestController)
@@ -55,26 +55,41 @@ class RankingController(RegisteredSubclassesBase, ObjectWithMixins):
 
     modules_with_subclasses = ["controllers"]
     abstract = True
+    PERMISSION_LEVELS = [
+        "admin",
+        "observer",
+        "regular",
+    ]
     PERMISSION_CHECKERS = [
         lambda request: "admin" if is_contest_basicadmin(request) else None,
         lambda request: "observer" if is_contest_observer(request) else None,
         lambda request: "regular",
     ]
 
+    def construct_full_key(self, perm_level, partial_key):
+        return perm_level + "#" + partial_key
+
     def get_partial_key(self, key):
         """Extracts partial key from a full key."""
         return key.split("#")[1]
 
     def replace_partial_key(self, key, new_partial):
         """Replaces partial key in a full key"""
-        return key.split("#")[0] + "#" + new_partial
+        return self.construct_full_key(self._key_permission(key), new_partial)
 
     def get_full_key(self, request, partial_key):
         """Returns a full key associated with request and partial_key"""
         for checker in self.PERMISSION_CHECKERS:
             res = checker(request)
             if res is not None:
-                return res + "#" + partial_key
+                return self.construct_full_key(res, partial_key)
+
+    def construct_all_full_keys(self, partial_keys):
+        fulls = []
+        for perm in self.PERMISSION_LEVELS:
+            for partial in partial_keys:
+                fulls.append(self.construct_full_key(perm, partial))
+        return fulls
 
     def _key_permission(self, key):
         """Returns a permission level associated with given full key"""
@@ -228,6 +243,23 @@ def available_rankings(self, request):
             return rankings[:1]
         return rankings
 
+    # Rankings with different partial key logic need must override this
+    # or invalidate_pi accordingly. As a last resort, the all rankings
+    # for the given contest may be invalidated.
+    def partial_keys_for_probleminstance(self, pi):
+        return [CONTEST_RANKING_KEY, str(pi.round_id)]
+
+    def keys_for_probleminstance(self, pi):
+        return self.construct_all_full_keys(self.partial_keys_for_probleminstance(pi))
+
+    def invalidate_pi(self, pi):
+        Ranking.invalidate_queryset(
+            Ranking.objects.filter(
+                contest_id=pi.contest_id,
+                key__in=self.keys_for_probleminstance(pi),
+            )
+        )
+
     def can_search_for_users(self):
         return True
 

oioioi/rankings/models.py

@@ -1,3 +1,4 @@
+import logging
 import pickle
 from datetime import timedelta  # pylint: disable=E0611
 
@@ -140,7 +141,16 @@ def clamp(minimum, x, maximum):
 @transaction.atomic
 def choose_for_recalculation():
     now = timezone.now()
-    r = Ranking.objects.filter(needs_recalculation=True, cooldown_date__lt=now).order_by("last_recalculation_date").select_for_update().first()
+    r = (
+        Ranking.objects.filter(
+            needs_recalculation=True,
+            cooldown_date__lt=now,
+            recalc_in_progress=None,
+        )
+        .order_by("last_recalculation_date")
+        .select_for_update()
+        .first()
+    )
     if r is None:
         return None
     cooldown_duration = clamp(
@@ -166,17 +176,21 @@ def save_pages(ranking, pages_list):
 
 
 @transaction.atomic
-def save_recalc_results(recalc, date_before, date_after, serialized, pages_list):
+def save_recalc_results(recalc, date_before, date_after, serialized, pages_list, cooldown_date):
     try:
         r = Ranking.objects.filter(recalc_in_progress=recalc).select_for_update().get()
     except Ranking.DoesNotExist:
         return
-    r.serialized_data = pickle.dumps(serialized)
-    save_pages(r, pages_list)
+    if serialized is not None:
+        assert pages_list is not None
+        r.serialized_data = pickle.dumps(serialized)
+        save_pages(r, pages_list)
     r.last_recalculation_date = date_before
     r.last_recalculation_duration = date_after - date_before
     old_recalc = r.recalc_in_progress
     r.recalc_in_progress = None
+    if cooldown_date is not None:
+        r.cooldown_date = cooldown_date
     r.save()
     old_recalc.delete()
 
@@ -188,9 +202,19 @@ def recalculate(recalc):
     except Ranking.DoesNotExist:
         return
     ranking_controller = r.controller()
-    serialized, pages_list = ranking_controller.build_ranking(r.key)
+    try:
+        serialized, pages_list = ranking_controller.build_ranking(r.key)
+        cooldown_date = None
+    except Exception as e:
+        if getattr(settings, "MOCK_RANKINGSD", False):
+            raise
+        logger = logging.getLogger(__name__ + ".recalculation")
+        logger.exception("An error occurred while recalculating ranking", e)
+        cooldown_duration = timedelta(seconds=settings.RANKINGS_ERROR_COOLDOWN)
+        cooldown_date = timezone.now() + cooldown_duration
+        serialized, pages_list = (None, None)
     date_after = timezone.now()
-    save_recalc_results(recalc, date_before, date_after, serialized, pages_list)
+    save_recalc_results(recalc, date_before, date_after, serialized, pages_list, cooldown_date)
 
 
 class RankingMessage(PublicMessage):

oioioi/rankings/templates/rankings/ranking_view.html

@@ -23,6 +23,11 @@
                 <i class="fa-solid fa-rotate-right"></i>
                 {% trans "Regenerate ranking" %}
             </a>
+            <a role="button" class="btn btn-sm btn-outline-secondary"
+                href="#" data-post-url="{% url 'ranking_invalidate_contest' contest_id=contest.id key=key %}">
+                <i class="fa-solid fa-rotate-right"></i>
+                {% trans "Regenerate all rankings" %}
+            </a>
         {% endif %}
         {% if form and user.is_authenticated and not is_admin %}
             <a role="button" class="btn btn-sm btn-outline-secondary"