diff --git a/oioioi/default_settings.py b/oioioi/default_settings.py
index cf5eb407b..63bfdb9df 100755
--- a/oioioi/default_settings.py
+++ b/oioioi/default_settings.py
@@ -658,6 +658,13 @@
'level': 'DEBUG',
'propagate': True,
},
+ # Errors in recalculation of rankings are rare, but not trivial to
+ # notice.
+ 'oioioi.rankings.models.recalculation': {
+ 'handlers': ['mail_admins'],
+ 'level': 'DEBUG',
+ 'propagate': True,
+ },
'celery': {
'handlers': ['console', 'emit_notification'],
'level': 'DEBUG',
@@ -774,8 +781,10 @@
}
# Ranking
+RANKINGSD_CONCURRENCY = 1 # Number of rankingsd instances to start.
RANKINGSD_POLLING_INTERVAL = 0.5 # seconds
RANKING_COOLDOWN_FACTOR = 2 # seconds
+RANKING_ERROR_COOLDOWN = 300 # seconds; Don't overwhelm the admins' mailbox :).
RANKING_MIN_COOLDOWN = 5 # seconds
RANKING_MAX_COOLDOWN = 100 # seconds
@@ -900,4 +909,4 @@
STATICFILES_DIRS = [
BASE_DIR / "dist_webpack",
BASE_DIR / "node_modules"
-]
\ No newline at end of file
+]
diff --git a/oioioi/deployment/settings.py.template b/oioioi/deployment/settings.py.template
index 0a4f91bb7..e79f8c662 100755
--- a/oioioi/deployment/settings.py.template
+++ b/oioioi/deployment/settings.py.template
@@ -495,6 +495,7 @@ ZEUS_INSTANCES = {
# }
# Ranking
+# RANKINGSD_CONCURRENCY = 1 # Number of rankingsd instances to start.
# RANKINGSD_POLLING_INTERVAL = 0.5 # seconds
# RANKING_COOLDOWN_FACTOR = 2 # seconds
# RANKING_MIN_COOLDOWN = 5 # seconds
@@ -608,4 +609,4 @@ ALLOW_ONLY_GET_FOR_SU_CONTEST_ADMINS = True
#
# REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] += (
# 'oauth2_provider.contrib.rest_framework.OAuth2Authentication',
-# )
\ No newline at end of file
+# )
diff --git a/oioioi/deployment/supervisord.conf.template b/oioioi/deployment/supervisord.conf.template
index fe7b9f375..ecd11a775 100644
--- a/oioioi/deployment/supervisord.conf.template
+++ b/oioioi/deployment/supervisord.conf.template
@@ -25,9 +25,11 @@ stderr_logfile={{ PROJECT_DIR }}/logs/runserver-err.log
[program:rankingsd]
command={{ PYTHON }} {{ PROJECT_DIR }}/manage.py rankingsd
startretries=0
+numprocs={{ settings.RANKINGSD_CONCURRENCY }}
redirect_stderr=false
-stdout_logfile={{ PROJECT_DIR }}/logs/rankingsd.log
-stderr_logfile={{ PROJECT_DIR }}/logs/rankingsd-err.log
+process_name=rankingsd_%(process_num)d
+stdout_logfile={{ PROJECT_DIR }}/logs/rankingsd_%(process_num)d.log
+stderr_logfile={{ PROJECT_DIR }}/logs/rankingsd_%(process_num)d-err.log
[program:mailnotifyd]
command={{ PYTHON }} {{ PROJECT_DIR }}/manage.py mailnotifyd
diff --git a/oioioi/pa/controllers.py b/oioioi/pa/controllers.py
index 2fd7441fc..9bc7aaa70 100644
--- a/oioioi/pa/controllers.py
+++ b/oioioi/pa/controllers.py
@@ -251,6 +251,19 @@ def available_rankings(self, request):
rankings.append((str(round.id), round.name))
return rankings
+ def partial_keys_for_probleminstance(self, pi):
+ partial_keys = []
+ division = pi.paprobleminstancedata.division
+ if division == "NONE":
+ if pi.round is not None and pi.round.is_trial:
+ partial_keys.append(str(pi.round_id))
+ else:
+ # This logic works for PADivCRankingController too.
+ partial_keys.append(A_PLUS_B_RANKING_KEY)
+ if division != "A":
+ partial_keys.append(B_RANKING_KEY)
+ return partial_keys
+
def _filter_pis_for_ranking(self, partial_key, queryset):
if partial_key == A_PLUS_B_RANKING_KEY:
return queryset.filter(paprobleminstancedata__division__in=["A", "B"])
diff --git a/oioioi/pa/tests.py b/oioioi/pa/tests.py
index 6ff8b729e..7a0dc9d19 100644
--- a/oioioi/pa/tests.py
+++ b/oioioi/pa/tests.py
@@ -13,6 +13,7 @@
from oioioi.contests.models import (
Contest,
ProblemInstance,
+ Round,
Submission,
UserResultForProblem,
)
@@ -23,6 +24,7 @@
from oioioi.pa.score import PAScore, ScoreDistribution
from oioioi.participants.models import Participant, TermsAcceptedPhrase
from oioioi.problems.models import Problem
+from oioioi.rankings.models import Ranking
class TestPAScore(TestCase):
@@ -178,6 +180,48 @@ def check_visibility(good_keys, response):
response = self.client.get(self._ranking_url(3))
check_visibility(["NONE"], response)
+ def test_probleminstance_invalidation(self):
+ contest = Contest.objects.get()
+
+ rc = contest.controller.ranking_controller()
+ trial_r = Round.objects.get(id=3)
+ self.assertTrue(trial_r.is_trial)
+ a_pi = ProblemInstance.objects.get(id=1)
+ b_pi = ProblemInstance.objects.get(id=3)
+ trial_pi = ProblemInstance.objects.get(id=5)
+
+ def make_ranking_for_key(perm, partial_key):
+ key = perm + "#" + partial_key
+ return Ranking.objects.get_or_create(contest=contest, key=key, needs_recalculation=False)[0]
+
+ ab_ranking = make_ranking_for_key("admin", A_PLUS_B_RANKING_KEY)
+ b_ranking = make_ranking_for_key("admin", B_RANKING_KEY)
+ trial_ranking = make_ranking_for_key("admin", str(trial_r.id))
+
+ rankings = (b_ranking, ab_ranking, trial_ranking)
+
+ def uninvalidate():
+ for r in rankings:
+ r.needs_recalculation = False
+ r.save()
+ self.assertTrue(r.is_up_to_date())
+
+ def test_pi_invalidation(pi, invalid_mask):
+ uninvalidate()
+ rc.invalidate_pi(pi)
+ i = 0
+ for r in rankings:
+ r.refresh_from_db()
+ self.assertEqual(r.is_up_to_date(), 0 == invalid_mask[i])
+ i += 1
+
+ test_pi_invalidation(a_pi, (0, 1, 0))
+ test_pi_invalidation(b_pi, (1, 1, 0))
+ test_pi_invalidation(trial_pi, (0, 0, 1))
+ trial_pi.round = None
+ trial_pi.save()
+ test_pi_invalidation(trial_pi, (0, 0, 0))
+
def test_no_zero_scores_in_ranking(self):
self.assertTrue(self.client.login(username="test_user"))
with fake_time(datetime(2013, 1, 1, 0, 0, tzinfo=UTC)):
diff --git a/oioioi/rankings/controllers.py b/oioioi/rankings/controllers.py
index 9f6b43cd2..1c6159e42 100644
--- a/oioioi/rankings/controllers.py
+++ b/oioioi/rankings/controllers.py
@@ -33,7 +33,7 @@ def ranking_controller(self):
def update_user_results(self, user, problem_instance, *args, **kwargs):
super().update_user_results(user, problem_instance, *args, **kwargs)
- Ranking.invalidate_contest(problem_instance.round.contest)
+ self.ranking_controller().invalidate_pi(problem_instance)
ContestController.mix_in(RankingMixinForContestController)
@@ -55,26 +55,41 @@ class RankingController(RegisteredSubclassesBase, ObjectWithMixins):
modules_with_subclasses = ["controllers"]
abstract = True
+ PERMISSION_LEVELS = [
+ "admin",
+ "observer",
+ "regular",
+ ]
PERMISSION_CHECKERS = [
lambda request: "admin" if is_contest_basicadmin(request) else None,
lambda request: "observer" if is_contest_observer(request) else None,
lambda request: "regular",
]
+ def construct_full_key(self, perm_level, partial_key):
+ return perm_level + "#" + partial_key
+
def get_partial_key(self, key):
"""Extracts partial key from a full key."""
return key.split("#")[1]
def replace_partial_key(self, key, new_partial):
"""Replaces partial key in a full key"""
- return key.split("#")[0] + "#" + new_partial
+ return self.construct_full_key(self._key_permission(key), new_partial)
def get_full_key(self, request, partial_key):
"""Returns a full key associated with request and partial_key"""
for checker in self.PERMISSION_CHECKERS:
res = checker(request)
if res is not None:
- return res + "#" + partial_key
+ return self.construct_full_key(res, partial_key)
+
+ def construct_all_full_keys(self, partial_keys):
+ fulls = []
+ for perm in self.PERMISSION_LEVELS:
+ for partial in partial_keys:
+ fulls.append(self.construct_full_key(perm, partial))
+ return fulls
def _key_permission(self, key):
"""Returns a permission level associated with given full key"""
@@ -228,6 +243,23 @@ def available_rankings(self, request):
return rankings[:1]
return rankings
+ # Rankings with different partial key logic need must override this
+ # or invalidate_pi accordingly. As a last resort, the all rankings
+ # for the given contest may be invalidated.
+ def partial_keys_for_probleminstance(self, pi):
+ return [CONTEST_RANKING_KEY, str(pi.round_id)]
+
+ def keys_for_probleminstance(self, pi):
+ return self.construct_all_full_keys(self.partial_keys_for_probleminstance(pi))
+
+ def invalidate_pi(self, pi):
+ Ranking.invalidate_queryset(
+ Ranking.objects.filter(
+ contest_id=pi.contest_id,
+ key__in=self.keys_for_probleminstance(pi),
+ )
+ )
+
def can_search_for_users(self):
return True
diff --git a/oioioi/rankings/models.py b/oioioi/rankings/models.py
index dc8fe2174..0dedb5921 100644
--- a/oioioi/rankings/models.py
+++ b/oioioi/rankings/models.py
@@ -1,3 +1,4 @@
+import logging
import pickle
from datetime import timedelta # pylint: disable=E0611
@@ -140,7 +141,16 @@ def clamp(minimum, x, maximum):
@transaction.atomic
def choose_for_recalculation():
now = timezone.now()
- r = Ranking.objects.filter(needs_recalculation=True, cooldown_date__lt=now).order_by("last_recalculation_date").select_for_update().first()
+ r = (
+ Ranking.objects.filter(
+ needs_recalculation=True,
+ cooldown_date__lt=now,
+ recalc_in_progress=None,
+ )
+ .order_by("last_recalculation_date")
+ .select_for_update()
+ .first()
+ )
if r is None:
return None
cooldown_duration = clamp(
@@ -166,17 +176,21 @@ def save_pages(ranking, pages_list):
@transaction.atomic
-def save_recalc_results(recalc, date_before, date_after, serialized, pages_list):
+def save_recalc_results(recalc, date_before, date_after, serialized, pages_list, cooldown_date):
try:
r = Ranking.objects.filter(recalc_in_progress=recalc).select_for_update().get()
except Ranking.DoesNotExist:
return
- r.serialized_data = pickle.dumps(serialized)
- save_pages(r, pages_list)
+ if serialized is not None:
+ assert pages_list is not None
+ r.serialized_data = pickle.dumps(serialized)
+ save_pages(r, pages_list)
r.last_recalculation_date = date_before
r.last_recalculation_duration = date_after - date_before
old_recalc = r.recalc_in_progress
r.recalc_in_progress = None
+ if cooldown_date is not None:
+ r.cooldown_date = cooldown_date
r.save()
old_recalc.delete()
@@ -188,9 +202,19 @@ def recalculate(recalc):
except Ranking.DoesNotExist:
return
ranking_controller = r.controller()
- serialized, pages_list = ranking_controller.build_ranking(r.key)
+ try:
+ serialized, pages_list = ranking_controller.build_ranking(r.key)
+ cooldown_date = None
+ except Exception as e:
+ if getattr(settings, "MOCK_RANKINGSD", False):
+ raise
+ logger = logging.getLogger(__name__ + ".recalculation")
+ logger.exception("An error occurred while recalculating ranking", e)
+ cooldown_duration = timedelta(seconds=settings.RANKINGS_ERROR_COOLDOWN)
+ cooldown_date = timezone.now() + cooldown_duration
+ serialized, pages_list = (None, None)
date_after = timezone.now()
- save_recalc_results(recalc, date_before, date_after, serialized, pages_list)
+ save_recalc_results(recalc, date_before, date_after, serialized, pages_list, cooldown_date)
class RankingMessage(PublicMessage):
diff --git a/oioioi/rankings/templates/rankings/ranking_view.html b/oioioi/rankings/templates/rankings/ranking_view.html
index c3aa18510..ffdca33ee 100644
--- a/oioioi/rankings/templates/rankings/ranking_view.html
+++ b/oioioi/rankings/templates/rankings/ranking_view.html
@@ -23,6 +23,11 @@
{% trans "Regenerate ranking" %}
+
+
+ {% trans "Regenerate all rankings" %}
+
{% endif %}
{% if form and user.is_authenticated and not is_admin %}
[a-z0-9_-]+)/invalidate_contest/$",
+ views.ranking_invalidate_view,
+ {"invalidate_whole_contest": True},
+ name="ranking_invalidate_contest",
+ ),
]
diff --git a/oioioi/rankings/views.py b/oioioi/rankings/views.py
index 553034053..90d758919 100644
--- a/oioioi/rankings/views.py
+++ b/oioioi/rankings/views.py
@@ -127,11 +127,14 @@ def ranking_csv_view(request, key):
@enforce_condition(contest_exists & is_contest_basicadmin)
@require_POST
-def ranking_invalidate_view(request, key):
- rcontroller = request.contest.controller.ranking_controller()
- full_key = rcontroller.get_full_key(request, key)
- ranking = Ranking.objects.filter(key=full_key)
- Ranking.invalidate_queryset(ranking)
+def ranking_invalidate_view(request, key, invalidate_whole_contest=False):
+ if invalidate_whole_contest:
+ Ranking.invalidate_contest(request.contest)
+ else:
+ rcontroller = request.contest.controller.ranking_controller()
+ full_keys = rcontroller.construct_all_full_keys([key])
+ rankings = Ranking.objects.filter(key__in=full_keys, contest=request.contest)
+ Ranking.invalidate_queryset(rankings)
return redirect("ranking", key=key)
diff --git a/oioioi/usergroups/controllers.py b/oioioi/usergroups/controllers.py
index 27346fbc4..064e33b58 100644
--- a/oioioi/usergroups/controllers.py
+++ b/oioioi/usergroups/controllers.py
@@ -3,6 +3,7 @@
from oioioi.contests.utils import is_contest_basicadmin, is_contest_observer
from oioioi.rankings.controllers import CONTEST_RANKING_KEY, DefaultRankingController
+from oioioi.rankings.models import Ranking
from oioioi.teachers.controllers import TeacherRegistrationController
from oioioi.usergroups.models import UserGroup, UserGroupRanking
@@ -60,6 +61,16 @@ def available_rankings(self, request):
rankings.append((user_group_ranking_id(user_group.id), user_group.name))
return rankings
+ def partial_keys_for_probleminstance(self, pi):
+ partial_keys = super().partial_keys_for_probleminstance(pi)
+ keys = Ranking.objects.filter(
+ contest_id=pi.contest_id,
+ # Somewhat hacky.
+ key__contains=self.construct_full_key("", USER_GROUP_RANKING_PREFIX),
+ ).values_list("key", flat=True)
+ partial_keys.extend(self.get_partial_key(key) for key in keys)
+ return partial_keys
+
def filter_users_for_ranking(self, key, queryset):
queryset = super().filter_users_for_ranking(key, queryset)
partial_key = self.get_partial_key(key)
diff --git a/oioioi/usergroups/tests.py b/oioioi/usergroups/tests.py
index a7e5efc39..eb243b9dc 100644
--- a/oioioi/usergroups/tests.py
+++ b/oioioi/usergroups/tests.py
@@ -6,7 +6,7 @@
from django.urls.exceptions import NoReverseMatch
from oioioi.base.tests import TestCase
-from oioioi.contests.models import Contest
+from oioioi.contests.models import Contest, ProblemInstance
from oioioi.contests.tests.utils import make_user_contest_admin
from oioioi.participants.models import Participant
from oioioi.rankings.models import Ranking
@@ -583,6 +583,38 @@ def test_ranking_view(self):
response = self.client.get(url)
self.assertEqual(response.status_code, 404)
+ def test_probleminstance_invalidation(self):
+ contest = Contest.objects.get(id="c")
+
+ rc = contest.controller.ranking_controller()
+ pi = ProblemInstance.objects.filter(contest=contest).first()
+
+ def make_ranking_for_key(perm, partial_key):
+ key = perm + "#" + partial_key
+ return Ranking.objects.get_or_create(contest=contest, key=key, needs_recalculation=False)[0]
+
+ usergroup_ranking = make_ranking_for_key("admin", "g2001")
+ contest_ranking = make_ranking_for_key("admin", "c")
+ self.assertTrue(contest_ranking.is_up_to_date())
+ self.assertTrue(usergroup_ranking.is_up_to_date())
+
+ round = pi.round
+ self.assertIsNotNone(round)
+ pi.round = None
+ pi.save()
+ rc.invalidate_pi(pi)
+ self.assertTrue(contest_ranking.is_up_to_date())
+ self.assertTrue(usergroup_ranking.is_up_to_date())
+
+ pi.round = round
+ pi.save()
+ rc.invalidate_pi(pi)
+ contest_ranking.refresh_from_db()
+ usergroup_ranking.refresh_from_db()
+ self.assertFalse(contest_ranking.is_up_to_date())
+ self.assertFalse(usergroup_ranking.is_up_to_date())
+
+
class RemoveUserGroupRankingsTest(TestCase):
fixtures = [