diff --git a/cms/conf.py b/cms/conf.py index e8dc452a4f..610995942c 100644 --- a/cms/conf.py +++ b/cms/conf.py @@ -192,6 +192,8 @@ def __init__(self): self.overview_listen_address = "127.0.0.1" self.overview_listen_port = 8891 self.task_repository = None + self.tasks_folders = [] + self.contests_folders = [] self.auto_sync = False self.max_compilations = 1000 diff --git a/cms/io/BackgroundScheduler.py b/cms/io/BackgroundScheduler.py new file mode 100644 index 0000000000..6b0739f5df --- /dev/null +++ b/cms/io/BackgroundScheduler.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Programming contest management system +# Copyright © 2026 Erik Sünderhauf +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +from sched import scheduler +from sys import exc_info +from traceback import format_exception +from typing import Callable + +logger = logging.getLogger(__name__) + + +class BackgroundScheduler(scheduler): + def __init__(self): + super().__init__() + + def every(self, interval: float, func: Callable, priority: int = 0, skip_first: bool = True, args=(), **kwargs) -> None: + def wrapped(): + try: + func(*args, **kwargs) + except Exception: + logger.error("Failed to execute background task", + "\n".join(format_exception(*exc_info()))) + finally: + self.enter(interval, priority, wrapped) + delay = 0 + if skip_first: + delay = interval + self.enter(delay, priority, wrapped) diff --git a/cms/io/Repository.py b/cms/io/Repository.py index ac994b07b6..5ac420ccbe 100644 --- a/cms/io/Repository.py +++ b/cms/io/Repository.py @@ -26,6 +26,7 @@ from multiprocessing import Manager from subprocess import check_output +from typing import Optional from cmscontrib.gerpythonformat.LocationStack import chdir @@ -91,46 +92,48 @@ def _push(self): logger.info("Finished pushing: " + "{}".format(gitout)) - # For GerTranslate + # For GerTranslate/cmsTaskOverview # TODO Show errors in web overview - def commit(self, file_path, file_identifier): + def commit(self, file_path, commit_message="", author="") -> Optional[str]: + '''Commits the changes in file_path as `author` with given `commit_message`. + If an error occurs, the error message will be returned. + + ''' + if commit_message == "" or author == "": + raise Exception("Missing commit message or author") # TODO Only do this if it's a git repository # if self.auto_sync: logger.info("Committing {} in {}".format(file_path, self.path)) with chdir(self.path): gitout = "" - try: - gitout = check_output(["git", "add", - file_path]) - except: - logger.error("Couldn't add file to git staging area: " + - "{}".format(gitout)) - else: + gitout = check_output(["git", "add", file_path]) + except Exception as e: + logger.error("Couldn't add file to git staging area: {}".format(e)) + return str(e) + try: + gitout = "" + # NOTE file_path is relative to self.path, which isn't + # necessarily the root of the git repo. So the commit + # message might be confusing. + gitout = \ + check_output( + ["git", "commit", + "-o", file_path, + "-m", commit_message, + "--author", author] + ) + except Exception as e: + logger.error("Couldn't commit in repository: {}".format(e)) try: - gitout = "" - # NOTE file_path is relative to self.path, which isn't - # necessarily the root of the git repo. So the commit - # message might be confusing. - gitout = \ - check_output( - ["git", "commit", - "-o", file_path, - # TODO Provide meaningful commit message and - # author - "-m", "Changes to " + - file_identifier + - ", uploaded via GerTranslate web " - "interface", - "--author", '"GerTranslate "'] - ) - except: - logger.error("Couldn't commit in repository: " + - "{}".format(gitout)) - else: - logger.info("Committed: " + - "{}".format(gitout)) + # try to unstage files if committing failed + check_output(["git", "restore", "--staged", file_path]) + except Exception as e: + logger.warning("unable to unstage files: {}".format(e)) + return str(e) + logger.info("Committed: {}".format(gitout)) + # For GerTranslate # TODO Show errors in web overview diff --git a/cms/io/TaskAccess.py b/cms/io/TaskAccess.py index a22d7ce403..bacf115cbb 100644 --- a/cms/io/TaskAccess.py +++ b/cms/io/TaskAccess.py @@ -34,6 +34,7 @@ from six import StringIO from ansi2html import Ansi2HTMLConverter +from cms.io.Repository import Repository from cms.io.TaskTranslateInfo import TaskTranslateInfo from cmscontrib.gerpythonformat.ContestConfig import MyGroup @@ -333,7 +334,7 @@ def get(self): class TaskTeXReceiver: def __init__(self, repository, name): - self.repository = repository + self.repository: Repository = repository self.name = name def receive(self, f): @@ -364,15 +365,21 @@ def receive(self, f): else: with open(tex_file, "wb") as target_file: target_file.write(f) + # TODO Provide meaningful commit message and + # author self.repository.commit( - str(tex_file.resolve()), str(_repository_code)) + str(tex_file.resolve()), + commit_message=f"Changes to {str(_repository_code)}, " + "uploaded via GerTranslate web interface", + author='"GerTranslate "', + ) return result class TaskMarker: def __init__(self, repository, name): - self.repository = repository + self.repository: Repository = repository self.name = name def mark(self): @@ -385,8 +392,12 @@ def mark(self): with open(lock_file, "w") as target_file: target_file.write("The translation in this language is locked.") - self.repository.commit(str(lock_file.resolve()), - str(_repository_lock_file_code)) + self.repository.commit( + str(lock_file.resolve()), + commit_message=f"Changes to {str(_repository_lock_file_code)}, " + "uploaded via GerTranslate web interface", + author='"GerTranslate "', + ) class TaskGitLog: diff --git a/cms/io/TaskFetch.py b/cms/io/TaskFetch.py index 11d22d9e0e..d0c6539fd2 100644 --- a/cms/io/TaskFetch.py +++ b/cms/io/TaskFetch.py @@ -22,14 +22,17 @@ from __future__ import unicode_literals import logging +from pathlib import Path import sys from sys import exc_info from traceback import format_exception from multiprocessing import Process, Manager +from typing import Optional from six import StringIO from ansi2html import Ansi2HTMLConverter +from cms.io.Repository import Repository from cms.io.TaskInfo import TaskInfo from cmscontrib.gerpythonformat.GerMakeTask import GerMakeTask @@ -39,9 +42,10 @@ class TaskCompileJob: - def __init__(self, repository, name, balancer): + def __init__(self, repository: Repository, name: str, task_folder: str, balancer): self.repository = repository self.name = name + self.task_folder = task_folder self.balancer = balancer self.current_handle = 1 @@ -61,10 +65,10 @@ def join(self): def _compile(self): self._reset_status() - logger.info("loading task {} in {}".format(self.name, - self.repository.path)) + directory = Path(self.repository.path) / self.task_folder + logger.info("loading task {} in {}".format(self.name, str(directory))) - def do(status, repository, balancer): + def do(status, repository: Repository, directory: str, balancer): # stdout is process local in Python, so we can simply use this # to redirect all output from GerMakeTask to a string sys.stdout = StringIO() @@ -73,16 +77,16 @@ def do(status, repository, balancer): with balancer: try: - comp = GerMakeTask(odir = repository.path, - task = self.name, - minimal = True, - no_test = True, - submission = None, - no_latex = False, - verbose_latex = True, - language = None, - clean = False, - ntcimp = True) + comp = GerMakeTask(odir=directory, + task=self.name, + minimal=True, + no_test=True, + submission=None, + no_latex=False, + verbose_latex=True, + language=None, + clean=False, + ntcimp=True) with repository: comp.prepare() @@ -112,6 +116,7 @@ def do(status, repository, balancer): self.compilation_process = Process(target=do, args=(self.status, self.repository, + str(directory), self.balancer)) self.compilation_process.daemon = True self.compilation_process.start() @@ -174,7 +179,7 @@ def get(self): class TaskFetch: jobs = {} - repository = None + repository: Optional[Repository] = None balancer = None @staticmethod @@ -187,10 +192,14 @@ def init(repository, max_compilations): @staticmethod def compile(name): + if TaskFetch.repository is None: + raise Exception("tasks repository not initialized") if name not in TaskInfo.tasks: raise KeyError("No such task") if name not in TaskFetch.jobs: - TaskFetch.jobs[name] = TaskCompileJob(TaskFetch.repository, name, + TaskFetch.jobs[name] = TaskCompileJob(TaskFetch.repository, + name, + TaskInfo.tasks[name]["folder"], TaskFetch.balancer) return TaskFetch.jobs[name].join() diff --git a/cms/io/TaskInfo.py b/cms/io/TaskInfo.py index 95ef6f8155..232d59ef1a 100644 --- a/cms/io/TaskInfo.py +++ b/cms/io/TaskInfo.py @@ -22,20 +22,29 @@ from __future__ import print_function from __future__ import unicode_literals +import contextlib import json import logging +import re from datetime import datetime from pathlib import Path from multiprocessing import Process, Manager +from subprocess import check_output from sys import exc_info from traceback import format_exception -from time import sleep, time +from time import time from copy import deepcopy from math import sqrt +from typing import Collection from six import iteritems +from cms.io.BackgroundScheduler import BackgroundScheduler +from cms.io.Repository import Repository +from cmscontrib.gerpythonformat.ContestConfig import ContestConfig +from cmscontrib.gerpythonformat.LocationStack import chdir + logger = logging.getLogger(__name__) @@ -53,24 +62,27 @@ def to_dict(self): class SingleTaskInfo: - def __init__(self, code, path): - info = {"code": code, - "title": "???", - "source": "N/A", - "algorithm": -1, - "implementation": -1, - "keywords": [], - "uses": [], - "remarks": "", - "public": False, - "old": None} + def __init__(self, code: str, path: Path): + self.code = code + self.title = "???" + self.source = "N/A" + self.algorithm = -1 + self.implementation = -1 + self.keywords = [] + self.uses: Collection[DateEntry] = [] + self.remarks = "" + self.public = False + self.old = None + self.folder = "" + self.timestamp = 0 + self.error = None if not path.exists(): i = {"error": "The info.json file is missing."} else: try: i = json.loads(path.open().read()) - except: + except Exception: i = {"error": "The info.json file is corrupt."} else: missing = [] @@ -85,7 +97,7 @@ def __init__(self, code, path): def bad(x): try: y = int(x) - except: + except Exception: return True else: return y < 0 or y > 10 @@ -100,36 +112,40 @@ def bad(x): i["error"] = "Invalid value for \"implementation\": " \ "expected an integer between 0 and 10." - info.update(i) - - if info["old"] is None: - info["old"] = len(info["uses"]) > 0 or info["public"] + self.update(i) - try: - info["uses"] = [DateEntry(e[0], e[1]) for e in info["uses"]] - except ValueError: - info["uses"] = [] + if self.old is None: + self.old = len(self.uses) > 0 or self.public - if "error" not in info: - info["error"] = "I couldn't parse the dates for " \ - "\"(previous) uses\"." + def update(self, i): + for key, value in iteritems(i): + if key == "uses": + try: + self.uses = [DateEntry(e[0], e[1]) for e in i["uses"]] + except ValueError: + self.uses = [] - for key, value in iteritems(info): - setattr(self, key, value) + if self.error is not None: + self.error = "I couldn't parse the dates for " \ + "\"(previous) uses\"." + else: + setattr(self, key, value) def to_dict(self): - result = {"code": self.code, - "title": self.title, - "source": self.source, - "algorithm": self.algorithm, - "implementation": self.implementation, - "keywords": self.keywords, - "uses": [e.to_dict() for e in self.uses], - "remarks": self.remarks, - "public": self.public, - "old": self.old} - - if hasattr(self, "error"): + result = { + "code": self.code, + "title": self.title, + "source": self.source, + "algorithm": self.algorithm, + "implementation": self.implementation, + "keywords": self.keywords, + "uses": [e.to_dict() for e in self.uses], + "remarks": self.remarks, + "public": self.public, + "old": self.old, + } + + if self.error is not None: result["error"] = self.error return result @@ -137,61 +153,361 @@ def to_dict(self): class TaskInfo: tasks = Manager().dict() + unconfirmed_usage = Manager().dict() @staticmethod - def init(repository): - def load(directory, tasks): - # Load all available tasks - for d in directory.iterdir(): - if not d.is_dir(): + def init( + repository: Repository, + tasks_folders: Collection[str], + contests_folders: Collection[str], + ): + def load(repository_root: Path, tasks_folders: Collection[str], tasks): + for folder in tasks_folders: + directory = repository_root / folder + if not directory.exists() or not directory.is_dir(): + logger.warning( + 'Task folder "{}" does not exist or is not ' + "a directory, skipping.".format(directory) + ) continue + # Load all available tasks + for d in directory.iterdir(): + if not d.is_dir(): + continue - # We catch all exceptions since the main loop must go on - try: - info_path = d / "info.json" - - code = d.parts[-1] - info = SingleTaskInfo(code, info_path).to_dict() + # We catch all exceptions since the main loop must go on + try: + info_path = d / "info.json" - old = tasks.get(code, {"timestamp": 0}) - info["timestamp"] = old["timestamp"] + code = d.parts[-1] + info = SingleTaskInfo(code, info_path).to_dict() - if old != info: - info["timestamp"] = time() - tasks[code] = info + old = tasks.get(code, {"timestamp": 0}) + info["timestamp"] = old["timestamp"] + info["folder"] = folder - except: - logger.info("\n".join(format_exception(*exc_info()))) + if old != info: + info["timestamp"] = time() + tasks[code] = info - def main_loop(repository, tasks, waiting_time): - directory = Path(repository.path) + except Exception: + logger.info("\n".join(format_exception(*exc_info()))) - while True: - start = time() + def update_tasklist(repository: Repository, tasks_folders: Collection[str], tasks): + repository_root = Path(repository.path) + start = time() + with repository: + # Remove tasks that are no longer available + for t in tasks.keys(): + info_path = repository_root / tasks[t]["folder"] / t - with repository: - # Remove tasks that are no longer available - for t in tasks.keys(): - info_path = directory / t + if not info_path.exists(): + del tasks[t] - if not info_path.exists(): - del tasks[t] + load(repository_root, tasks_folders, tasks) - load(directory, tasks) + logger.info("finished iteration of TaskInfo.update_tasklist in {}ms". + format(int(1000 * (time() - start)))) - logger.info("finished iteration of TaskInfo.main_loop in {}ms". - format(int(1000 * (time() - start)))) - - sleep(waiting_time) + def load_usage(repository_root: Path, unconfirmed_usage) -> None: + try: + with open(repository_root / ".unconfirmed_usage.json", "r") as f: + data = f.read().splitlines() + unconfirmed_usage.clear() + for line in data: + parsed_line = json.loads(line) + if "task" not in parsed_line or "uses" not in parsed_line: + logger.warning("skipping corrupted line {}".format(line)) + continue + task_code = parsed_line["task"] + if task_code not in unconfirmed_usage: + unconfirmed_usage[task_code] = [] + confirmed = ( + parsed_line["confirmed"] + if "confirmed" in parsed_line + else False + ) + unconfirmed_usage[task_code] += [ + { + "uses": DateEntry(*parsed_line["uses"]).to_dict(), + "confirmed": confirmed, + } + ] + except FileNotFoundError: + pass + except Exception: + logger.warning("file .unconfirmed_usage.json is corrupt") + logger.warning("\n".join(format_exception(*exc_info()))) + + def is_same_contest( + contest_code: str, contestconfig: ContestConfig, info + ) -> bool: + if contestconfig.defaultgroup is None: + return False + start = contestconfig.defaultgroup.start.toordinal() + stop = contestconfig.defaultgroup.stop.toordinal() + if start <= info["timestamp"] and info["timestamp"] <= stop: + return True + # date in old info.json files might be off by a few days + # => try to match info.info with contest_code / config + if info["timestamp"] < start - 42 or stop + 42 < info["timestamp"]: + return False + description = re.sub(r"\s+\d+$", "", info["info"]).lower() + clean_info = make_clean_info(contest_code).lower() + if (clean_info in description) or (description in clean_info): + return True + + olympiad = re.search( + r"^(.{1,4}oi)\d+[-_].*", contest_code, flags=re.IGNORECASE + ) + if olympiad is None: + return False + olympiad = olympiad.group() + if hasattr(contestconfig, "_short_name"): + short_name = re.sub( + r"\s+\d+$", "", getattr(contestconfig, "_short_name") + ).lower() + if (short_name in description) or (description in short_name): + return True + return False + + def should_track_usage(contest_code: str) -> bool: + """Returns wheter the usage of a task in this contest should be tracked + in the info.json + + """ + if "test" in contest_code.lower(): + # ignore technical tests + return False + return True + + def make_clean_info(contest_code: str) -> str: + """Returns a "clean" description of the contest from its + contest_code to be used in the weboverview. + For example "ioiYYYY_1-1" => "lg1" + + """ + match = re.match( + r"^(.{1,4}oi)\d+[-_](.*)", contest_code, flags=re.IGNORECASE + ) + if match is None: + return contest_code + olympiad, usage = map(lambda s: s.lower(), match.groups()) + # remove enumerating suffix "-1", "-2", ..., usually campname-dayX + usage = re.sub(r"[-_]?\d$", "", usage) + info = "" + if olympiad == "ioi" and re.match(r"\d", usage): + info = olympiad + " lg" + usage + else: + info = olympiad + " " + usage + return info + + def parse_contests( + repository_root: Path, + contests_folders: Collection[str], + tasks, + unconfirmed_usage, + ) -> list: + untracked = [] + for folder in contests_folders: + directory = repository_root / folder + if not directory.exists() or not directory.is_dir(): + logger.warning( + 'Contest folder "{}" does not exist or is not' + " a directory, skipping.".format(directory) + ) + continue + for d in directory.iterdir(): + try: + if not d.is_dir(): + continue + contest_code = d.parts[-1] + if not should_track_usage(contest_code): + continue + config_path = d / "contest-config.py" + if not config_path.exists(): + logger.info( + 'Contest "{}" has no contest-config, ignoring.'.format( + d + ) + ) + continue + with chdir(d): + contestconfig = ContestConfig( + d / ".rules", + contest_code, + ignore_latex=True, + minimal=True, + ) + with contextlib.redirect_stdout(None): + # _parseconfig doesn't perform any actions, so we suppress + # log messages like "Creating ..." + contestconfig._parseconfig("contest-config.py") + if contestconfig.defaultgroup is None: + logger.info( + 'Contest "{}" has no default group, ignoring.'.format(d) + ) + continue + # only update the usage after the contest ended + if datetime.now() < contestconfig.defaultgroup.stop: + continue + for t in contestconfig.tasks.keys(): + task_code = t + task_path = d / t + # if the symlink to the task got deleted, we still try to + # update usage with the task code that was used in the contest + if task_path.exists(): + task_code = task_path.resolve().parts[-1] + if task_code not in tasks: + continue + task_info = tasks[task_code] + if any( + is_same_contest(contest_code, contestconfig, x) + for x in task_info["uses"] + ): + # usage is already stored in the info.json + continue + if task_code in unconfirmed_usage and any( + is_same_contest(contest_code, contestconfig, x["uses"]) + for x in unconfirmed_usage[task_code] + ): + # at this point the usage was detected and stored + # in a previous run, but marked as false positive, + # so we ignore it + continue + contest_day = contestconfig.defaultgroup.start.strftime( + "%Y-%m-%d" + ) + info = make_clean_info(contest_code) + usage = DateEntry(contest_day, info).to_dict() + untracked.append({"task": task_code, "usage": usage}) + logger.info(f"found usage for task {task_code} in {info}") + except Exception: + logger.error("Failed to process contest {}".format(d)) + logger.error("\n".join(format_exception(*exc_info()))) + return untracked + + def store_usage( + repository: Repository, untracked: list, tasks, unconfirmed_usage + ) -> None: + repository_root = Path(repository.path) + for v in untracked: + try: + task_code, usage = v["task"], v["usage"] + task = tasks[task_code] + usage_time = datetime.fromordinal(usage["timestamp"]) + usage_time = usage_time.strftime("%Y-%m-%d") + # remove trailing year + usage_info = re.sub(r"\s+\d+$", "", usage["info"]) + info_path = ( + repository_root / Path(task["folder"]) / task_code / "info.json" + ) + info_path = info_path.resolve() + with open(info_path, "r") as f: + old_data = json.load(f) + if "uses" not in old_data: + old_data["uses"] = [] + old_data["uses"].append([usage_time, usage_info]) + old_data["uses"].sort() + with open(info_path, "w") as f: + json.dump(old_data, f, indent=4) + err = repository.commit( + str(info_path), + commit_message=f"Add usage in contest {usage['info']} " + f"to task {task_code}, from TaskOverviewBackend", + author='"cmsTaskOverviewWebserver "', + ) + if err is not None: + # committing changes failed, so we try to restore the changes + # and retry in the next iteration + try: + check_output(["git", "checkout", "--", str(info_path)]) + except Exception as e: + logger.error( + "restoring old version of {} failed with error {}".format( + info_path, e + ) + ) + continue + task["uses"] += [usage] + task["timestamp"] = time() + if task_code not in unconfirmed_usage: + unconfirmed_usage[task_code] = [] + unconfirmed_usage[task_code] += [ + {"uses": usage, "confirmed": False} + ] + with open(repository_root / ".unconfirmed_usage.json", "a") as f: + f.write( + f'{{"task":"{task_code}", \ + "uses":["{usage_time}", "{usage_info}"], \ + "confirmed":false}}\n' + ) + except Exception: + logger.error("Failed to update task {}".format(task_code)) + logger.error("\n".join(format_exception(*exc_info()))) + + def update_usage( + repository: Repository, + contests_folders: Collection[str], + tasks, + unconfirmed_usage, + only_load=False, + ) -> None: + repository_root = Path(repository.path) + start = time() + with repository: + load_usage(repository_root, unconfirmed_usage) + if not only_load: + untracked = parse_contests( + repository_root, contests_folders, tasks, unconfirmed_usage + ) + store_usage(repository, untracked, tasks, unconfirmed_usage) + + logger.info( + "finished iteration of TaskInfo.update_usage in {}ms".format( + int(1000 * (time() - start)) + ) + ) + + def run(repository, tasks_folders, contests_folders, tasks, unconfirmed_usage): + scheduler = BackgroundScheduler() + scheduler.every( + 0.5 * (1 + sqrt(5)), + update_tasklist, + args=(repository, tasks_folders, tasks), + ) + scheduler.every( + 0.5 * (1 + sqrt(5)), + update_usage, + args=(repository, contests_folders, tasks, unconfirmed_usage), + only_load=True, + ) + # Usage data will be updated only after a contest ended + # so we can query less frequent + scheduler.every( + 3600, + update_usage, + args=(repository, contests_folders, tasks, unconfirmed_usage), + skip_first=False, + priority=1, + ) + scheduler.run(blocking=True) # Load data once on start-up (otherwise tasks might get removed when # the server is restarted) with repository: - load(Path(repository.path), TaskInfo.tasks) - - TaskInfo.info_process = Process(target=main_loop, - args=(repository, TaskInfo.tasks, - .5 * (1 + sqrt(5)))) + load(Path(repository.path), tasks_folders, TaskInfo.tasks) + + TaskInfo.info_process = Process( + target=run, + args=( + repository, + tasks_folders, + contests_folders, + TaskInfo.tasks, + TaskInfo.unconfirmed_usage, + ), + ) TaskInfo.info_process.daemon = True TaskInfo.info_process.start() @@ -204,9 +520,20 @@ def task_list(): @staticmethod def get_info(keys): - data = deepcopy(TaskInfo.tasks) - - return {x: data[x] for x in keys if x in data} + task_data = deepcopy(TaskInfo.tasks) + usage_data = deepcopy(TaskInfo.unconfirmed_usage) + res = {} + for t in keys: + if t not in task_data: + continue + res[t] = task_data[t] + if t in usage_data: + for u in res[t]["uses"]: + for v in usage_data[t]: + if u == v["uses"]: + u["confirmed"] = v["confirmed"] + break + return res @staticmethod def entries(): diff --git a/cms/io/TaskTranslateInfo.py b/cms/io/TaskTranslateInfo.py index a2cbf31904..dc4db56687 100644 --- a/cms/io/TaskTranslateInfo.py +++ b/cms/io/TaskTranslateInfo.py @@ -31,13 +31,15 @@ from multiprocessing import Process, Manager from sys import exc_info from traceback import format_exception -from time import sleep, time +from time import time from copy import deepcopy from math import sqrt from six import iteritems from babel.core import Locale +from cms.io.BackgroundScheduler import BackgroundScheduler + logger = logging.getLogger(__name__) #TODO Delete this from this file? @@ -101,7 +103,7 @@ def __init__(self, code, path, contest=""): info["locked"] = [l.name[:-5] for l in path.parent.iterdir() if l.is_file() and l.name.endswith(".lock")] - if info["filename"]==None: + if info["filename"] == None: info["filename"] = "statement" info["translated"] = [l.name[len(info["filename"])+1:-4] for l in path.parent.iterdir() @@ -137,9 +139,9 @@ class TaskTranslateInfo: def init(repository): def load_single(d, tasks, contests, is_contest): if not d.is_dir() \ - or d.name.startswith('.') \ - or (d.is_symlink() and d.name=="general")\ - or d.name=="build": + or d.name.startswith('.') \ + or (d.is_symlink() and d.name == "general")\ + or d.name == "build": return False # We catch all exceptions since the main loop must go on @@ -157,7 +159,8 @@ def load_single(d, tasks, contests, is_contest): else: contest = d.parts[-2] code = d.parts[-1] - info = SingleTaskTranslateInfo(code, info_path, contest).to_dict() + info = SingleTaskTranslateInfo( + code, info_path, contest).to_dict() old = tasks.get(code, {"timestamp": 0}) info["timestamp"] = old["timestamp"] @@ -194,41 +197,43 @@ def load(directory, tasks, contests, languages): for e in d.iterdir(): load_single(e, tasks, contests, is_contest=False) - def main_loop(repository, tasks, contests, languages, waiting_time): + def update_tasklist(repository, tasks, contests, languages): directory = Path(repository.path) - while True: - start = time() + start = time() - with repository: - # Remove tasks that are no longer available - for t in tasks.keys(): - if t.endswith("-overview"): - p = t[:-9] - elif tasks[t]["contest"] == "": - p = t - else: - p = Path(tasks[t]["contest"]) / t - info_path = directory / p + with repository: + # Remove tasks that are no longer available + for t in tasks.keys(): + if t.endswith("-overview"): + p = t[:-9] + elif tasks[t]["contest"] == "": + p = t + else: + p = Path(tasks[t]["contest"]) / t + info_path = directory / p - if not info_path.exists(): - del tasks[t] + if not info_path.exists(): + del tasks[t] - load(directory, tasks, contests, languages) + load(directory, tasks, contests, languages) - logger.info("finished iteration of TaskTranslateInfo.main_loop in {}ms". - format(int(1000 * (time() - start)))) + logger.info("finished iteration of TaskTranslateInfo.update_tasklist in {}ms". + format(int(1000 * (time() - start)))) - sleep(waiting_time) + def run(*args): + scheduler = BackgroundScheduler() + scheduler.every(.5 * (1 + sqrt(5)), update_tasklist, args=args) + scheduler.run(blocking=True) # Load data once on start-up (otherwise tasks might get removed when # the server is restarted) with repository: - load(Path(repository.path), TaskTranslateInfo.tasks, TaskTranslateInfo.contests, TaskTranslateInfo.languages) + load(Path(repository.path), TaskTranslateInfo.tasks, + TaskTranslateInfo.contests, TaskTranslateInfo.languages) - TaskTranslateInfo.info_process = Process(target=main_loop, - args=(repository, TaskTranslateInfo.tasks, TaskTranslateInfo.contests, TaskTranslateInfo.languages, - .5 * (1 + sqrt(5)))) + TaskTranslateInfo.info_process = Process(target=run, args=( + repository, TaskTranslateInfo.tasks, TaskTranslateInfo.contests, TaskTranslateInfo.languages,)) TaskTranslateInfo.info_process.daemon = True TaskTranslateInfo.info_process.start() @@ -253,31 +258,31 @@ def language_list(): def gertranslate_entries(): return ["contest", "code", "title", "keywords", "remarks", "pdf", "tex"] +\ - ["pdf-"+l for l in TaskTranslateInfo.languages] +\ - ["tex-"+l for l in TaskTranslateInfo.languages] +\ - ["log-"+l for l in TaskTranslateInfo.languages] +\ - ["upload-"+l for l in TaskTranslateInfo.languages] +\ - ["mark-"+l for l in TaskTranslateInfo.languages] +\ - ["pdf-ALL"] + ["pdf-"+l for l in TaskTranslateInfo.languages] +\ + ["tex-"+l for l in TaskTranslateInfo.languages] +\ + ["log-"+l for l in TaskTranslateInfo.languages] +\ + ["upload-"+l for l in TaskTranslateInfo.languages] +\ + ["mark-"+l for l in TaskTranslateInfo.languages] +\ + ["pdf-ALL"] @staticmethod def gertranslate_desc(): return { - **{"contest": "Contest", - "code": "Code", - "title": "Title", - "keywords": "Keywords", - "remarks": "Remarks / Changelog", - "pdf": "PDF [en]", - "tex": "TeX [en]", - "log": "Log [en]", - "pdf-ALL": "PDF [ALL]"}, - **{"pdf-"+l: "PDF ["+l+"]" for l in TaskTranslateInfo.languages}, - **{"tex-"+l: "TeX ["+l+"]" for l in TaskTranslateInfo.languages}, - **{"log-"+l: "Log ["+l+"]" for l in TaskTranslateInfo.languages}, - **{"upload-"+l: "Upload TeX ["+l+"]" for l in TaskTranslateInfo.languages}, - **{"mark-"+l: "Finalize ["+l+"]" for l in TaskTranslateInfo.languages} - } + **{"contest": "Contest", + "code": "Code", + "title": "Title", + "keywords": "Keywords", + "remarks": "Remarks / Changelog", + "pdf": "PDF [en]", + "tex": "TeX [en]", + "log": "Log [en]", + "pdf-ALL": "PDF [ALL]"}, + **{"pdf-"+l: "PDF ["+l+"]" for l in TaskTranslateInfo.languages}, + **{"tex-"+l: "TeX ["+l+"]" for l in TaskTranslateInfo.languages}, + **{"log-"+l: "Log ["+l+"]" for l in TaskTranslateInfo.languages}, + **{"upload-"+l: "Upload TeX ["+l+"]" for l in TaskTranslateInfo.languages}, + **{"mark-"+l: "Finalize ["+l+"]" for l in TaskTranslateInfo.languages} + } @staticmethod def languages_desc(): diff --git a/cms/server/taskoverview/server.py b/cms/server/taskoverview/server.py index a66737c796..9c59934982 100644 --- a/cms/server/taskoverview/server.py +++ b/cms/server/taskoverview/server.py @@ -106,10 +106,12 @@ def __init__(self): "static_path": resource_filename("cms.server", "taskoverview/static")} - repository = Repository(config.task_repository, config.auto_sync) + repository = Repository( + config.task_repository, config.auto_sync, auto_push=True + ) TaskFetch.init(repository, config.max_compilations) - TaskInfo.init(repository) + TaskInfo.init(repository, config.tasks_folders, config.contests_folders) self.app = Application(handlers, **params) diff --git a/cms/server/taskoverview/static/js/overview.js b/cms/server/taskoverview/static/js/overview.js index 3002867ba9..b9cc6339e5 100644 --- a/cms/server/taskoverview/static/js/overview.js +++ b/cms/server/taskoverview/static/js/overview.js @@ -4,29 +4,33 @@ function inner_cell(t, entry) { return '
'; } - + if(entry == "keywords") - { + { var r = '
    '; if(t[entry].length == 0) r += "—"; - + for(var i = 0; i < t[entry].length; ++i) r += '
  • ' + t[entry][i] + '
    • '; r += '
'; return r; } - + if(entry == "uses") { var r = '
    '; if(t[entry].length == 0) r += "—"; - - for(var i = 0; i < t[entry].length; ++i) - r += '
  • ' + t[entry][i].info + '
    • '; + + for(var i = 0; i < t[entry].length; ++i) { + r += '
  • ' + t[entry][i].info; + if (Object.hasOwn(t[entry][i], "confirmed") && t[entry][i].confirmed === false) + r += "*"; + r += '
    • '; + } r += '
'; return r; } - + else { return t[entry] @@ -65,7 +69,7 @@ function relevant(t, c) if("error" in t) return true; uses_ok = true; - + for(var i = 0; i < t.uses.length; ++i) if(t.uses[i].timestamp > criteria.only_before) uses_ok = false; @@ -81,173 +85,173 @@ var __info = {}; var __tasks = []; var __task_dict = {}; var __removed = {}; - + function build_row(task_code) { var t = __info[task_code]; var r = ""; - - r += ''; + + r += ''; r += cell(t, entries[0]); r += error_cell(t); - - for(var j = 1; j < entries.length; ++j) + + for(var j = 1; j < entries.length; ++j) r += cell(t, entries[j]); r += ''; - + return r; } - + function fill_table(new_tasks, updated_tasks, show_col, criteria, init) { new_tasks = new_tasks.sort(); - + if(init) { var table_body = ""; - + table_body += ''; for(var j = 0; j < entries.length; ++j) if(entries[j] == "download") table_body += '' + desc[entries[j]] + ''; // table-bordered doesn't work with th, so we emulate it else table_body += '' + desc[entries[j]] + ''; // table-bordered doesn't work with th, so we emulate it - table_body += ''; + table_body += ''; window.document.getElementById("overview").innerHTML = table_body; } - + function make_class_removal_function(id, cl) { return function() { window.document.getElementById(id).classList.remove(cl); } } - + // Insert new rows (à la mergesort) var last_entry = "overview-heading"; var old_tasks_idx = 0, new_tasks_idx = 0; - + while(true) { if(old_tasks_idx >= __tasks.length && new_tasks_idx >= new_tasks.length) break; - + if(old_tasks_idx >= __tasks.length || __tasks[old_tasks_idx] >= new_tasks[new_tasks_idx]) { $(build_row(new_tasks[new_tasks_idx], show_col)).insertAfter("#" + last_entry); init_download_icon(new_tasks[new_tasks_idx]) last_entry = "row-" + new_tasks[new_tasks_idx]; - + if(!init) { window.document.getElementById(last_entry).classList.add("fresh"); // We need to remove the attribute after the animation is done for other animations to be triggered -- moreover, it is semantically nicer window.document.getElementById(last_entry).addEventListener("animationend", make_class_removal_function(last_entry, "fresh")); - + } - + ++new_tasks_idx; } - + else { last_entry = "row-" + __tasks[old_tasks_idx]; ++old_tasks_idx; } } - + __tasks = __tasks.concat(new_tasks).sort(); - + var num_interesting_columns = 0; for(var j = 1; j < entries.length - 1; ++j) if(show_col[entries[j]]) ++num_interesting_columns; - + for(var i = 0; i < __tasks.length; ++i) { // Apply row selection var id = "row-" + __tasks[i]; var row = window.document.getElementById(id); - + if(relevant(__info[__tasks[i]], criteria)) row.classList.remove("hidden"); else row.classList.add("hidden"); - + // Remove tasks that are no longer available if(__tasks[i] in __removed) { row.classList.add("removed"); - + // Indirectness needed to make currying work function make_removal_function(s) { return function() { $("#" + s).remove(); } } - + window.setTimeout(make_removal_function("row-" + __tasks[i]), 1000); } - + // Highlight unused tasks if(__info[__tasks[i]].old || "error" in __info[__tasks[i]]) row.classList.remove("new"); else row.classList.add("new"); - - // Apply column selection + + // Apply column selection for(var j = 0; j < entries.length; ++j) { var cell = window.document.getElementById("cell-" + __tasks[i] + "-" + entries[j]); - + if(show_col[entries[j]]) cell.classList.remove("hidden"); else cell.classList.add("hidden"); } - + window.document.getElementById("cell-" + __tasks[i] + "-error-msg").colSpan = num_interesting_columns; - + // Error or correct entry? var t = __info[__tasks[i]]; - + if("error" in t) { for(var j = 1; j < entries.length - 1; ++j) { var cell = window.document.getElementById("cell-" + __tasks[i] + "-" + entries[j]); cell.classList.add("hidden"); - } - - window.document.getElementById("cell-" + __tasks[i] + "-error-msg").classList.remove("no-error"); + } + + window.document.getElementById("cell-" + __tasks[i] + "-error-msg").classList.remove("no-error"); } - + else { - window.document.getElementById("cell-" + __tasks[i] + "-error-msg").classList.add("no-error"); + window.document.getElementById("cell-" + __tasks[i] + "-error-msg").classList.add("no-error"); } } - + // Update tasks that changed since last query for(var i = 0; i < updated_tasks.length; ++i) { var row = window.document.getElementById("row-" + updated_tasks[i]); - + for(var j = 0; j < entries.length; ++j) { if(entries[j] == "download") continue; - + var cell = window.document.getElementById("cell-" + updated_tasks[i] + "-" + entries[j]); - + cell.innerHTML = inner_cell(__info[updated_tasks[i]], entries[j]); - + /*if(show_col[entries[j]]) cell.classList.remove("hidden"); else cell.classList.add("hidden");*/ } - + window.document.getElementById("cell-" + updated_tasks[i] + "-error-msg").innerHTML = inner_error_cell(__info[updated_tasks[i]]); - + row.classList.add("updated"); window.setTimeout(make_class_removal_function("row-" + updated_tasks[i], "updated"), 1000); } - + // Remove tasks that have been deleted from the filesystem var __tasks_backup = __tasks; __tasks = []; @@ -258,21 +262,21 @@ function fill_table(new_tasks, updated_tasks, show_col, criteria, init) else __tasks.push(__tasks_backup[i]); } - + for(var j = 0; j < entries.length; ++j) { var cell = window.document.getElementById("overview-heading-" + entries[j]); - + if(show_col[entries[j]]) cell.classList.remove("hidden"); else - cell.classList.add("hidden"); + cell.classList.add("hidden"); } - + // Coloring and rounded borders var count = 0; var prefix = "overview-heading"; - + for(var j = 0; j < entries.length; ++j) { var id = prefix + "-" + entries[j]; @@ -282,20 +286,20 @@ function fill_table(new_tasks, updated_tasks, show_col, criteria, init) cell.classList.remove("upper-left"); cell.classList.remove("upper-right"); } - + for(var i = 0; i < __tasks.length; ++i) { var id = "row-" + __tasks[i]; var row = window.document.getElementById(id); - + if(row.classList.contains("hidden")) continue; - + if(count % 2 == 0) row.classList.remove("odd"); else row.classList.add("odd"); - + ++count; prefix = "cell-" + __tasks[i]; - + for(var j = 0; j < entries.length; ++j) { var id = prefix + "-" + entries[j]; @@ -306,43 +310,43 @@ function fill_table(new_tasks, updated_tasks, show_col, criteria, init) cell.classList.remove("upper-right"); } } - + var first_col = null; var last_col = null; - + for(var j = 0; j < entries.length; ++j) { var id = prefix + "-" + entries[j]; var cell = window.document.getElementById(id); - + if(cell.classList.contains("hidden")) continue; - + if(first_col == null) first_col = entries[j]; last_col = entries[j]; } - + if(last_col != null) { window.document.getElementById(prefix + "-" + first_col).classList.add("lower-left"); window.document.getElementById(prefix + "-" + last_col).classList.add("lower-right"); - + window.document.getElementById("overview-heading-" + first_col).classList.add("upper-left"); window.document.getElementById("overview-heading-" + last_col).classList.add("upper-right"); } - + // Create overview of problematic info.json files var json_issues = []; - + for(var i = 0; i < __tasks.length; ++i) if("error" in __info[__tasks[i]]) json_issues.push(__tasks[i]); - + if(json_issues.length == 0) window.document.getElementById("json-errors").classList.add("no-errors"); - + else window.document.getElementById("json-errors").classList.remove("no-errors"); - + window.document.getElementById("tasks-with-issues").innerHTML = json_issues.join(", "); } @@ -351,7 +355,7 @@ function update(init=false, sliders=false) function on_list(l) { var response = JSON.parse(l); - + var available_tasks = []; var available_tasks_dict = {}; for(var i = 0; i < response.length; ++i) @@ -359,20 +363,20 @@ function update(init=false, sliders=false) available_tasks.push(response[i].task); available_tasks_dict[response[i].task] = response[i].timestamp; } - + var new_tasks = []; var updated_tasks = []; - + for(var i = 0; i < available_tasks.length; ++i) { var t = available_tasks[i]; - + if(!(t in __task_dict)) { new_tasks.push(t); __task_dict[t] = available_tasks_dict[t]; } - + if(__task_dict[t] < available_tasks_dict[t]) { updated_tasks.push(t); @@ -381,11 +385,11 @@ function update(init=false, sliders=false) } __removed = {}; - + for(var i = 0; i < __tasks.length; ++i) if(!(__tasks[i] in available_tasks_dict)) __removed[__tasks[i]] = true; - + function on_info(i) { var info = JSON.parse(i); @@ -393,16 +397,16 @@ function update(init=false, sliders=false) __info[new_tasks[i]] = info[new_tasks[i]]; for(var i = 0; i < updated_tasks.length; ++i) __info[updated_tasks[i]] = info[updated_tasks[i]]; - + fill_table(new_tasks, updated_tasks, show_col, criteria, init); if(sliders) update_sliders(init); } - + if(new_tasks.length > 0 || updated_tasks.length > 0) { $.get(__url_root + "/info", {"tasks": JSON.stringify(new_tasks.concat(updated_tasks))}, on_info); } - + else { fill_table(new_tasks, updated_tasks, show_col, criteria, init); @@ -416,39 +420,39 @@ function update(init=false, sliders=false) function update_sliders(init=false) { init_date_slider("uses", interesting_dates(), true); - if(!init) date_slider_set("uses", criteria.only_before); - if(init) criteria.only_before = date_slider_get_val("uses"); + if(!init) date_slider_set("uses", criteria.only_before); + if(init) criteria.only_before = date_slider_get_val("uses"); } function interesting_dates() { var raw = []; - + for(var i = 0; i < __tasks.length; ++i) raw = raw.concat(__info[__tasks[i]].uses); - + function my_comp(a, b) { if(a.timestamp < b.timestamp) return -1; if(a.timestamp > b.timestamp) return 1; return 0; } - + raw = raw.sort(my_comp); /* Merge multiple entries for the same contest (multiple days, typos in the files etc.): - + If there are multiple entries with the same description at successive time points, we keep only the last one. - + This is of course not perfect, but it should work in most cases. Since the selection criteria are just for convenience, this should be fine (and it is certainly much better than having to sanitize all info.json files by hand) */ var dates = []; - + for(var i = 0; i < raw.length; ++i) { if(dates.length > 0 && dates[dates.length - 1].info == raw[i].info) diff --git a/cmscontrib/gerpythonformat/ContestConfig.py b/cmscontrib/gerpythonformat/ContestConfig.py index c6477403a5..9ce028c50c 100644 --- a/cmscontrib/gerpythonformat/ContestConfig.py +++ b/cmscontrib/gerpythonformat/ContestConfig.py @@ -165,9 +165,11 @@ def __init__(self, rules, name, ignore_latex=False, verbose_latex=False, # a standard tokenwise comparator (specified here so that it has to be # compiled at most once per contest) - shutil.copy(os.path.join(self._get_ready_dir(), "tokens.cpp"), - "tokens.cpp") - self.token_equ_fp = self.compile("tokens") + if not minimal: + shutil.copy(os.path.join(self._get_ready_dir(), "tokens.cpp"), "tokens.cpp") + self.token_equ_fp = self.compile("tokens") + else: + self.token_equ_fp = None # there is no upstream for contest self.bequeathing = False @@ -187,6 +189,35 @@ def _readconfig(self, filename): super(ContestConfig, self)._readconfig(filename) self._initialize_ranking() + def _parseconfig(self, filename): + """attempts to parse a contest-config file and extract metadata without + performing any build/compile tasks + + """ + + def dummy_func(*args, **kwargs): + pass + + original_exported = copy.deepcopy(self.exported) + for k in self.exported: + if k not in ["time", "timezone", "team", "user_group"]: + self.exported[k] = dummy_func + + def new_load_template(name, *args, **kwargs): + if "short_name" in kwargs: + setattr(self, "_short_name", kwargs["short_name"]) + + def new_task(s, *args, **kwargs): + self.tasks[s] = "" + + self.exported["task"] = new_task + self.exported["load_template"] = new_load_template + try: + super(ContestConfig, self)._readconfig(filename) + except Exception as e: + print("Failed to parse contest-config with error", e) + self.exported = original_exported + def finish(self): asy_cnt = self.asy_warnings + sum(t.asy_warnings for t in self.tasks.values()) diff --git a/config/cms.conf.sample b/config/cms.conf.sample index 872bf7674c..dff311181b 100644 --- a/config/cms.conf.sample +++ b/config/cms.conf.sample @@ -192,11 +192,16 @@ "_section": "TaskOverviewWebServer", "_help": "Listening port and address for TaskOverviewWebServer.", - "listen_address": "127.0.0.1", - "listen_port": 8891, + "overview_listen_address": "0.0.0.0", + "overview_listen_port": 8891, - "_help": "Directory to search for tasks (you have to fill this yourself!).", + "_help": "Repository to search for tasks (you have to fill this yourself!).", "task_repository": "", + "_help": "Directories relative to the repository to search for tasks.", + "tasks_folders": ["tasks"], + "_help": "Directories relative to the repository where contests are configured.", + "_help": "Used to automatically collect usage data for tasks.", + "contests_folders": ["contests"], "_help": "Whether to automatically sync the task repository (only use this", "_help": "for git repositories).",