From 8e2118bceb1a10d92e1b5113c1c5a113526d3115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20S=C3=BCnderhauf?= Date: Wed, 14 Jan 2026 11:33:32 +0100 Subject: [PATCH 1/6] move main_loop to background scheduler --- cms/io/BackgroundScheduler.py | 46 ++++++++++++++ cms/io/TaskInfo.py | 39 ++++++------ cms/io/TaskTranslateInfo.py | 114 ++++++++++++++++++---------------- 3 files changed, 127 insertions(+), 72 deletions(-) create mode 100644 cms/io/BackgroundScheduler.py diff --git a/cms/io/BackgroundScheduler.py b/cms/io/BackgroundScheduler.py new file mode 100644 index 0000000000..6b0739f5df --- /dev/null +++ b/cms/io/BackgroundScheduler.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Programming contest management system +# Copyright © 2026 Erik Sünderhauf +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +from sched import scheduler +from sys import exc_info +from traceback import format_exception +from typing import Callable + +logger = logging.getLogger(__name__) + + +class BackgroundScheduler(scheduler): + def __init__(self): + super().__init__() + + def every(self, interval: float, func: Callable, priority: int = 0, skip_first: bool = True, args=(), **kwargs) -> None: + def wrapped(): + try: + func(*args, **kwargs) + except Exception: + logger.error("Failed to execute background task", + "\n".join(format_exception(*exc_info()))) + finally: + self.enter(interval, priority, wrapped) + delay = 0 + if skip_first: + delay = interval + self.enter(delay, priority, wrapped) diff --git a/cms/io/TaskInfo.py b/cms/io/TaskInfo.py index 95ef6f8155..6caa7f38e3 100644 --- a/cms/io/TaskInfo.py +++ b/cms/io/TaskInfo.py @@ -36,6 +36,8 @@ from six import iteritems +from cms.io.BackgroundScheduler import BackgroundScheduler + logger = logging.getLogger(__name__) @@ -57,7 +59,7 @@ def __init__(self, code, path): info = {"code": code, "title": "???", "source": "N/A", - "algorithm": -1, + "algorithm": -1, "implementation": -1, "keywords": [], "uses": [], @@ -163,35 +165,34 @@ def load(directory, tasks): except: logger.info("\n".join(format_exception(*exc_info()))) - def main_loop(repository, tasks, waiting_time): + def update_tasklist(repository, tasks): directory = Path(repository.path) + start = time() + with repository: + # Remove tasks that are no longer available + for t in tasks.keys(): + info_path = directory / t - while True: - start = time() - - with repository: - # Remove tasks that are no longer available - for t in tasks.keys(): - info_path = directory / t - - if not info_path.exists(): - del tasks[t] + if not info_path.exists(): + del tasks[t] - load(directory, tasks) + load(directory, tasks) - logger.info("finished iteration of TaskInfo.main_loop in {}ms". - format(int(1000 * (time() - start)))) + logger.info("finished iteration of TaskInfo.update_tasklist in {}ms". + format(int(1000 * (time() - start)))) - sleep(waiting_time) + def run(*args): + scheduler = BackgroundScheduler() + scheduler.every(.5 * (1 + sqrt(5)), update_tasklist, args=args) + scheduler.run(blocking=True) # Load data once on start-up (otherwise tasks might get removed when # the server is restarted) with repository: load(Path(repository.path), TaskInfo.tasks) - TaskInfo.info_process = Process(target=main_loop, - args=(repository, TaskInfo.tasks, - .5 * (1 + sqrt(5)))) + TaskInfo.info_process = Process( + target=run, args=(repository, TaskInfo.tasks)) TaskInfo.info_process.daemon = True TaskInfo.info_process.start() diff --git a/cms/io/TaskTranslateInfo.py b/cms/io/TaskTranslateInfo.py index a2cbf31904..9ebabfb9ab 100644 --- a/cms/io/TaskTranslateInfo.py +++ b/cms/io/TaskTranslateInfo.py @@ -38,9 +38,13 @@ from six import iteritems from babel.core import Locale +from cms.io.BackgroundScheduler import BackgroundScheduler + logger = logging.getLogger(__name__) -#TODO Delete this from this file? +# TODO Delete this from this file? + + class DateEntry: def __init__(self, date_code, info): self.date = datetime.strptime(date_code, "%Y-%m-%d").date() @@ -85,7 +89,7 @@ def __init__(self, code, path, contest=""): if len(missing) > 0: i["error"] = "Some important entries are missing: " + \ - ", ".join(missing) + "." + ", ".join(missing) + "." info.update(i) @@ -101,7 +105,7 @@ def __init__(self, code, path, contest=""): info["locked"] = [l.name[:-5] for l in path.parent.iterdir() if l.is_file() and l.name.endswith(".lock")] - if info["filename"]==None: + if info["filename"] == None: info["filename"] = "statement" info["translated"] = [l.name[len(info["filename"])+1:-4] for l in path.parent.iterdir() @@ -137,9 +141,9 @@ class TaskTranslateInfo: def init(repository): def load_single(d, tasks, contests, is_contest): if not d.is_dir() \ - or d.name.startswith('.') \ - or (d.is_symlink() and d.name=="general")\ - or d.name=="build": + or d.name.startswith('.') \ + or (d.is_symlink() and d.name == "general")\ + or d.name == "build": return False # We catch all exceptions since the main loop must go on @@ -157,7 +161,8 @@ def load_single(d, tasks, contests, is_contest): else: contest = d.parts[-2] code = d.parts[-1] - info = SingleTaskTranslateInfo(code, info_path, contest).to_dict() + info = SingleTaskTranslateInfo( + code, info_path, contest).to_dict() old = tasks.get(code, {"timestamp": 0}) info["timestamp"] = old["timestamp"] @@ -178,8 +183,9 @@ def load(directory, tasks, contests, languages): logger.error("The languages.json file is missing.") return try: - #TODO Languages are never deleted (and neither are contests) - language_list = json.loads(open(languages_path).read())["languages"] + # TODO Languages are never deleted (and neither are contests) + language_list = json.loads(open(languages_path).read())[ + "languages"] for l in language_list: languages[l] = l @@ -194,41 +200,43 @@ def load(directory, tasks, contests, languages): for e in d.iterdir(): load_single(e, tasks, contests, is_contest=False) - def main_loop(repository, tasks, contests, languages, waiting_time): + def update_tasklist(repository, tasks, contests, languages): directory = Path(repository.path) - while True: - start = time() + start = time() - with repository: - # Remove tasks that are no longer available - for t in tasks.keys(): - if t.endswith("-overview"): - p = t[:-9] - elif tasks[t]["contest"] == "": - p = t - else: - p = Path(tasks[t]["contest"]) / t - info_path = directory / p + with repository: + # Remove tasks that are no longer available + for t in tasks.keys(): + if t.endswith("-overview"): + p = t[:-9] + elif tasks[t]["contest"] == "": + p = t + else: + p = Path(tasks[t]["contest"]) / t + info_path = directory / p - if not info_path.exists(): - del tasks[t] + if not info_path.exists(): + del tasks[t] - load(directory, tasks, contests, languages) + load(directory, tasks, contests, languages) - logger.info("finished iteration of TaskTranslateInfo.main_loop in {}ms". - format(int(1000 * (time() - start)))) + logger.info("finished iteration of TaskTranslateInfo.update_tasklist in {}ms". + format(int(1000 * (time() - start)))) - sleep(waiting_time) + def run(*args): + scheduler = BackgroundScheduler() + scheduler.every(.5 * (1 + sqrt(5)), update_tasklist, args=args) + scheduler.run(blocking=True) # Load data once on start-up (otherwise tasks might get removed when # the server is restarted) with repository: - load(Path(repository.path), TaskTranslateInfo.tasks, TaskTranslateInfo.contests, TaskTranslateInfo.languages) + load(Path(repository.path), TaskTranslateInfo.tasks, + TaskTranslateInfo.contests, TaskTranslateInfo.languages) - TaskTranslateInfo.info_process = Process(target=main_loop, - args=(repository, TaskTranslateInfo.tasks, TaskTranslateInfo.contests, TaskTranslateInfo.languages, - .5 * (1 + sqrt(5)))) + TaskTranslateInfo.info_process = Process(target=run, args=( + repository, TaskTranslateInfo.tasks, TaskTranslateInfo.contests, TaskTranslateInfo.languages,)) TaskTranslateInfo.info_process.daemon = True TaskTranslateInfo.info_process.start() @@ -253,31 +261,31 @@ def language_list(): def gertranslate_entries(): return ["contest", "code", "title", "keywords", "remarks", "pdf", "tex"] +\ - ["pdf-"+l for l in TaskTranslateInfo.languages] +\ - ["tex-"+l for l in TaskTranslateInfo.languages] +\ - ["log-"+l for l in TaskTranslateInfo.languages] +\ - ["upload-"+l for l in TaskTranslateInfo.languages] +\ - ["mark-"+l for l in TaskTranslateInfo.languages] +\ - ["pdf-ALL"] + ["pdf-"+l for l in TaskTranslateInfo.languages] +\ + ["tex-"+l for l in TaskTranslateInfo.languages] +\ + ["log-"+l for l in TaskTranslateInfo.languages] +\ + ["upload-"+l for l in TaskTranslateInfo.languages] +\ + ["mark-"+l for l in TaskTranslateInfo.languages] +\ + ["pdf-ALL"] @staticmethod def gertranslate_desc(): return { - **{"contest": "Contest", - "code": "Code", - "title": "Title", - "keywords": "Keywords", - "remarks": "Remarks / Changelog", - "pdf": "PDF [en]", - "tex": "TeX [en]", - "log": "Log [en]", - "pdf-ALL": "PDF [ALL]"}, - **{"pdf-"+l: "PDF ["+l+"]" for l in TaskTranslateInfo.languages}, - **{"tex-"+l: "TeX ["+l+"]" for l in TaskTranslateInfo.languages}, - **{"log-"+l: "Log ["+l+"]" for l in TaskTranslateInfo.languages}, - **{"upload-"+l: "Upload TeX ["+l+"]" for l in TaskTranslateInfo.languages}, - **{"mark-"+l: "Finalize ["+l+"]" for l in TaskTranslateInfo.languages} - } + **{"contest": "Contest", + "code": "Code", + "title": "Title", + "keywords": "Keywords", + "remarks": "Remarks / Changelog", + "pdf": "PDF [en]", + "tex": "TeX [en]", + "log": "Log [en]", + "pdf-ALL": "PDF [ALL]"}, + **{"pdf-"+l: "PDF ["+l+"]" for l in TaskTranslateInfo.languages}, + **{"tex-"+l: "TeX ["+l+"]" for l in TaskTranslateInfo.languages}, + **{"log-"+l: "Log ["+l+"]" for l in TaskTranslateInfo.languages}, + **{"upload-"+l: "Upload TeX ["+l+"]" for l in TaskTranslateInfo.languages}, + **{"mark-"+l: "Finalize ["+l+"]" for l in TaskTranslateInfo.languages} + } @staticmethod def languages_desc(): From 4bf5d36ffa74b7e52fa680903d11b77859082b21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20S=C3=BCnderhauf?= Date: Wed, 14 Jan 2026 17:37:41 +0100 Subject: [PATCH 2/6] allow for multiple tasks folders --- cms/conf.py | 2 + cms/io/TaskFetch.py | 41 +++++++++++++-------- cms/io/TaskInfo.py | 61 ++++++++++++++++++------------- cms/io/TaskTranslateInfo.py | 13 +++---- cms/server/taskoverview/server.py | 2 +- config/cms.conf.sample | 11 ++++-- 6 files changed, 76 insertions(+), 54 deletions(-) diff --git a/cms/conf.py b/cms/conf.py index e8dc452a4f..610995942c 100644 --- a/cms/conf.py +++ b/cms/conf.py @@ -192,6 +192,8 @@ def __init__(self): self.overview_listen_address = "127.0.0.1" self.overview_listen_port = 8891 self.task_repository = None + self.tasks_folders = [] + self.contests_folders = [] self.auto_sync = False self.max_compilations = 1000 diff --git a/cms/io/TaskFetch.py b/cms/io/TaskFetch.py index 11d22d9e0e..d0c6539fd2 100644 --- a/cms/io/TaskFetch.py +++ b/cms/io/TaskFetch.py @@ -22,14 +22,17 @@ from __future__ import unicode_literals import logging +from pathlib import Path import sys from sys import exc_info from traceback import format_exception from multiprocessing import Process, Manager +from typing import Optional from six import StringIO from ansi2html import Ansi2HTMLConverter +from cms.io.Repository import Repository from cms.io.TaskInfo import TaskInfo from cmscontrib.gerpythonformat.GerMakeTask import GerMakeTask @@ -39,9 +42,10 @@ class TaskCompileJob: - def __init__(self, repository, name, balancer): + def __init__(self, repository: Repository, name: str, task_folder: str, balancer): self.repository = repository self.name = name + self.task_folder = task_folder self.balancer = balancer self.current_handle = 1 @@ -61,10 +65,10 @@ def join(self): def _compile(self): self._reset_status() - logger.info("loading task {} in {}".format(self.name, - self.repository.path)) + directory = Path(self.repository.path) / self.task_folder + logger.info("loading task {} in {}".format(self.name, str(directory))) - def do(status, repository, balancer): + def do(status, repository: Repository, directory: str, balancer): # stdout is process local in Python, so we can simply use this # to redirect all output from GerMakeTask to a string sys.stdout = StringIO() @@ -73,16 +77,16 @@ def do(status, repository, balancer): with balancer: try: - comp = GerMakeTask(odir = repository.path, - task = self.name, - minimal = True, - no_test = True, - submission = None, - no_latex = False, - verbose_latex = True, - language = None, - clean = False, - ntcimp = True) + comp = GerMakeTask(odir=directory, + task=self.name, + minimal=True, + no_test=True, + submission=None, + no_latex=False, + verbose_latex=True, + language=None, + clean=False, + ntcimp=True) with repository: comp.prepare() @@ -112,6 +116,7 @@ def do(status, repository, balancer): self.compilation_process = Process(target=do, args=(self.status, self.repository, + str(directory), self.balancer)) self.compilation_process.daemon = True self.compilation_process.start() @@ -174,7 +179,7 @@ def get(self): class TaskFetch: jobs = {} - repository = None + repository: Optional[Repository] = None balancer = None @staticmethod @@ -187,10 +192,14 @@ def init(repository, max_compilations): @staticmethod def compile(name): + if TaskFetch.repository is None: + raise Exception("tasks repository not initialized") if name not in TaskInfo.tasks: raise KeyError("No such task") if name not in TaskFetch.jobs: - TaskFetch.jobs[name] = TaskCompileJob(TaskFetch.repository, name, + TaskFetch.jobs[name] = TaskCompileJob(TaskFetch.repository, + name, + TaskInfo.tasks[name]["folder"], TaskFetch.balancer) return TaskFetch.jobs[name].join() diff --git a/cms/io/TaskInfo.py b/cms/io/TaskInfo.py index 6caa7f38e3..74f2956700 100644 --- a/cms/io/TaskInfo.py +++ b/cms/io/TaskInfo.py @@ -30,13 +30,15 @@ from multiprocessing import Process, Manager from sys import exc_info from traceback import format_exception -from time import sleep, time +from time import time from copy import deepcopy from math import sqrt +from typing import Collection from six import iteritems from cms.io.BackgroundScheduler import BackgroundScheduler +from cms.io.Repository import Repository logger = logging.getLogger(__name__) @@ -55,7 +57,7 @@ def to_dict(self): class SingleTaskInfo: - def __init__(self, code, path): + def __init__(self, code: str, path: Path): info = {"code": code, "title": "???", "source": "N/A", @@ -141,42 +143,49 @@ class TaskInfo: tasks = Manager().dict() @staticmethod - def init(repository): - def load(directory, tasks): - # Load all available tasks - for d in directory.iterdir(): - if not d.is_dir(): + def init(repository: Repository, tasks_folders: Collection[str]): + def load(repository_root: Path, tasks_folders: Collection[str], tasks): + for folder in tasks_folders: + directory = repository_root / folder + if not directory.exists() or not directory.is_dir(): + logger.warn("Task folder {} does not exist or is not" + " a directory, skipping.".format(directory)) continue + # Load all available tasks + for d in directory.iterdir(): + if not d.is_dir(): + continue - # We catch all exceptions since the main loop must go on - try: - info_path = d / "info.json" + # We catch all exceptions since the main loop must go on + try: + info_path = d / "info.json" - code = d.parts[-1] - info = SingleTaskInfo(code, info_path).to_dict() + code = d.parts[-1] + info = SingleTaskInfo(code, info_path).to_dict() - old = tasks.get(code, {"timestamp": 0}) - info["timestamp"] = old["timestamp"] + old = tasks.get(code, {"timestamp": 0}) + info["timestamp"] = old["timestamp"] + info["folder"] = folder - if old != info: - info["timestamp"] = time() - tasks[code] = info + if old != info: + info["timestamp"] = time() + tasks[code] = info - except: - logger.info("\n".join(format_exception(*exc_info()))) + except: + logger.info("\n".join(format_exception(*exc_info()))) - def update_tasklist(repository, tasks): - directory = Path(repository.path) + def update_tasklist(repository: Repository, tasks_folders: Collection[str], tasks): + repository_root = Path(repository.path) start = time() with repository: # Remove tasks that are no longer available for t in tasks.keys(): - info_path = directory / t + info_path = repository_root / tasks[t]["folder"] / t if not info_path.exists(): del tasks[t] - load(directory, tasks) + load(repository_root, tasks_folders, tasks) logger.info("finished iteration of TaskInfo.update_tasklist in {}ms". format(int(1000 * (time() - start)))) @@ -189,10 +198,10 @@ def run(*args): # Load data once on start-up (otherwise tasks might get removed when # the server is restarted) with repository: - load(Path(repository.path), TaskInfo.tasks) + load(Path(repository.path), tasks_folders, TaskInfo.tasks) - TaskInfo.info_process = Process( - target=run, args=(repository, TaskInfo.tasks)) + TaskInfo.info_process = Process(target=run, + args=(repository, tasks_folders, TaskInfo.tasks)) TaskInfo.info_process.daemon = True TaskInfo.info_process.start() diff --git a/cms/io/TaskTranslateInfo.py b/cms/io/TaskTranslateInfo.py index 9ebabfb9ab..dc4db56687 100644 --- a/cms/io/TaskTranslateInfo.py +++ b/cms/io/TaskTranslateInfo.py @@ -31,7 +31,7 @@ from multiprocessing import Process, Manager from sys import exc_info from traceback import format_exception -from time import sleep, time +from time import time from copy import deepcopy from math import sqrt @@ -42,9 +42,7 @@ logger = logging.getLogger(__name__) -# TODO Delete this from this file? - - +#TODO Delete this from this file? class DateEntry: def __init__(self, date_code, info): self.date = datetime.strptime(date_code, "%Y-%m-%d").date() @@ -89,7 +87,7 @@ def __init__(self, code, path, contest=""): if len(missing) > 0: i["error"] = "Some important entries are missing: " + \ - ", ".join(missing) + "." + ", ".join(missing) + "." info.update(i) @@ -183,9 +181,8 @@ def load(directory, tasks, contests, languages): logger.error("The languages.json file is missing.") return try: - # TODO Languages are never deleted (and neither are contests) - language_list = json.loads(open(languages_path).read())[ - "languages"] + #TODO Languages are never deleted (and neither are contests) + language_list = json.loads(open(languages_path).read())["languages"] for l in language_list: languages[l] = l diff --git a/cms/server/taskoverview/server.py b/cms/server/taskoverview/server.py index a66737c796..9a1e625c71 100644 --- a/cms/server/taskoverview/server.py +++ b/cms/server/taskoverview/server.py @@ -109,7 +109,7 @@ def __init__(self): repository = Repository(config.task_repository, config.auto_sync) TaskFetch.init(repository, config.max_compilations) - TaskInfo.init(repository) + TaskInfo.init(repository, config.tasks_folders) self.app = Application(handlers, **params) diff --git a/config/cms.conf.sample b/config/cms.conf.sample index 872bf7674c..dff311181b 100644 --- a/config/cms.conf.sample +++ b/config/cms.conf.sample @@ -192,11 +192,16 @@ "_section": "TaskOverviewWebServer", "_help": "Listening port and address for TaskOverviewWebServer.", - "listen_address": "127.0.0.1", - "listen_port": 8891, + "overview_listen_address": "0.0.0.0", + "overview_listen_port": 8891, - "_help": "Directory to search for tasks (you have to fill this yourself!).", + "_help": "Repository to search for tasks (you have to fill this yourself!).", "task_repository": "", + "_help": "Directories relative to the repository to search for tasks.", + "tasks_folders": ["tasks"], + "_help": "Directories relative to the repository where contests are configured.", + "_help": "Used to automatically collect usage data for tasks.", + "contests_folders": ["contests"], "_help": "Whether to automatically sync the task repository (only use this", "_help": "for git repositories).", From 84baef61edd605a0ee8230a8e8e56b3454139ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20S=C3=BCnderhauf?= Date: Sat, 17 Jan 2026 03:08:40 +0100 Subject: [PATCH 3/6] detect missing usages --- cms/io/TaskInfo.py | 266 ++++++++++++++++---- cms/server/taskoverview/server.py | 2 +- cmscontrib/gerpythonformat/ContestConfig.py | 37 ++- 3 files changed, 259 insertions(+), 46 deletions(-) diff --git a/cms/io/TaskInfo.py b/cms/io/TaskInfo.py index 74f2956700..9aa5603458 100644 --- a/cms/io/TaskInfo.py +++ b/cms/io/TaskInfo.py @@ -22,8 +22,10 @@ from __future__ import print_function from __future__ import unicode_literals +import contextlib import json import logging +import re from datetime import datetime from pathlib import Path @@ -39,6 +41,8 @@ from cms.io.BackgroundScheduler import BackgroundScheduler from cms.io.Repository import Repository +from cmscontrib.gerpythonformat.ContestConfig import ContestConfig +from cmscontrib.gerpythonformat.LocationStack import chdir logger = logging.getLogger(__name__) @@ -58,16 +62,19 @@ def to_dict(self): class SingleTaskInfo: def __init__(self, code: str, path: Path): - info = {"code": code, - "title": "???", - "source": "N/A", - "algorithm": -1, - "implementation": -1, - "keywords": [], - "uses": [], - "remarks": "", - "public": False, - "old": None} + self.code = code + self.title = "???" + self.source = "N/A" + self.algorithm = -1 + self.implementation = -1 + self.keywords = [] + self.uses: Collection[DateEntry] = [] + self.remarks = "" + self.public = False + self.old = None + self.folder = "" + self.timestamp = 0 + self.error = None if not path.exists(): i = {"error": "The info.json file is missing."} @@ -104,36 +111,40 @@ def bad(x): i["error"] = "Invalid value for \"implementation\": " \ "expected an integer between 0 and 10." - info.update(i) + self.update(i) - if info["old"] is None: - info["old"] = len(info["uses"]) > 0 or info["public"] + if self.old is None: + self.old = len(self.uses) > 0 or self.public - try: - info["uses"] = [DateEntry(e[0], e[1]) for e in info["uses"]] - except ValueError: - info["uses"] = [] + def update(self, i): + for key, value in iteritems(i): + if key == "uses": + try: + self.uses = [DateEntry(e[0], e[1]) for e in i["uses"]] + except ValueError: + self.uses = [] - if "error" not in info: - info["error"] = "I couldn't parse the dates for " \ - "\"(previous) uses\"." - - for key, value in iteritems(info): - setattr(self, key, value) + if self.error is not None: + self.error = "I couldn't parse the dates for " \ + "\"(previous) uses\"." + else: + setattr(self, key, value) def to_dict(self): - result = {"code": self.code, - "title": self.title, - "source": self.source, - "algorithm": self.algorithm, - "implementation": self.implementation, - "keywords": self.keywords, - "uses": [e.to_dict() for e in self.uses], - "remarks": self.remarks, - "public": self.public, - "old": self.old} - - if hasattr(self, "error"): + result = { + "code": self.code, + "title": self.title, + "source": self.source, + "algorithm": self.algorithm, + "implementation": self.implementation, + "keywords": self.keywords, + "uses": [e.to_dict() for e in self.uses], + "remarks": self.remarks, + "public": self.public, + "old": self.old, + } + + if self.error is not None: result["error"] = self.error return result @@ -143,13 +154,19 @@ class TaskInfo: tasks = Manager().dict() @staticmethod - def init(repository: Repository, tasks_folders: Collection[str]): + def init( + repository: Repository, + tasks_folders: Collection[str], + contests_folders: Collection[str], + ): def load(repository_root: Path, tasks_folders: Collection[str], tasks): for folder in tasks_folders: directory = repository_root / folder if not directory.exists() or not directory.is_dir(): - logger.warn("Task folder {} does not exist or is not" - " a directory, skipping.".format(directory)) + logger.warning( + 'Task folder "{}" does not exist or is not ' + "a directory, skipping.".format(directory) + ) continue # Load all available tasks for d in directory.iterdir(): @@ -190,9 +207,172 @@ def update_tasklist(repository: Repository, tasks_folders: Collection[str], task logger.info("finished iteration of TaskInfo.update_tasklist in {}ms". format(int(1000 * (time() - start)))) - def run(*args): + def is_same_contest( + contest_code: str, contestconfig: ContestConfig, info + ) -> bool: + if contestconfig.defaultgroup is None: + return False + start = contestconfig.defaultgroup.start.toordinal() + stop = contestconfig.defaultgroup.stop.toordinal() + if start <= info["timestamp"] and info["timestamp"] <= stop: + return True + # date in old info.json files might be off by a few days + # => try to match info.info with contest_code / config + if info["timestamp"] < start - 42 or stop + 42 < info["timestamp"]: + return False + description = re.sub(r" \d+$", "", info["info"]).lower() + clean_info = make_clean_info(contest_code).lower() + if (clean_info in description) or (description in clean_info): + return True + + olympiad = re.search( + r"^(.{1,4}oi)\d+[-_].*", contest_code, flags=re.IGNORECASE + ) + if olympiad is None: + return False + olympiad = olympiad.group() + if hasattr(contestconfig, "_short_name"): + short_name = re.sub(r" \d+$", "", contestconfig._short_name).lower() + if (short_name in description) or (description in short_name): + return True + return False + + def should_track_usage(contest_code: str) -> bool: + """Returns wheter the usage of a task in this contest should be tracked + in the info.json + + """ + if "test" in contest_code.lower(): + # ignore technical tests + return False + return True + + def make_clean_info(contest_code: str) -> str: + """Returns a "clean" description of the contest from its + contest_code to be used in the weboverview. + For example "ioiYYYY_1-1" => "lg1" + + """ + match = re.match( + r"^(.{1,4}oi)\d+[-_](.*)", contest_code, flags=re.IGNORECASE + ) + if match is None: + return contest_code + olympiad, usage = map(lambda s: s.lower(), match.groups()) + # remove enumerating suffix "-1", "-2", ..., usually campname-dayX + usage = re.sub(r"[-_]?\d$", "", usage) + info = "" + if olympiad == "ioi" and re.match(r"\d", usage): + info = olympiad + " lg" + usage + else: + info = olympiad + " " + usage + return info + + def update_usage( + repository: Repository, + contests_folders: Collection[str], + tasks, + ): + repository_root = Path(repository.path) + start = time() + with repository: + for folder in contests_folders: + directory = repository_root / folder + if not directory.exists() or not directory.is_dir(): + logger.warning( + 'Contest folder "{}" does not exist or is not' + " a directory, skipping.".format(directory) + ) + continue + for d in directory.iterdir(): + try: + if not d.is_dir(): + continue + contest_code = d.parts[-1] + if not should_track_usage(contest_code): + continue + config_path = d / "contest-config.py" + if not config_path.exists(): + logger.info( + 'Contest "{}" has no contest-config,' + " ignoring.".format(d) + ) + continue + with chdir(d): + contestconfig = ContestConfig( + d / ".rules", + contest_code, + ignore_latex=True, + minimal=True, + ) + with contextlib.redirect_stdout(None): + # _parseconfig doesn't perform any actions, so we suppress + # log messages like "Creating ..." + contestconfig._parseconfig("contest-config.py") + if contestconfig.defaultgroup is None: + logger.info( + 'Contest "{}" has no default group, ' + "ignoring.".format(d) + ) + continue + # only update the usage after the contest ended + if datetime.now() < contestconfig.defaultgroup.stop: + continue + for t in contestconfig.tasks.keys(): + task_code = t + task_path = d / t + # if the symlink to the task got deleted, we still try to + # update usage with the task code that was used in the contest + if task_path.exists(): + task_code = task_path.resolve().parts[-1] + if task_code not in tasks: + continue + task_info = tasks[task_code] + if any( + is_same_contest(contest_code, contestconfig, x) + for x in task_info["uses"] + ): + continue + contest_day = contestconfig.defaultgroup.start.strftime( + "%Y-%m-%d" + ) + info = make_clean_info(contest_code) + task_info["uses"].append( + DateEntry(contest_day, info).to_dict() + ) + logger.info( + "found usage for task {} in {} ({})".format( + task_code, contest_code, info + ) + ) + except: + logger.error("Failed to process contest {}".format(d)) + logger.error("\n".join(format_exception(*exc_info()))) + + load(repository_root, tasks_folders, tasks) + + logger.info( + "finished iteration of TaskInfo.update_usage in {}ms".format( + int(1000 * (time() - start)) + ) + ) + + def run(repository, tasks_folders, contests_folders, tasks): scheduler = BackgroundScheduler() - scheduler.every(.5 * (1 + sqrt(5)), update_tasklist, args=args) + scheduler.every( + 0.5 * (1 + sqrt(5)), + update_tasklist, + args=(repository, tasks_folders, tasks), + ) + # Usage data will be updated after a contest ended + # so we can query less frequent + scheduler.every( + 3600, + update_usage, + args=(repository, contests_folders, tasks), + skip_first=False, + priority=1, + ) scheduler.run(blocking=True) # Load data once on start-up (otherwise tasks might get removed when @@ -200,8 +380,10 @@ def run(*args): with repository: load(Path(repository.path), tasks_folders, TaskInfo.tasks) - TaskInfo.info_process = Process(target=run, - args=(repository, tasks_folders, TaskInfo.tasks)) + TaskInfo.info_process = Process( + target=run, + args=(repository, tasks_folders, contests_folders, TaskInfo.tasks), + ) TaskInfo.info_process.daemon = True TaskInfo.info_process.start() diff --git a/cms/server/taskoverview/server.py b/cms/server/taskoverview/server.py index 9a1e625c71..d71fc04617 100644 --- a/cms/server/taskoverview/server.py +++ b/cms/server/taskoverview/server.py @@ -109,7 +109,7 @@ def __init__(self): repository = Repository(config.task_repository, config.auto_sync) TaskFetch.init(repository, config.max_compilations) - TaskInfo.init(repository, config.tasks_folders) + TaskInfo.init(repository, config.tasks_folders, config.contests_folders) self.app = Application(handlers, **params) diff --git a/cmscontrib/gerpythonformat/ContestConfig.py b/cmscontrib/gerpythonformat/ContestConfig.py index c6477403a5..dfbee5593b 100644 --- a/cmscontrib/gerpythonformat/ContestConfig.py +++ b/cmscontrib/gerpythonformat/ContestConfig.py @@ -165,9 +165,11 @@ def __init__(self, rules, name, ignore_latex=False, verbose_latex=False, # a standard tokenwise comparator (specified here so that it has to be # compiled at most once per contest) - shutil.copy(os.path.join(self._get_ready_dir(), "tokens.cpp"), - "tokens.cpp") - self.token_equ_fp = self.compile("tokens") + if not self.minimal: + shutil.copy(os.path.join(self._get_ready_dir(), "tokens.cpp"), "tokens.cpp") + self.token_equ_fp = self.compile("tokens") + else: + self.token_equ_fp = None # there is no upstream for contest self.bequeathing = False @@ -187,6 +189,35 @@ def _readconfig(self, filename): super(ContestConfig, self)._readconfig(filename) self._initialize_ranking() + def _parseconfig(self, filename): + """attempts to parse a contest-config file and extract metadata without + performing any build/compile tasks + + """ + + def dummy_func(*args, **kwargs): + pass + + original_exported = copy.deepcopy(self.exported) + for k in self.exported: + if k not in ["time", "timezone", "team", "user_group"]: + self.exported[k] = dummy_func + + def new_load_template(name, *args, **kwargs): + if "short_name" in kwargs: + setattr(self, "_short_name", kwargs["short_name"]) + + def new_task(s, *args, **kwargs): + self.tasks[s] = "" + + self.exported["task"] = new_task + self.exported["load_template"] = new_load_template + try: + super(ContestConfig, self)._readconfig(filename) + except Exception as e: + print("Failed to parse contest-config with error", e) + self.exported = original_exported + def finish(self): asy_cnt = self.asy_warnings + sum(t.asy_warnings for t in self.tasks.values()) From f49d9abd0f2abf1f52d07724c0ab78828e6699f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20S=C3=BCnderhauf?= Date: Sun, 18 Jan 2026 04:27:00 +0100 Subject: [PATCH 4/6] auto commit changes and track updates on disk --- cms/io/Repository.py | 15 ++-- cms/io/TaskAccess.py | 21 +++-- cms/io/TaskInfo.py | 98 +++++++++++++++++---- cmscontrib/gerpythonformat/ContestConfig.py | 2 +- 4 files changed, 106 insertions(+), 30 deletions(-) diff --git a/cms/io/Repository.py b/cms/io/Repository.py index ac994b07b6..1587b6a67a 100644 --- a/cms/io/Repository.py +++ b/cms/io/Repository.py @@ -91,9 +91,11 @@ def _push(self): logger.info("Finished pushing: " + "{}".format(gitout)) - # For GerTranslate + # For GerTranslate/cmsTaskOverview # TODO Show errors in web overview - def commit(self, file_path, file_identifier): + def commit(self, file_path, commit_message="", author=""): + if commit_message == "" or author == "": + raise Exception("Missing commit message or author") # TODO Only do this if it's a git repository # if self.auto_sync: logger.info("Committing {} in {}".format(file_path, self.path)) @@ -117,13 +119,8 @@ def commit(self, file_path, file_identifier): check_output( ["git", "commit", "-o", file_path, - # TODO Provide meaningful commit message and - # author - "-m", "Changes to " + - file_identifier + - ", uploaded via GerTranslate web " - "interface", - "--author", '"GerTranslate "'] + "-m", commit_message, + "--author", author] ) except: logger.error("Couldn't commit in repository: " + diff --git a/cms/io/TaskAccess.py b/cms/io/TaskAccess.py index a22d7ce403..bacf115cbb 100644 --- a/cms/io/TaskAccess.py +++ b/cms/io/TaskAccess.py @@ -34,6 +34,7 @@ from six import StringIO from ansi2html import Ansi2HTMLConverter +from cms.io.Repository import Repository from cms.io.TaskTranslateInfo import TaskTranslateInfo from cmscontrib.gerpythonformat.ContestConfig import MyGroup @@ -333,7 +334,7 @@ def get(self): class TaskTeXReceiver: def __init__(self, repository, name): - self.repository = repository + self.repository: Repository = repository self.name = name def receive(self, f): @@ -364,15 +365,21 @@ def receive(self, f): else: with open(tex_file, "wb") as target_file: target_file.write(f) + # TODO Provide meaningful commit message and + # author self.repository.commit( - str(tex_file.resolve()), str(_repository_code)) + str(tex_file.resolve()), + commit_message=f"Changes to {str(_repository_code)}, " + "uploaded via GerTranslate web interface", + author='"GerTranslate "', + ) return result class TaskMarker: def __init__(self, repository, name): - self.repository = repository + self.repository: Repository = repository self.name = name def mark(self): @@ -385,8 +392,12 @@ def mark(self): with open(lock_file, "w") as target_file: target_file.write("The translation in this language is locked.") - self.repository.commit(str(lock_file.resolve()), - str(_repository_lock_file_code)) + self.repository.commit( + str(lock_file.resolve()), + commit_message=f"Changes to {str(_repository_lock_file_code)}, " + "uploaded via GerTranslate web interface", + author='"GerTranslate "', + ) class TaskGitLog: diff --git a/cms/io/TaskInfo.py b/cms/io/TaskInfo.py index 9aa5603458..e630bfdf49 100644 --- a/cms/io/TaskInfo.py +++ b/cms/io/TaskInfo.py @@ -81,7 +81,7 @@ def __init__(self, code: str, path: Path): else: try: i = json.loads(path.open().read()) - except: + except Exception: i = {"error": "The info.json file is corrupt."} else: missing = [] @@ -96,7 +96,7 @@ def __init__(self, code: str, path: Path): def bad(x): try: y = int(x) - except: + except Exception: return True else: return y < 0 or y > 10 @@ -188,7 +188,7 @@ def load(repository_root: Path, tasks_folders: Collection[str], tasks): info["timestamp"] = time() tasks[code] = info - except: + except Exception: logger.info("\n".join(format_exception(*exc_info()))) def update_tasklist(repository: Repository, tasks_folders: Collection[str], tasks): @@ -220,7 +220,7 @@ def is_same_contest( # => try to match info.info with contest_code / config if info["timestamp"] < start - 42 or stop + 42 < info["timestamp"]: return False - description = re.sub(r" \d+$", "", info["info"]).lower() + description = re.sub(r"\s+\d+$", "", info["info"]).lower() clean_info = make_clean_info(contest_code).lower() if (clean_info in description) or (description in clean_info): return True @@ -232,7 +232,7 @@ def is_same_contest( return False olympiad = olympiad.group() if hasattr(contestconfig, "_short_name"): - short_name = re.sub(r" \d+$", "", contestconfig._short_name).lower() + short_name = re.sub(r"\s+\d+$", "", contestconfig._short_name).lower() if (short_name in description) or (description in short_name): return True return False @@ -272,10 +272,37 @@ def update_usage( repository: Repository, contests_folders: Collection[str], tasks, - ): + ) -> None: repository_root = Path(repository.path) start = time() with repository: + old_processed = {} + try: + with open(repository_root / ".unconfirmed_usage.json", "r") as f: + data = f.read().splitlines() + for line in data: + parsed_line = json.loads(line) + if "task" not in parsed_line or "uses" not in parsed_line: + logger.warning("skipping corrupted line {}".format(line)) + continue + task_code = parsed_line["task"] + if task_code not in old_processed: + old_processed[task_code] = [] + confirmed = ( + parsed_line["confirmed"] + if "confirmed" in parsed_line + else False + ) + old_processed[task_code].append( + {"uses": parsed_line["uses"], "confirmed": confirmed} + ) + except FileNotFoundError: + pass + except Exception: + logger.warning("file .unconfirmed_usage.json is corrupt") + logger.warning("\n".join(format_exception(*exc_info()))) + logger.info("loaded old data: {}".format(old_processed)) + untracked = [] for folder in contests_folders: directory = repository_root / folder if not directory.exists() or not directory.is_dir(): @@ -332,24 +359,65 @@ def update_usage( is_same_contest(contest_code, contestconfig, x) for x in task_info["uses"] ): + # usage is already stored in the info.json + continue + if task_code in old_processed and any( + is_same_contest(contest_code, contestconfig, x["uses"]) + for x in old_processed[task_code] + ): + # at this point the usage was detected and stored + # in a previous run, but marked as false positive, + # so we ignore it continue contest_day = contestconfig.defaultgroup.start.strftime( "%Y-%m-%d" ) info = make_clean_info(contest_code) - task_info["uses"].append( - DateEntry(contest_day, info).to_dict() - ) + usage = DateEntry(contest_day, info).to_dict() + untracked.append({"task": task_code, "usage": usage}) + task_info["uses"].append(usage) + task_info["timestamp"] = time() logger.info( - "found usage for task {} in {} ({})".format( - task_code, contest_code, info - ) + f"found usage for task {task_code} in {info}" ) - except: + except Exception: logger.error("Failed to process contest {}".format(d)) logger.error("\n".join(format_exception(*exc_info()))) - - load(repository_root, tasks_folders, tasks) + for v in untracked: + try: + task_code, usage = v["task"], v["usage"] + task = tasks[task_code] + usage_time = datetime.fromordinal(usage["timestamp"]) + usage_time = usage_time.strftime("%Y-%m-%d") + # remove trailing year + usage_info = re.sub(r"\s+\d+$", "", usage["info"]) + info_path = ( + repository_root + / Path(task["folder"]) + / task_code + / "info.json" + ) + info_path = info_path.resolve() + old_data = json.loads(info_path.open().read()) + if "uses" not in old_data: + old_data["uses"] = [] + old_data["uses"].append([usage_time, usage_info]) + open(info_path, "w").write(json.dumps(old_data, indent=4)) + repository.commit( + str(info_path.resolve()), + commit_message=f"Add usage in contest {usage['info']} " + "to task {task_code}, from TaskOverviewBackend", + author='"cmsTaskOverviewWebserver "', + ) + with open(repository_root / ".unconfirmed_usage.json", "a") as f: + f.write( + f'{{"task":"{task_code}", \ + "uses":["{usage_time}", "{usage_info}"], \ + "confirmed":false}}\n' + ) + except Exception: + logger.error("Failed to update task {}".format(task_code)) + logger.error("\n".join(format_exception(*exc_info()))) logger.info( "finished iteration of TaskInfo.update_usage in {}ms".format( diff --git a/cmscontrib/gerpythonformat/ContestConfig.py b/cmscontrib/gerpythonformat/ContestConfig.py index dfbee5593b..9ce028c50c 100644 --- a/cmscontrib/gerpythonformat/ContestConfig.py +++ b/cmscontrib/gerpythonformat/ContestConfig.py @@ -165,7 +165,7 @@ def __init__(self, rules, name, ignore_latex=False, verbose_latex=False, # a standard tokenwise comparator (specified here so that it has to be # compiled at most once per contest) - if not self.minimal: + if not minimal: shutil.copy(os.path.join(self._get_ready_dir(), "tokens.cpp"), "tokens.cpp") self.token_equ_fp = self.compile("tokens") else: From e0d65ea02a33c09a7afd8e52b1e8dcbbbae8316c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20S=C3=BCnderhauf?= Date: Tue, 17 Feb 2026 23:46:09 +0100 Subject: [PATCH 5/6] auto push and better error handling --- cms/io/Repository.py | 56 +++--- cms/io/TaskInfo.py | 323 +++++++++++++++++------------- cms/server/taskoverview/server.py | 4 +- 3 files changed, 214 insertions(+), 169 deletions(-) diff --git a/cms/io/Repository.py b/cms/io/Repository.py index 1587b6a67a..5ac420ccbe 100644 --- a/cms/io/Repository.py +++ b/cms/io/Repository.py @@ -26,6 +26,7 @@ from multiprocessing import Manager from subprocess import check_output +from typing import Optional from cmscontrib.gerpythonformat.LocationStack import chdir @@ -93,7 +94,11 @@ def _push(self): # For GerTranslate/cmsTaskOverview # TODO Show errors in web overview - def commit(self, file_path, commit_message="", author=""): + def commit(self, file_path, commit_message="", author="") -> Optional[str]: + '''Commits the changes in file_path as `author` with given `commit_message`. + If an error occurs, the error message will be returned. + + ''' if commit_message == "" or author == "": raise Exception("Missing commit message or author") # TODO Only do this if it's a git repository @@ -102,32 +107,33 @@ def commit(self, file_path, commit_message="", author=""): with chdir(self.path): gitout = "" - try: - gitout = check_output(["git", "add", - file_path]) - except: - logger.error("Couldn't add file to git staging area: " + - "{}".format(gitout)) - else: + gitout = check_output(["git", "add", file_path]) + except Exception as e: + logger.error("Couldn't add file to git staging area: {}".format(e)) + return str(e) + try: + gitout = "" + # NOTE file_path is relative to self.path, which isn't + # necessarily the root of the git repo. So the commit + # message might be confusing. + gitout = \ + check_output( + ["git", "commit", + "-o", file_path, + "-m", commit_message, + "--author", author] + ) + except Exception as e: + logger.error("Couldn't commit in repository: {}".format(e)) try: - gitout = "" - # NOTE file_path is relative to self.path, which isn't - # necessarily the root of the git repo. So the commit - # message might be confusing. - gitout = \ - check_output( - ["git", "commit", - "-o", file_path, - "-m", commit_message, - "--author", author] - ) - except: - logger.error("Couldn't commit in repository: " + - "{}".format(gitout)) - else: - logger.info("Committed: " + - "{}".format(gitout)) + # try to unstage files if committing failed + check_output(["git", "restore", "--staged", file_path]) + except Exception as e: + logger.warning("unable to unstage files: {}".format(e)) + return str(e) + logger.info("Committed: {}".format(gitout)) + # For GerTranslate # TODO Show errors in web overview diff --git a/cms/io/TaskInfo.py b/cms/io/TaskInfo.py index e630bfdf49..e0cbff7c39 100644 --- a/cms/io/TaskInfo.py +++ b/cms/io/TaskInfo.py @@ -30,6 +30,7 @@ from datetime import datetime from pathlib import Path from multiprocessing import Process, Manager +from subprocess import check_output from sys import exc_info from traceback import format_exception from time import time @@ -152,6 +153,7 @@ def to_dict(self): class TaskInfo: tasks = Manager().dict() + unconfirmed_usage = Manager().dict() @staticmethod def init( @@ -207,6 +209,33 @@ def update_tasklist(repository: Repository, tasks_folders: Collection[str], task logger.info("finished iteration of TaskInfo.update_tasklist in {}ms". format(int(1000 * (time() - start)))) + def load_usage(repository_root: Path, unconfirmed_usage) -> None: + try: + with open(repository_root / ".unconfirmed_usage.json", "r") as f: + data = f.read().splitlines() + unconfirmed_usage.clear() + for line in data: + parsed_line = json.loads(line) + if "task" not in parsed_line or "uses" not in parsed_line: + logger.warning("skipping corrupted line {}".format(line)) + continue + task_code = parsed_line["task"] + if task_code not in data: + unconfirmed_usage[task_code] = [] + confirmed = ( + parsed_line["confirmed"] + if "confirmed" in parsed_line + else False + ) + unconfirmed_usage[task_code].append( + {"uses": parsed_line["uses"], "confirmed": confirmed} + ) + except FileNotFoundError: + pass + except Exception: + logger.warning("file .unconfirmed_usage.json is corrupt") + logger.warning("\n".join(format_exception(*exc_info()))) + def is_same_contest( contest_code: str, contestconfig: ContestConfig, info ) -> bool: @@ -268,156 +297,158 @@ def make_clean_info(contest_code: str) -> str: info = olympiad + " " + usage return info - def update_usage( - repository: Repository, + def parse_contests( + repository_root: Path, contests_folders: Collection[str], tasks, - ) -> None: - repository_root = Path(repository.path) - start = time() - with repository: - old_processed = {} - try: - with open(repository_root / ".unconfirmed_usage.json", "r") as f: - data = f.read().splitlines() - for line in data: - parsed_line = json.loads(line) - if "task" not in parsed_line or "uses" not in parsed_line: - logger.warning("skipping corrupted line {}".format(line)) + unconfirmed_usage, + ) -> list: + untracked = [] + for folder in contests_folders: + directory = repository_root / folder + if not directory.exists() or not directory.is_dir(): + logger.warning( + 'Contest folder "{}" does not exist or is not' + " a directory, skipping.".format(directory) + ) + continue + for d in directory.iterdir(): + try: + if not d.is_dir(): continue - task_code = parsed_line["task"] - if task_code not in old_processed: - old_processed[task_code] = [] - confirmed = ( - parsed_line["confirmed"] - if "confirmed" in parsed_line - else False - ) - old_processed[task_code].append( - {"uses": parsed_line["uses"], "confirmed": confirmed} - ) - except FileNotFoundError: - pass - except Exception: - logger.warning("file .unconfirmed_usage.json is corrupt") - logger.warning("\n".join(format_exception(*exc_info()))) - logger.info("loaded old data: {}".format(old_processed)) - untracked = [] - for folder in contests_folders: - directory = repository_root / folder - if not directory.exists() or not directory.is_dir(): - logger.warning( - 'Contest folder "{}" does not exist or is not' - " a directory, skipping.".format(directory) - ) - continue - for d in directory.iterdir(): - try: - if not d.is_dir(): - continue - contest_code = d.parts[-1] - if not should_track_usage(contest_code): - continue - config_path = d / "contest-config.py" - if not config_path.exists(): - logger.info( - 'Contest "{}" has no contest-config,' - " ignoring.".format(d) + contest_code = d.parts[-1] + if not should_track_usage(contest_code): + continue + config_path = d / "contest-config.py" + if not config_path.exists(): + logger.info( + 'Contest "{}" has no contest-config, ignoring.'.format( + d ) + ) + continue + with chdir(d): + contestconfig = ContestConfig( + d / ".rules", + contest_code, + ignore_latex=True, + minimal=True, + ) + with contextlib.redirect_stdout(None): + # _parseconfig doesn't perform any actions, so we suppress + # log messages like "Creating ..." + contestconfig._parseconfig("contest-config.py") + if contestconfig.defaultgroup is None: + logger.info( + 'Contest "{}" has no default group, ignoring.'.format(d) + ) + continue + # only update the usage after the contest ended + if datetime.now() < contestconfig.defaultgroup.stop: + continue + for t in contestconfig.tasks.keys(): + task_code = t + task_path = d / t + # if the symlink to the task got deleted, we still try to + # update usage with the task code that was used in the contest + if task_path.exists(): + task_code = task_path.resolve().parts[-1] + if task_code not in tasks: continue - with chdir(d): - contestconfig = ContestConfig( - d / ".rules", - contest_code, - ignore_latex=True, - minimal=True, - ) - with contextlib.redirect_stdout(None): - # _parseconfig doesn't perform any actions, so we suppress - # log messages like "Creating ..." - contestconfig._parseconfig("contest-config.py") - if contestconfig.defaultgroup is None: - logger.info( - 'Contest "{}" has no default group, ' - "ignoring.".format(d) - ) + task_info = tasks[task_code] + if any( + is_same_contest(contest_code, contestconfig, x) + for x in task_info["uses"] + ): + # usage is already stored in the info.json continue - # only update the usage after the contest ended - if datetime.now() < contestconfig.defaultgroup.stop: + if task_code in unconfirmed_usage and any( + is_same_contest(contest_code, contestconfig, x["uses"]) + for x in unconfirmed_usage[task_code] + ): + # at this point the usage was detected and stored + # in a previous run, but marked as false positive, + # so we ignore it continue - for t in contestconfig.tasks.keys(): - task_code = t - task_path = d / t - # if the symlink to the task got deleted, we still try to - # update usage with the task code that was used in the contest - if task_path.exists(): - task_code = task_path.resolve().parts[-1] - if task_code not in tasks: - continue - task_info = tasks[task_code] - if any( - is_same_contest(contest_code, contestconfig, x) - for x in task_info["uses"] - ): - # usage is already stored in the info.json - continue - if task_code in old_processed and any( - is_same_contest(contest_code, contestconfig, x["uses"]) - for x in old_processed[task_code] - ): - # at this point the usage was detected and stored - # in a previous run, but marked as false positive, - # so we ignore it - continue - contest_day = contestconfig.defaultgroup.start.strftime( - "%Y-%m-%d" - ) - info = make_clean_info(contest_code) - usage = DateEntry(contest_day, info).to_dict() - untracked.append({"task": task_code, "usage": usage}) - task_info["uses"].append(usage) - task_info["timestamp"] = time() - logger.info( - f"found usage for task {task_code} in {info}" - ) - except Exception: - logger.error("Failed to process contest {}".format(d)) - logger.error("\n".join(format_exception(*exc_info()))) - for v in untracked: - try: - task_code, usage = v["task"], v["usage"] - task = tasks[task_code] - usage_time = datetime.fromordinal(usage["timestamp"]) - usage_time = usage_time.strftime("%Y-%m-%d") - # remove trailing year - usage_info = re.sub(r"\s+\d+$", "", usage["info"]) - info_path = ( - repository_root - / Path(task["folder"]) - / task_code - / "info.json" - ) - info_path = info_path.resolve() - old_data = json.loads(info_path.open().read()) - if "uses" not in old_data: - old_data["uses"] = [] - old_data["uses"].append([usage_time, usage_info]) - open(info_path, "w").write(json.dumps(old_data, indent=4)) - repository.commit( - str(info_path.resolve()), - commit_message=f"Add usage in contest {usage['info']} " - "to task {task_code}, from TaskOverviewBackend", - author='"cmsTaskOverviewWebserver "', - ) - with open(repository_root / ".unconfirmed_usage.json", "a") as f: - f.write( - f'{{"task":"{task_code}", \ - "uses":["{usage_time}", "{usage_info}"], \ - "confirmed":false}}\n' + contest_day = contestconfig.defaultgroup.start.strftime( + "%Y-%m-%d" ) + info = make_clean_info(contest_code) + usage = DateEntry(contest_day, info).to_dict() + untracked.append({"task": task_code, "usage": usage}) + task_info["uses"].append(usage) + task_info["timestamp"] = time() + logger.info(f"found usage for task {task_code} in {info}") except Exception: - logger.error("Failed to update task {}".format(task_code)) + logger.error("Failed to process contest {}".format(d)) logger.error("\n".join(format_exception(*exc_info()))) + return untracked + + def store_usage(repository: Repository, untracked: list, tasks) -> None: + repository_root = Path(repository.path) + for v in untracked: + try: + task_code, usage = v["task"], v["usage"] + task = tasks[task_code] + usage_time = datetime.fromordinal(usage["timestamp"]) + usage_time = usage_time.strftime("%Y-%m-%d") + # remove trailing year + usage_info = re.sub(r"\s+\d+$", "", usage["info"]) + info_path = ( + repository_root / Path(task["folder"]) / task_code / "info.json" + ) + info_path = info_path.resolve() + with open(info_path, "r") as f: + old_data = json.load(f) + if "uses" not in old_data: + old_data["uses"] = [] + old_data["uses"].append([usage_time, usage_info]) + old_data["uses"].sort() + with open(info_path, "w") as f: + json.dump(old_data, f, indent=4) + err = repository.commit( + str(info_path), + commit_message=f"Add usage in contest {usage['info']} " + f"to task {task_code}, from TaskOverviewBackend", + author='"cmsTaskOverviewWebserver "', + ) + if err is not None: + # committing changes failed, so we try to restore the changes + # and retry in the next iteration + try: + check_output(["git", "checkout", "--", str(info_path)]) + except Exception as e: + logger.error( + "restoring old version of {} failed with error {}".format( + info_path, e + ) + ) + continue + with open(repository_root / ".unconfirmed_usage.json", "a") as f: + f.write( + f'{{"task":"{task_code}", \ + "uses":["{usage_time}", "{usage_info}"], \ + "confirmed":false}}\n' + ) + except Exception: + logger.error("Failed to update task {}".format(task_code)) + logger.error("\n".join(format_exception(*exc_info()))) + + def update_usage( + repository: Repository, + contests_folders: Collection[str], + tasks, + unconfirmed_usage, + ) -> None: + repository_root = Path(repository.path) + start = time() + with repository: + load_usage(repository_root, unconfirmed_usage) + logger.info("loaded old data: {}".format(unconfirmed_usage)) + untracked = parse_contests( + repository_root, contests_folders, tasks, unconfirmed_usage + ) + store_usage(repository, untracked, tasks) logger.info( "finished iteration of TaskInfo.update_usage in {}ms".format( @@ -425,7 +456,7 @@ def update_usage( ) ) - def run(repository, tasks_folders, contests_folders, tasks): + def run(repository, tasks_folders, contests_folders, tasks, unconfirmed_usage): scheduler = BackgroundScheduler() scheduler.every( 0.5 * (1 + sqrt(5)), @@ -437,7 +468,7 @@ def run(repository, tasks_folders, contests_folders, tasks): scheduler.every( 3600, update_usage, - args=(repository, contests_folders, tasks), + args=(repository, contests_folders, tasks, unconfirmed_usage), skip_first=False, priority=1, ) @@ -450,7 +481,13 @@ def run(repository, tasks_folders, contests_folders, tasks): TaskInfo.info_process = Process( target=run, - args=(repository, tasks_folders, contests_folders, TaskInfo.tasks), + args=( + repository, + tasks_folders, + contests_folders, + TaskInfo.tasks, + TaskInfo.unconfirmed_usage, + ), ) TaskInfo.info_process.daemon = True TaskInfo.info_process.start() diff --git a/cms/server/taskoverview/server.py b/cms/server/taskoverview/server.py index d71fc04617..9c59934982 100644 --- a/cms/server/taskoverview/server.py +++ b/cms/server/taskoverview/server.py @@ -106,7 +106,9 @@ def __init__(self): "static_path": resource_filename("cms.server", "taskoverview/static")} - repository = Repository(config.task_repository, config.auto_sync) + repository = Repository( + config.task_repository, config.auto_sync, auto_push=True + ) TaskFetch.init(repository, config.max_compilations) TaskInfo.init(repository, config.tasks_folders, config.contests_folders) From 3d42124259937b5f5905592ca86b88d79eaa3784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20S=C3=BCnderhauf?= Date: Wed, 18 Feb 2026 02:30:33 +0100 Subject: [PATCH 6/6] bugfix data not loading, show data in UI --- cms/io/TaskInfo.py | 64 ++++-- cms/server/taskoverview/static/js/overview.js | 190 +++++++++--------- 2 files changed, 144 insertions(+), 110 deletions(-) diff --git a/cms/io/TaskInfo.py b/cms/io/TaskInfo.py index e0cbff7c39..232d59ef1a 100644 --- a/cms/io/TaskInfo.py +++ b/cms/io/TaskInfo.py @@ -220,16 +220,19 @@ def load_usage(repository_root: Path, unconfirmed_usage) -> None: logger.warning("skipping corrupted line {}".format(line)) continue task_code = parsed_line["task"] - if task_code not in data: + if task_code not in unconfirmed_usage: unconfirmed_usage[task_code] = [] confirmed = ( parsed_line["confirmed"] if "confirmed" in parsed_line else False ) - unconfirmed_usage[task_code].append( - {"uses": parsed_line["uses"], "confirmed": confirmed} - ) + unconfirmed_usage[task_code] += [ + { + "uses": DateEntry(*parsed_line["uses"]).to_dict(), + "confirmed": confirmed, + } + ] except FileNotFoundError: pass except Exception: @@ -261,7 +264,9 @@ def is_same_contest( return False olympiad = olympiad.group() if hasattr(contestconfig, "_short_name"): - short_name = re.sub(r"\s+\d+$", "", contestconfig._short_name).lower() + short_name = re.sub( + r"\s+\d+$", "", getattr(contestconfig, "_short_name") + ).lower() if (short_name in description) or (description in short_name): return True return False @@ -376,15 +381,15 @@ def parse_contests( info = make_clean_info(contest_code) usage = DateEntry(contest_day, info).to_dict() untracked.append({"task": task_code, "usage": usage}) - task_info["uses"].append(usage) - task_info["timestamp"] = time() logger.info(f"found usage for task {task_code} in {info}") except Exception: logger.error("Failed to process contest {}".format(d)) logger.error("\n".join(format_exception(*exc_info()))) return untracked - def store_usage(repository: Repository, untracked: list, tasks) -> None: + def store_usage( + repository: Repository, untracked: list, tasks, unconfirmed_usage + ) -> None: repository_root = Path(repository.path) for v in untracked: try: @@ -424,6 +429,13 @@ def store_usage(repository: Repository, untracked: list, tasks) -> None: ) ) continue + task["uses"] += [usage] + task["timestamp"] = time() + if task_code not in unconfirmed_usage: + unconfirmed_usage[task_code] = [] + unconfirmed_usage[task_code] += [ + {"uses": usage, "confirmed": False} + ] with open(repository_root / ".unconfirmed_usage.json", "a") as f: f.write( f'{{"task":"{task_code}", \ @@ -439,16 +451,17 @@ def update_usage( contests_folders: Collection[str], tasks, unconfirmed_usage, + only_load=False, ) -> None: repository_root = Path(repository.path) start = time() with repository: load_usage(repository_root, unconfirmed_usage) - logger.info("loaded old data: {}".format(unconfirmed_usage)) - untracked = parse_contests( - repository_root, contests_folders, tasks, unconfirmed_usage - ) - store_usage(repository, untracked, tasks) + if not only_load: + untracked = parse_contests( + repository_root, contests_folders, tasks, unconfirmed_usage + ) + store_usage(repository, untracked, tasks, unconfirmed_usage) logger.info( "finished iteration of TaskInfo.update_usage in {}ms".format( @@ -463,7 +476,13 @@ def run(repository, tasks_folders, contests_folders, tasks, unconfirmed_usage): update_tasklist, args=(repository, tasks_folders, tasks), ) - # Usage data will be updated after a contest ended + scheduler.every( + 0.5 * (1 + sqrt(5)), + update_usage, + args=(repository, contests_folders, tasks, unconfirmed_usage), + only_load=True, + ) + # Usage data will be updated only after a contest ended # so we can query less frequent scheduler.every( 3600, @@ -501,9 +520,20 @@ def task_list(): @staticmethod def get_info(keys): - data = deepcopy(TaskInfo.tasks) - - return {x: data[x] for x in keys if x in data} + task_data = deepcopy(TaskInfo.tasks) + usage_data = deepcopy(TaskInfo.unconfirmed_usage) + res = {} + for t in keys: + if t not in task_data: + continue + res[t] = task_data[t] + if t in usage_data: + for u in res[t]["uses"]: + for v in usage_data[t]: + if u == v["uses"]: + u["confirmed"] = v["confirmed"] + break + return res @staticmethod def entries(): diff --git a/cms/server/taskoverview/static/js/overview.js b/cms/server/taskoverview/static/js/overview.js index 3002867ba9..b9cc6339e5 100644 --- a/cms/server/taskoverview/static/js/overview.js +++ b/cms/server/taskoverview/static/js/overview.js @@ -4,29 +4,33 @@ function inner_cell(t, entry) { return '
'; } - + if(entry == "keywords") - { + { var r = '
    '; if(t[entry].length == 0) r += "—"; - + for(var i = 0; i < t[entry].length; ++i) r += '
  • ' + t[entry][i] + '
    • '; r += '
'; return r; } - + if(entry == "uses") { var r = '
    '; if(t[entry].length == 0) r += "—"; - - for(var i = 0; i < t[entry].length; ++i) - r += '
  • ' + t[entry][i].info + '
    • '; + + for(var i = 0; i < t[entry].length; ++i) { + r += '
  • ' + t[entry][i].info; + if (Object.hasOwn(t[entry][i], "confirmed") && t[entry][i].confirmed === false) + r += "*"; + r += '
    • '; + } r += '
'; return r; } - + else { return t[entry] @@ -65,7 +69,7 @@ function relevant(t, c) if("error" in t) return true; uses_ok = true; - + for(var i = 0; i < t.uses.length; ++i) if(t.uses[i].timestamp > criteria.only_before) uses_ok = false; @@ -81,173 +85,173 @@ var __info = {}; var __tasks = []; var __task_dict = {}; var __removed = {}; - + function build_row(task_code) { var t = __info[task_code]; var r = ""; - - r += ''; + + r += ''; r += cell(t, entries[0]); r += error_cell(t); - - for(var j = 1; j < entries.length; ++j) + + for(var j = 1; j < entries.length; ++j) r += cell(t, entries[j]); r += ''; - + return r; } - + function fill_table(new_tasks, updated_tasks, show_col, criteria, init) { new_tasks = new_tasks.sort(); - + if(init) { var table_body = ""; - + table_body += ''; for(var j = 0; j < entries.length; ++j) if(entries[j] == "download") table_body += '' + desc[entries[j]] + ''; // table-bordered doesn't work with th, so we emulate it else table_body += '' + desc[entries[j]] + ''; // table-bordered doesn't work with th, so we emulate it - table_body += ''; + table_body += ''; window.document.getElementById("overview").innerHTML = table_body; } - + function make_class_removal_function(id, cl) { return function() { window.document.getElementById(id).classList.remove(cl); } } - + // Insert new rows (à la mergesort) var last_entry = "overview-heading"; var old_tasks_idx = 0, new_tasks_idx = 0; - + while(true) { if(old_tasks_idx >= __tasks.length && new_tasks_idx >= new_tasks.length) break; - + if(old_tasks_idx >= __tasks.length || __tasks[old_tasks_idx] >= new_tasks[new_tasks_idx]) { $(build_row(new_tasks[new_tasks_idx], show_col)).insertAfter("#" + last_entry); init_download_icon(new_tasks[new_tasks_idx]) last_entry = "row-" + new_tasks[new_tasks_idx]; - + if(!init) { window.document.getElementById(last_entry).classList.add("fresh"); // We need to remove the attribute after the animation is done for other animations to be triggered -- moreover, it is semantically nicer window.document.getElementById(last_entry).addEventListener("animationend", make_class_removal_function(last_entry, "fresh")); - + } - + ++new_tasks_idx; } - + else { last_entry = "row-" + __tasks[old_tasks_idx]; ++old_tasks_idx; } } - + __tasks = __tasks.concat(new_tasks).sort(); - + var num_interesting_columns = 0; for(var j = 1; j < entries.length - 1; ++j) if(show_col[entries[j]]) ++num_interesting_columns; - + for(var i = 0; i < __tasks.length; ++i) { // Apply row selection var id = "row-" + __tasks[i]; var row = window.document.getElementById(id); - + if(relevant(__info[__tasks[i]], criteria)) row.classList.remove("hidden"); else row.classList.add("hidden"); - + // Remove tasks that are no longer available if(__tasks[i] in __removed) { row.classList.add("removed"); - + // Indirectness needed to make currying work function make_removal_function(s) { return function() { $("#" + s).remove(); } } - + window.setTimeout(make_removal_function("row-" + __tasks[i]), 1000); } - + // Highlight unused tasks if(__info[__tasks[i]].old || "error" in __info[__tasks[i]]) row.classList.remove("new"); else row.classList.add("new"); - - // Apply column selection + + // Apply column selection for(var j = 0; j < entries.length; ++j) { var cell = window.document.getElementById("cell-" + __tasks[i] + "-" + entries[j]); - + if(show_col[entries[j]]) cell.classList.remove("hidden"); else cell.classList.add("hidden"); } - + window.document.getElementById("cell-" + __tasks[i] + "-error-msg").colSpan = num_interesting_columns; - + // Error or correct entry? var t = __info[__tasks[i]]; - + if("error" in t) { for(var j = 1; j < entries.length - 1; ++j) { var cell = window.document.getElementById("cell-" + __tasks[i] + "-" + entries[j]); cell.classList.add("hidden"); - } - - window.document.getElementById("cell-" + __tasks[i] + "-error-msg").classList.remove("no-error"); + } + + window.document.getElementById("cell-" + __tasks[i] + "-error-msg").classList.remove("no-error"); } - + else { - window.document.getElementById("cell-" + __tasks[i] + "-error-msg").classList.add("no-error"); + window.document.getElementById("cell-" + __tasks[i] + "-error-msg").classList.add("no-error"); } } - + // Update tasks that changed since last query for(var i = 0; i < updated_tasks.length; ++i) { var row = window.document.getElementById("row-" + updated_tasks[i]); - + for(var j = 0; j < entries.length; ++j) { if(entries[j] == "download") continue; - + var cell = window.document.getElementById("cell-" + updated_tasks[i] + "-" + entries[j]); - + cell.innerHTML = inner_cell(__info[updated_tasks[i]], entries[j]); - + /*if(show_col[entries[j]]) cell.classList.remove("hidden"); else cell.classList.add("hidden");*/ } - + window.document.getElementById("cell-" + updated_tasks[i] + "-error-msg").innerHTML = inner_error_cell(__info[updated_tasks[i]]); - + row.classList.add("updated"); window.setTimeout(make_class_removal_function("row-" + updated_tasks[i], "updated"), 1000); } - + // Remove tasks that have been deleted from the filesystem var __tasks_backup = __tasks; __tasks = []; @@ -258,21 +262,21 @@ function fill_table(new_tasks, updated_tasks, show_col, criteria, init) else __tasks.push(__tasks_backup[i]); } - + for(var j = 0; j < entries.length; ++j) { var cell = window.document.getElementById("overview-heading-" + entries[j]); - + if(show_col[entries[j]]) cell.classList.remove("hidden"); else - cell.classList.add("hidden"); + cell.classList.add("hidden"); } - + // Coloring and rounded borders var count = 0; var prefix = "overview-heading"; - + for(var j = 0; j < entries.length; ++j) { var id = prefix + "-" + entries[j]; @@ -282,20 +286,20 @@ function fill_table(new_tasks, updated_tasks, show_col, criteria, init) cell.classList.remove("upper-left"); cell.classList.remove("upper-right"); } - + for(var i = 0; i < __tasks.length; ++i) { var id = "row-" + __tasks[i]; var row = window.document.getElementById(id); - + if(row.classList.contains("hidden")) continue; - + if(count % 2 == 0) row.classList.remove("odd"); else row.classList.add("odd"); - + ++count; prefix = "cell-" + __tasks[i]; - + for(var j = 0; j < entries.length; ++j) { var id = prefix + "-" + entries[j]; @@ -306,43 +310,43 @@ function fill_table(new_tasks, updated_tasks, show_col, criteria, init) cell.classList.remove("upper-right"); } } - + var first_col = null; var last_col = null; - + for(var j = 0; j < entries.length; ++j) { var id = prefix + "-" + entries[j]; var cell = window.document.getElementById(id); - + if(cell.classList.contains("hidden")) continue; - + if(first_col == null) first_col = entries[j]; last_col = entries[j]; } - + if(last_col != null) { window.document.getElementById(prefix + "-" + first_col).classList.add("lower-left"); window.document.getElementById(prefix + "-" + last_col).classList.add("lower-right"); - + window.document.getElementById("overview-heading-" + first_col).classList.add("upper-left"); window.document.getElementById("overview-heading-" + last_col).classList.add("upper-right"); } - + // Create overview of problematic info.json files var json_issues = []; - + for(var i = 0; i < __tasks.length; ++i) if("error" in __info[__tasks[i]]) json_issues.push(__tasks[i]); - + if(json_issues.length == 0) window.document.getElementById("json-errors").classList.add("no-errors"); - + else window.document.getElementById("json-errors").classList.remove("no-errors"); - + window.document.getElementById("tasks-with-issues").innerHTML = json_issues.join(", "); } @@ -351,7 +355,7 @@ function update(init=false, sliders=false) function on_list(l) { var response = JSON.parse(l); - + var available_tasks = []; var available_tasks_dict = {}; for(var i = 0; i < response.length; ++i) @@ -359,20 +363,20 @@ function update(init=false, sliders=false) available_tasks.push(response[i].task); available_tasks_dict[response[i].task] = response[i].timestamp; } - + var new_tasks = []; var updated_tasks = []; - + for(var i = 0; i < available_tasks.length; ++i) { var t = available_tasks[i]; - + if(!(t in __task_dict)) { new_tasks.push(t); __task_dict[t] = available_tasks_dict[t]; } - + if(__task_dict[t] < available_tasks_dict[t]) { updated_tasks.push(t); @@ -381,11 +385,11 @@ function update(init=false, sliders=false) } __removed = {}; - + for(var i = 0; i < __tasks.length; ++i) if(!(__tasks[i] in available_tasks_dict)) __removed[__tasks[i]] = true; - + function on_info(i) { var info = JSON.parse(i); @@ -393,16 +397,16 @@ function update(init=false, sliders=false) __info[new_tasks[i]] = info[new_tasks[i]]; for(var i = 0; i < updated_tasks.length; ++i) __info[updated_tasks[i]] = info[updated_tasks[i]]; - + fill_table(new_tasks, updated_tasks, show_col, criteria, init); if(sliders) update_sliders(init); } - + if(new_tasks.length > 0 || updated_tasks.length > 0) { $.get(__url_root + "/info", {"tasks": JSON.stringify(new_tasks.concat(updated_tasks))}, on_info); } - + else { fill_table(new_tasks, updated_tasks, show_col, criteria, init); @@ -416,39 +420,39 @@ function update(init=false, sliders=false) function update_sliders(init=false) { init_date_slider("uses", interesting_dates(), true); - if(!init) date_slider_set("uses", criteria.only_before); - if(init) criteria.only_before = date_slider_get_val("uses"); + if(!init) date_slider_set("uses", criteria.only_before); + if(init) criteria.only_before = date_slider_get_val("uses"); } function interesting_dates() { var raw = []; - + for(var i = 0; i < __tasks.length; ++i) raw = raw.concat(__info[__tasks[i]].uses); - + function my_comp(a, b) { if(a.timestamp < b.timestamp) return -1; if(a.timestamp > b.timestamp) return 1; return 0; } - + raw = raw.sort(my_comp); /* Merge multiple entries for the same contest (multiple days, typos in the files etc.): - + If there are multiple entries with the same description at successive time points, we keep only the last one. - + This is of course not perfect, but it should work in most cases. Since the selection criteria are just for convenience, this should be fine (and it is certainly much better than having to sanitize all info.json files by hand) */ var dates = []; - + for(var i = 0; i < raw.length; ++i) { if(dates.length > 0 && dates[dates.length - 1].info == raw[i].info)