diff --git a/.description b/.description index 7c549a7..3935708 100644 --- a/.description +++ b/.description @@ -1,2 +1 @@ UKBot is a bot for updating results in weekly contests held at Norwegian and Finnish Wikipedia (source) - diff --git a/.env.dist b/.env.dist index 263cc88..679c0ff 100644 --- a/.env.dist +++ b/.env.dist @@ -10,4 +10,4 @@ DB_PASSWORD=ukbot MW_CONSUMER_TOKEN= MW_CONSUMER_SECRET= MW_ACCESS_TOKEN= -MW_ACCESS_SECRET= \ No newline at end of file +MW_ACCESS_SECRET= diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index e5091c5..2c4eb01 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -24,4 +24,4 @@ Describe what happens when you follow the steps above. A clear and concise description of what you expected to happen. **Additional context** -Add any other context about the problem here. \ No newline at end of file +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index b5c925c..f488634 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -17,4 +17,4 @@ A clear and concise description of one or more use cases. Ex. As a ..., I want t A clear and concise description of any alternative solutions or features you've considered. **Additional context** -Add any other context or screenshots about the feature request here. \ No newline at end of file +Add any other context or screenshots about the feature request here. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 44b0437..0930cdf 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,8 +1,8 @@ ## Description - ## What type of PR is this? (check all applicable) @@ -21,7 +21,7 @@ This PR [adds/removes/fixes/replaces] the [feature/bug/etc]. ## Related Tickets & Documents - @@ -39,6 +39,6 @@ Please use this format link issue numbers: Fixes #123 ## [optional] Are there any pre- or post-deployment tasks we need to perform? - \ No newline at end of file +--> diff --git a/.github/workflows/check-format-and-typing.yml b/.github/workflows/check-format-and-typing.yml new file mode 100644 index 0000000..54978f0 --- /dev/null +++ b/.github/workflows/check-format-and-typing.yml @@ -0,0 +1,32 @@ +name: Code formatting and typing + +on: + pull_request: + branches: + - main + +jobs: + CodeFormattingAndTyping: + name: Code formatting and typing + runs-on: ubuntu-latest + steps: + - name: Git checkout + uses: actions/checkout@v3 + + - name: Install Python 3 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements_dev.txt + + - name: check format using pycodestyle (ignore error on line length>79) + id: formatChecks + run: flake8 ukbot + + - name: mypy static type checker + run: mypy ukbot diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..be85169 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,27 @@ +name: "CodeQL" + +on: + schedule: + - cron: "40 2 * * 1" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: ["python"] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..94d82a1 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,31 @@ +name: Run Tests +on: + workflow_dispatch: + pull_request: + branches: + - main + +jobs: + run_tests: + runs-on: ubuntu-latest + steps: + - name: Git checkout + uses: actions/checkout@v3 + + - name: Install Python 3 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements_dev.txt + pip install -e . + + - name: Run tests with pytest + run: coverage run -m pytest test + + - name: Print coverage report + run: coverage report -m diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..42dd59c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,37 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-docstring-first + - id: check-added-large-files + - id: check-case-conflict + - id: debug-statements + - id: requirements-txt-fixer +- repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v1.6.0 + hooks: + - id: autopep8 + additional_dependencies: ['pycodestyle==2.10.0'] +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.6.0 + hooks: + - id: mypy + name: mypy ukbot + exclude: config/, locale/ + entry: mypy ukbot/ + pass_filenames: false + args: [--config-file=mypy.ini] + additional_dependencies: + - types-PyMySQL + - types-pytz + - types-PyYAML + - types-requests + - urllib3 + +- repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 diff --git a/.travis.yml b/.travis.yml index 890ed70..bebe30d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ python: - 3.5 - 3.6 -# command to install dependencies, e.g. -install: +# command to install dependencies, e.g. +install: - pip install -r requirements.txt script: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 576429f..d3e3319 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -124,4 +124,4 @@ enforcement ladder](https://github.com/mozilla/diversity). For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. \ No newline at end of file +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0017290..1bd3227 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,4 +21,4 @@ We welcome all feature requests, whether to add new functionality to `UKBot` or ### ⚙️ Close a Bug / Feature issue -We welcome contributions that help make `UKBot` bug free & improve the experience of our users. You can also find issues tagged `bug` or `enhancement`. Check out our [Code Contribution Guide](docs/Code_Contributions_Guidelines.md) to begin. \ No newline at end of file +We welcome contributions that help make `UKBot` bug free & improve the experience of our users. You can also find issues tagged `bug` or `enhancement`. Check out our [Code Contribution Guide](docs/Code_Contributions_Guidelines.md) to begin. diff --git a/ISSUES.md b/ISSUES.md index 0b63d02..7d15640 100644 --- a/ISSUES.md +++ b/ISSUES.md @@ -14,4 +14,4 @@ The `bug report` and `feature request` issue templates are ready for you! 2. The `UKBot` member provides feedback as soon as possible. 3. A conversation or discussion between `UKBot` and the user. 4. A pull request related to the issue will close it. -5. Otherwise, we'll close the issue after seven days of no update. \ No newline at end of file +5. Otherwise, we'll close the issue after seven days of no update. diff --git a/MANIFEST.in b/MANIFEST.in index 85c9a00..41525a1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include locale/*/*/*.mo \ No newline at end of file +include locale/*/*/*.mo diff --git a/MIT-LICENSE.txt b/MIT-LICENSE.txt index df30861..08e40ec 100644 --- a/MIT-LICENSE.txt +++ b/MIT-LICENSE.txt @@ -18,4 +18,3 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/README.md b/README.md index 1bea66b..566d7c2 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

Created by Dan Michael and maintained with ❤︎ by contributors

- + ## Getting Started Clone code from github @@ -66,8 +66,8 @@ To modify the code and test it locally, you'll need to install it as a pip packa ## Deployment -The bot is deployed at [ToolForge](https://wikitech.wikimedia.org/wiki/Portal:Toolforge) under the "UKBot" account using -[Webservice](https://wikitech.wikimedia.org/wiki/Help:Toolforge/Web) and the [Toolforge Jobs Framework](https://wikitech.wikimedia.org/wiki/Portal:Toolforge/Admin/Kubernetes/Jobs_framework). +The bot is deployed at [ToolForge](https://wikitech.wikimedia.org/wiki/Portal:Toolforge) under the "UKBot" account using +[Webservice](https://wikitech.wikimedia.org/wiki/Help:Toolforge/Web) and the [Toolforge Jobs Framework](https://wikitech.wikimedia.org/wiki/Portal:Toolforge/Admin/Kubernetes/Jobs_framework). See https://wikitech.wikimedia.org/wiki/Tool:UKBot for deployment notes. @@ -75,4 +75,3 @@ See https://wikitech.wikimedia.org/wiki/Tool:UKBot for deployment notes. Forenklet flytkart: ![Flowchart](https://github.com/danmichaelo/UKBot/raw/master/flowchart.png) - diff --git a/config/config.en.yml b/config/config.en.yml index 1ffdbef..8fd8ff4 100644 --- a/config/config.en.yml +++ b/config/config.en.yml @@ -1,2 +1 @@ _extends: sites/enwiki.yml - diff --git a/config/config.es-estonia.yml b/config/config.es-estonia.yml index 064ce93..67d3666 100644 --- a/config/config.es-estonia.yml +++ b/config/config.es-estonia.yml @@ -18,4 +18,4 @@ plot: == {{int:license-header}} == {{PD-self}} - [[Category:Wikipedia competitions]] \ No newline at end of file + [[Category:Wikipedia competitions]] diff --git a/config/config.fi-100.yml b/config/config.fi-100.yml index 50293d3..51f51c3 100644 --- a/config/config.fi-100.yml +++ b/config/config.fi-100.yml @@ -47,8 +47,8 @@ plot: |Date = %(start)s |Author = [[User:UKBot|UKBot]] }} - + == {{int:license-header}} == {{PD-self}} - + [[Category:Viikon kilpailu]] diff --git a/config/config.fi-ek.yml b/config/config.fi-ek.yml index 86bcef1..46a700b 100644 --- a/config/config.fi-ek.yml +++ b/config/config.fi-ek.yml @@ -22,8 +22,8 @@ plot: |Date = %(start)s |Author = [[User:UKBot|UKBot]] }} - + == {{int:license-header}} == {{PD-self}} - + [[Category:Elokuun kuvitustalkoot]] diff --git a/config/config.fi-hl.yml b/config/config.fi-hl.yml index dca1719..a824f02 100644 --- a/config/config.fi-hl.yml +++ b/config/config.fi-hl.yml @@ -20,8 +20,8 @@ plot: |Date = %(start)s |Author = [[User:UKBot|UKBot]] }} - + == {{int:license-header}} == {{PD-self}} - + [[Category:Heinäkuun lähdetalkoot]] diff --git a/config/config.fi-pln.yml b/config/config.fi-pln.yml index 7725682..e47fbd9 100644 --- a/config/config.fi-pln.yml +++ b/config/config.fi-pln.yml @@ -23,8 +23,8 @@ plot: |Date = %(start)s |Author = [[User:UKBot|UKBot]] }} - + == {{int:license-header}} == {{PD-self}} - + [[Category:Wikipedia contests of Finnish Wikipedia]] diff --git a/config/config.fi-vk.yml b/config/config.fi-vk.yml index ac399d5..b0c2915 100644 --- a/config/config.fi-vk.yml +++ b/config/config.fi-vk.yml @@ -23,10 +23,10 @@ plot: |Date = %(start)s |Author = [[User:UKBot|UKBot]] }} - + == {{int:license-header}} == {{PD-self}} - + [[Category:Viikon kilpailu %(year)s]] awards: sininen: { file: Article blue.svg, winner: true } diff --git a/config/config.gl-estonia.yml b/config/config.gl-estonia.yml index 1edda41..f1c7b41 100644 --- a/config/config.gl-estonia.yml +++ b/config/config.gl-estonia.yml @@ -18,4 +18,4 @@ plot: == {{int:license-header}} == {{PD-self}} - [[Category:Wikipedia competitions]] \ No newline at end of file + [[Category:Wikipedia competitions]] diff --git a/config/config.no-fd.yml b/config/config.no-fd.yml index b291a2b..64e46ce 100644 --- a/config/config.no-fd.yml +++ b/config/config.no-fd.yml @@ -28,10 +28,10 @@ plot: |Date = %(start)s |Author = [[User:UKBot|UKBot]] }} - + == {{int:license-header}} == {{PD-self}} - + [[Category:Fiks det!]] awards: blå: { file: FD-premie blå.svg, winner: true } diff --git a/config/config.no-mk.yml b/config/config.no-mk.yml index a86bbc6..3768cf3 100644 --- a/config/config.no-mk.yml +++ b/config/config.no-mk.yml @@ -40,8 +40,8 @@ plot: |Date = %(start)s |Author = [[User:UKBot|UKBot]] }} - + == {{int:license-header}} == {{PD-self}} - + [[Category:Månedens konkurranse %(year)s]] diff --git a/config/config.no-uk.yml b/config/config.no-uk.yml index ff9e855..7032922 100644 --- a/config/config.no-uk.yml +++ b/config/config.no-uk.yml @@ -30,8 +30,8 @@ plot: |Date = %(start)s |Author = [[User:UKBot|UKBot]] }} - + == {{int:license-header}} == {{PD-self}} - + [[Category:Ukens konkurranse %(year)s]] diff --git a/config/config.se.yml b/config/config.se.yml index 8a427f0..f48ddc3 100644 --- a/config/config.se.yml +++ b/config/config.se.yml @@ -15,10 +15,10 @@ plot: |Date = %(start)s |Author = [[User:UKBot|UKBot]] }} - + == {{int:license-header}} == {{PD-self}} - + [[Category:Wikipedia contests]] awards: sininen: { file: Article blue.svg, winner: true } diff --git a/docker-compose.yml b/docker-compose.yml index 50d7c85..0818e0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,4 +38,4 @@ services: # - ./:/src volumes: db: - driver: local \ No newline at end of file + driver: local diff --git a/docs/Code_Contributions_Guidelines.md b/docs/Code_Contributions_Guidelines.md index 84faf4a..a7915c3 100644 --- a/docs/Code_Contributions_Guidelines.md +++ b/docs/Code_Contributions_Guidelines.md @@ -15,4 +15,15 @@ We use [GitHub Flow](https://guides.github.com/introduction/flow/index.html), so 6. When you raise a pull request, we automatically run tests on our CI. Please ensure that all the tests are passing for your code change. We will not be able to accept your change if the test suite doesn't pass. ## 🏡 Setup for local development -- In progress \ No newline at end of file +Install packages needed for development +```bash +pip install -r requirements.txt -r requirements_dev.txt +``` +Install [pre-commit](https://pre-commit.com/) +```bash +pre-commit install +``` +When you have done your changes pre-commit will run when running `git commit`. To run it whenever you want type: +```bash +pre-commit run --all-files +``` diff --git a/jobs/update-venv.sh b/jobs/update-venv.sh index 61158bf..94e5205 100755 --- a/jobs/update-venv.sh +++ b/jobs/update-venv.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Usage: +# Usage: # toolforge-jobs run update-venv --command "./update_venv.sh" --image tf-python39 --wait set -euo pipefail diff --git a/jobs/upload.sh b/jobs/upload.sh index e3583c4..7f28217 100755 --- a/jobs/upload.sh +++ b/jobs/upload.sh @@ -22,4 +22,3 @@ stdbuf -oL -eL ukbot config/config.${CONTEST}.yml --job_id "$JOB_ID" --action "u status="${PIPESTATUS[0]}" echo "$(date) : Job $JOB_NAME ($JOB_ID) on $HOSTNAME finished with exit code $status" >> $logfile - diff --git a/locale/ca_ES.po b/locale/ca_ES.po index fca60d1..dbb3fd2 100644 --- a/locale/ca_ES.po +++ b/locale/ca_ES.po @@ -426,4 +426,3 @@ msgstr "%(count)d declaracions de %(property)s" #, python-format msgid "%(words).f words" msgstr "%(words).f paraules" - diff --git a/locale/gl_ES.po b/locale/gl_ES.po index 6cc1b34..a3697ec 100644 --- a/locale/gl_ES.po +++ b/locale/gl_ES.po @@ -1,6 +1,6 @@ # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER # This file is distributed under the same license as the PACKAGE package. -# +# # Francisco Prieto , 2025. msgid "" msgstr "" diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..1075b37 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +ignore_missing_imports = True +exclude = config/, locale/ + +[mypy-pkg_resources] +ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 12d31a6..3df4bea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "pytz", "PyYAML", "requests", - "requests-oauthlib", + "requests-oauthlib" ] classifiers = [ diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..8e45191 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,13 @@ +autopep8==2.0.2 +faker==37.4.0 +flake8==6.0.0 +freezegun==1.2.2 +mypy==1.4.0 +pre-commit==3.3.3 +pytest==7.3.2 +pytest-cov==4.1.0 +pytest-mock==3.11.1 +types-PyMySQL==1.1.0.20250516 +types-pytz==2025.2.0.20250516 +types-PyYAML==6.0.12.20250516 +types-requests==2.32.4.20250611 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..63648d5 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 120 +ignore = E501, W504, W503, E704 +statistics = True +per-file-ignores = __init__.py:F401 diff --git a/test/test_filters.py b/test/test_filters.py index 2286867..d2d28ae 100644 --- a/test/test_filters.py +++ b/test/test_filters.py @@ -1,8 +1,7 @@ # encoding=utf-8 -import re from collections import OrderedDict import unittest -from mock import Mock +import unittest.mock as mock from unittest import TestCase from faker import Faker @@ -20,15 +19,15 @@ class DummyDataProvider: def page_mock(self, name=None, site=None, prefix=''): """ Create a MWClient Page instance mock """ - page = Mock(Page) + page = mock.Mock(Page) page.site = site or self.site page.name = '%s%s' % (prefix, name or fake.name()) return page def article_mock(self, name=None, site=None): """ Create an UKBOt Article instance mock """ - article = Mock(Article) - article.site = Mock(return_value=site or self.site) + article = mock.Mock(Article) + article.site = mock.Mock(return_value=site or self.site) article.name = name or fake.name() article.key = article.site().key + ':' + article.name return article @@ -36,15 +35,15 @@ def article_mock(self, name=None, site=None): def __init__(self, articles: int, categories: int): # Define a dummy site - self.site = Mock(Site) + self.site = mock.Mock(Site) self.site.key = 'dummy.wikipedia.org' # self.site.redirect_regexp = re.compile(u'(?:%s)' % u'|'.join(['redirect']), re.I) self.site.rights = ['bot'] # And a site manager - self.sites = Mock(SiteManager) - self.sites.keys = Mock(return_value=[self.site.key]) - self.sites.items = Mock(return_value=[(self.site.key, self.site)]) + self.sites = mock.Mock(SiteManager) + self.sites.keys = mock.Mock(return_value=[self.site.key]) + self.sites.items = mock.Mock(return_value=[(self.site.key, self.site)]) # Create some articles and categories self.articles = [self.article_mock() for n in range(articles)] diff --git a/test/test_rules.py b/test/test_rules.py index 2bbc7a1..7944564 100644 --- a/test/test_rules.py +++ b/test/test_rules.py @@ -1,8 +1,7 @@ # encoding=utf-8 import re from datetime import datetime -import mock -import json +import unittest.mock as mock from unittest import TestCase import pytz @@ -150,8 +149,8 @@ def test_it_yields_nothing_by_default(self): def test_it_gives_points_for_template_removal(self): points_per_template = 10 - self.rev.text='Hello' - self.rev.parenttext='Hello {{world}} {{ world }} {{ world | param | 3=other param }}' + self.rev.text = 'Hello' + self.rev.parenttext = 'Hello {{world}} {{ world }} {{ world | param | 3=other param }}' self.sites.resolve_page.return_value = self.page_mock('World', [], site=self.site) rule = TemplateRemovalRule(self.sites, {2: points_per_template, 3: 'world'}) diff --git a/ukbot/article.py b/ukbot/article.py index 2ad606a..743c529 100644 --- a/ukbot/article.py +++ b/ukbot/article.py @@ -30,6 +30,7 @@ def __init__(self, site, user, name, ns): self.revisions = OrderedDict() self.errors = [] + self.cat_path = [] # type: ignore[assignment] def __eq__(self, other): if self.site() == other.site() and self.name == other.name: @@ -71,11 +72,10 @@ def key(self): @property def firstrev(self): return self.revisions[first(self.revisions)] - + @property def lastrev(self): return self.revisions[last(self.revisions)] - @property def redirect(self): diff --git a/ukbot/common.py b/ukbot/common.py index c77ab63..12c39de 100644 --- a/ukbot/common.py +++ b/ukbot/common.py @@ -3,10 +3,8 @@ import sys import locale import gettext -import yaml import os import psutil -import pkg_resources import logging logger = logging.getLogger(__name__) @@ -32,27 +30,26 @@ def init(self, cl): '''prepare i18n''' if not isinstance(cl, list): cl = [cl] - #['nb_NO.UTF-8', 'nb_NO.utf8', 'no_NO']: + # ['nb_NO.UTF-8', 'nb_NO.utf8', 'no_NO']: for loc in cl: try: - # print "Trying (", loc.encode('utf-8'), 'utf-8',")" - locale.setlocale(locale.LC_ALL, (loc, 'utf-8')) + # print "Trying (", loc.encode('utf-8'), 'utf-8',")" locale.setlocale(locale.LC_ALL, (loc, 'utf-8')) logger.info('Using locale %s', loc) - #logger.info('Locale set to %s' % loc) + # logger.info('Locale set to %s' % loc) break except locale.Error: try: locstr = loc + '.UTF-8' # print "Trying",locstr - locale.setlocale(locale.LC_ALL, locstr ) + locale.setlocale(locale.LC_ALL, locstr) logger.info('Using locale %s', loc) break except locale.Error: pass lang, charset = locale.getlocale() - if lang == None: - raise StandardError('Failed to set locale!') + if lang is None: + raise Exception('Failed to set locale!') t = gettext.translation('messages', LOCALE_PATH, fallback=True, languages=[lang]) @@ -60,6 +57,7 @@ def init(self, cl): self._ = t.gettext instance = None + def __init__(self): if not Localization.instance: Localization.instance = Localization.__Localization() @@ -72,6 +70,7 @@ def __getattr__(self, name): localization = Localization() + def ngettext(*args, **kwargs): try: return localization.t.ngettext(*args, **kwargs) @@ -79,11 +78,15 @@ def ngettext(*args, **kwargs): # During tests, Localization might not be initialized return args[0] + def _(*args, **kwargs): return localization._(*args, **kwargs) + logfile = sys.stdout -def log(msg, newline = True): + + +def log(msg, newline=True): if newline: msg = msg + '\n' logfile.write(msg.encode('utf-8')) @@ -92,6 +95,7 @@ def log(msg, newline = True): process = psutil.Process(os.getpid()) + def get_mem_usage(): """ Returns memory usage in MBs """ return process.memory_info().rss / 1024.**2 @@ -103,3 +107,8 @@ class InvalidContestPage(Exception): def __init__(self, msg): self.msg = msg + def __str__(self): + return self.msg + + def __repr__(self): + return 'InvalidContestPage(%r)' % self.msg diff --git a/ukbot/contest.py b/ukbot/contest.py index f534a21..a7d7385 100644 --- a/ukbot/contest.py +++ b/ukbot/contest.py @@ -182,11 +182,10 @@ def extract_rules(self, txt, catignore_page=''): if config['templates']['rule']['name'] not in dp.templates: raise InvalidContestPage(_('There are no point rules defined for this contest. Point rules are defined by {{tl|%(template)s}}.') % {'template': config['templates']['rule']['name']}) - #if not 'ukens konkurranse kriterium' in dp.templates.keys(): + # if not 'ukens konkurranse kriterium' in dp.templates.keys(): # raise InvalidContestPage('Denne konkurransen har ingen bidragskriterier. Kriterier defineres med {{tl|ukens konkurranse kriterium}}.') - - ######################## Read infobox ######################## + # Read infobox infobox = parse_infobox(txt, self.sites.homesite.namespaces[2], self.config) self.start = infobox['start_time'] @@ -207,7 +206,7 @@ def extract_rules(self, txt, catignore_page=''): self.prices = infobox['awards'] self.prices.sort(key=lambda x: x[2], reverse=True) - ######################## Read filters ######################## + # Read filters nfilters = 0 # print dp.templates.keys() @@ -241,7 +240,7 @@ def extract_rules(self, txt, catignore_page=''): # Prepend filter to the filters list filters.insert(0, filter_inst) - ######################## Read rules ######################## + # Read rules rule_classes_map = { rulecfg[rule_cls.rule_name]: rule_cls for rule_cls in rule_classes @@ -253,7 +252,7 @@ def extract_rules(self, txt, catignore_page=''): rule_name = rule_template.parameters[1].value.lower() try: rule_cls = rule_classes_map[rule_name] - except: + except KeyError: raise InvalidContestPage( _('Unkown argument given to {{tl|%(template)s}}: %(argument)s') % {'template': rulecfg['name'], 'argument': rule_name} @@ -262,7 +261,7 @@ def extract_rules(self, txt, catignore_page=''): rule = rule_cls(self.sites, rule_template.parameters, rulecfg) rules.append(rule) - ####################### Check if contest is in DB yet ################## + # Check if contest is in DB yet cur = self.sql.cursor() now = datetime.now() @@ -277,7 +276,7 @@ def extract_rules(self, txt, catignore_page=''): self.sql.commit() cur.close() - ######################## Read disqualifications ######################## + # Read disqualifications sucfg = self.config['templates']['suspended'] if sucfg['name'] in dp.templates: @@ -288,17 +287,17 @@ def extract_rules(self, txt, catignore_page=''): except ValueError: raise InvalidContestPage(_("Couldn't parse the date given to the {{tl|%(template)s}} template.") % sucfg['name']) - #print 'Suspendert bruker:',uname,sdate + # print 'Suspendert bruker:',uname,sdate ufound = False for u in self.users: if u.name == uname: - #print " > funnet" + # print " > funnet" u.suspended_since = sdate ufound = True if not ufound: pass # TODO: logging.warning - #raise InvalidContestPage('Fant ikke brukeren %s gitt til {{tl|UK bruker suspendert}}-malen.' % uname) + # raise InvalidContestPage('Fant ikke brukeren %s gitt til {{tl|UK bruker suspendert}}-malen.' % uname) dicfg = self.config['templates']['disqualified'] if dicfg['name'] in dp.templates: @@ -458,9 +457,8 @@ def plot(self, plotdata): else: x.append(xt[-1]) y.append(y[-1]) - l = ax.plot(x, y, linewidth=1.2, label=result['name']) # markerfacecolor='#FF8C00', markeredgecolor='#888888', label = u['name']) - c = l[0].get_color() - #ax.plot(x[1:-1], y[1:-1], marker='.', markersize=4, markerfacecolor=c, markeredgecolor=c, linewidth=0., alpha=0.5) # markerfacecolor='#FF8C00', markeredgecolor='#888888', label = u['name']) + line = ax.plot(x, y, linewidth=1.2, label=result['name']) # markerfacecolor='#FF8C00', markeredgecolor='#888888', label = u['name']) + # ax.plot(x[1:-1], y[1:-1], marker='.', markersize=4, markerfacecolor=c, markeredgecolor=c, linewidth=0., alpha=0.5) # markerfacecolor='#FF8C00', markeredgecolor='#888888', label = u['name']) if cnt >= 15: break @@ -480,7 +478,7 @@ def plot(self, plotdata): # Tick labels at the middle of every day # Just a number if ndays < 7 ax.set_xticks(xt_mid, minor=True) - ax.set_xticklabels(list(range(1,ndays+1)), minor=True) + ax.set_xticklabels(list(range(1, ndays + 1)), minor=True) elif ndays == 7: # Tick marker every midnight ax.set_xticks(xt, minor=False) @@ -514,8 +512,6 @@ def plot(self, plotdata): # elif ndays == 31: # ax.set_xticklabels(['1', '', '', '', '5', '', '', '', '', '10', '', '', '', '', '15', '', '', '', '', '20', '', '', '', '', '25', '', '', '', '', '', '31'], minor=True) - - for i in range(1, ndays, 2): ax.axvspan(xt[i], xt[i + 1], facecolor='#000099', linewidth=0., alpha=0.03) @@ -542,8 +538,8 @@ def plot(self, plotdata): now2 = now.astimezone(self.wiki_tz).strftime(_('%e. %B %Y, %H:%M')) ax_title = _('Updated %(date)s') - #print ax_title.encode('utf-8') - #print now2.encode('utf-8') + # print ax_title.encode('utf-8') + # print now2.encode('utf-8') ax_title = ax_title % {'date': now2} ax.set_title(ax_title) @@ -596,19 +592,18 @@ def deliver_message(self, username, topic, body, sig='~~~~'): if flow_enabled: token = self.sites.homesite.get_token('csrf') self.sites.homesite.api(action='flow', - submodule='new-topic', - page=pagename, - nttopic=topic, - ntcontent=body, - ntformat='wikitext', - token=token) + submodule='new-topic', + page=pagename, + nttopic=topic, + ntcontent=body, + ntformat='wikitext', + token=token) else: page = self.sites.homesite.pages[pagename] page.save(text=body + ' ' + sig, bot=False, section='new', summary=topic) def deliver_prices(self, results, simulate=False): - config = self.config heading = self.format_heading() cur = self.sql.cursor() @@ -676,7 +671,6 @@ def deliver_ended_contest_notification(self): 'title': urllib.parse.quote(self.config['awardstatus']['send']) } link = '%(prefix)s?title=%(page)s&action=edit§ion=new&preload=%(page)s/Preload&preloadtitle=%(title)s' % args - usertalkprefix = self.sites.homesite.namespaces[3] awards = [] for key, award in self.config['awards'].items(): if 'organizer' in award: @@ -778,15 +772,15 @@ def deliver_warnings(self, simulate=False): heading = '== Viktig informasjon angående Ukens konkurranse uke %d ==' % self.startweek else: heading = '== Viktig informasjon angående Ukens konkurranse uke %d–%d ==' % (self.startweek, self.endweek) - #msg = 'Arrangøren av denne [[%(pagename)s|ukens konkurranse]] har registrert problemer ved noen av dine bidrag: - #så langt. Det er dessverre registrert problemer med enkelte av dine bidrag som medfører at vi er nødt til å informere deg om følgende:\n' % { 'pagename': self.name } + # msg = 'Arrangøren av denne [[%(pagename)s|ukens konkurranse]] har registrert problemer ved noen av dine bidrag: + # så langt. Det er dessverre registrert problemer med enkelte av dine bidrag som medfører at vi er nødt til å informere deg om følgende:\n' % { 'pagename': self.name } msg = ''.join(['* %s\n' % m for m in msgs]) msg += 'Denne meldingen er generert fra anmerkninger gjort av konkurransearrangør på [[%(pagename)s|konkurransesiden]]. Du finner mer informasjon på konkurransesiden og/eller tilhørende diskusjonsside. Så lenge konkurransen ikke er avsluttet, kan problemer løses i løpet av konkurransen. Om du ønsker det, kan du fjerne denne meldingen når du har lest den. ~~~~' % {'pagename': self.name} - #print '------------------------------',u.name - #print msg - #print '------------------------------' + # print '------------------------------',u.name + # print msg + # print '------------------------------' page = self.sites.homesite.pages['%s:%s' % (usertalkprefix, u.name)] logger.info('Leverer advarsel til %s', page.name) @@ -801,11 +795,7 @@ def run(self, simulate=False, output=''): if not self.page.exists: logger.error('Contest page [[%s]] does not exist! Exiting', self.page.name) - return - - # Loop over users - - narticles = 0 + return # Loop over users stats = [] @@ -900,7 +890,7 @@ def run(self, simulate=False, output=''): # Make outpage out = '' - #out += '[[File:Nowp Ukens konkurranse %s.svg|thumb|400px|Resultater (oppdateres normalt hver natt i halv ett-tiden, viser kun de ti med høyest poengsum)]]\n' % self.start.strftime('%Y-%W') + # out += '[[File:Nowp Ukens konkurranse %s.svg|thumb|400px|Resultater (oppdateres normalt hver natt i halv ett-tiden, viser kun de ti med høyest poengsum)]]\n' % self.start.strftime('%Y-%W') summary_tpl = None if 'status' in config['templates']: @@ -1158,7 +1148,6 @@ def run(self, simulate=False, output=''): oppslagstavle.save(txt2, summary=_('The weekly contest is: %(link)s') % {'link': tema}) # Make a nice plot - if 'plot' in config: plotdata = self.prepare_plotdata(results) self.plot(plotdata) @@ -1171,7 +1160,7 @@ def uploadplot(self, simulate=False, output=''): logger.error('Contest page [[%s]] does not exist! Exiting', self.page.name) return - if not 'plot' in self.config: + if 'plot' not in self.config: return figname = self.config['plot']['figname'] % { @@ -1223,4 +1212,3 @@ def uploadplot(self, simulate=False, output=''): comment='Bot: Updating plot', ignore=True) logger.info(res) - diff --git a/ukbot/contests.py b/ukbot/contests.py index a478dd7..6ba0b21 100644 --- a/ukbot/contests.py +++ b/ukbot/contests.py @@ -90,7 +90,7 @@ def get_contest_page_titles(sql, homesite, config): ) for row in cursor.fetchall(): page_title = row[0] - logger.info(f"Contest [[%s]] ended at %s. Current time is %s", page_title, row[1].strftime('%F %T'), now_s) + logger.info("Contest [[%s]] ended at %s. Current time is %s", page_title, row[1].strftime('%F %T'), now_s) yield STATE_ENDING, page_title # 3) Check if there are other contests to update diff --git a/ukbot/contributions.py b/ukbot/contributions.py index 45a4e20..6591024 100644 --- a/ukbot/contributions.py +++ b/ukbot/contributions.py @@ -24,7 +24,7 @@ def add(self, contribution): """ Add a contribution and calculate the actual number of points given to it when taking point capping into account. - + After capping, some contributions may end up with zero points, but we still store them so we can collect statistics like 'total number of words'. @@ -133,7 +133,7 @@ def get_article_points(self, article, ignore_max=False, ignore_suspension_period for deduction in revision.point_deductions: logger.debug('Subtracting %.1f points from point deduction', deduction[0]) points -= deduction[0] - + return points # p = 0. @@ -208,7 +208,7 @@ def summarize(self, homesite): article_formatted = self.summarize_article(article, revisions_formatted) articles_formatted.append(article_formatted) - + return articles_formatted def summarize_revision(self, revision, homesite): @@ -229,7 +229,7 @@ def summarize_revision(self, revision, homesite): desc += ', ' + _('capped at max') contrib_points.append('%.1f p (%s)' % (contribution.points, desc)) formatted += ' + '.join(contrib_points) - + # Add deductions, if any for deduction in revision.point_deductions: if deduction[0] > 0: diff --git a/ukbot/filters.py b/ukbot/filters.py index 85b0320..877737b 100644 --- a/ukbot/filters.py +++ b/ukbot/filters.py @@ -1,6 +1,5 @@ # encoding=utf-8 # vim: fenc=utf-8 et sw=4 ts=4 sts=4 ai -import sys import re from copy import copy @@ -11,18 +10,19 @@ import requests from collections import OrderedDict from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry +from urllib3.util.retry import Retry from mwtemplates.templateeditor2 import TemplateParseError -from .common import _, InvalidContestPage +from mwclient.page import Page from .site import WildcardPage +from .common import _ -from typing import List, Union, Optional +from typing import List, Union, Optional, Set, Dict from typing import TYPE_CHECKING if TYPE_CHECKING: - from .ukbot import FilterTemplate, SiteManager - from .article import Article - from mwclient.page import Page - Articles = OrderedDict[str, 'Article'] + from .ukbot import FilterTemplate, SiteManager # type: ignore[attr-defined] + from .article import Article # noqa: F401 + +Articles = OrderedDict[str, 'Article'] logger = logging.getLogger(__name__) @@ -49,6 +49,7 @@ def requests_retry_session( class CategoryLoopError(Exception): """Raised when a category loop is found.""" + def __init__(self, catpath): """ Args: @@ -66,7 +67,7 @@ def __init__(self, sites: 'SiteManager'): sites (SiteManager): A SiteManager instance with the sites relevant for this filter. """ self.sites = sites - self.page_keys = set() + self.page_keys: Set[str] = set() @classmethod def make(cls, tpl: 'FilterTemplate', **kwargs): @@ -87,7 +88,7 @@ def filter(self, articles: 'Articles'): type(self).__name__, len(articles), len(out)) return out -#class StubFilter(Filter): +# class StubFilter(Filter): # """ Filters articles that was stubs, but is no more """ # def __init__(self): @@ -103,7 +104,7 @@ def filter(self, articles: 'Articles'): # return True # return False - #def filter(self, articles): + # def filter(self, articles): # out = OrderedDict() # for article_key, article in articles.items(): @@ -130,11 +131,11 @@ def filter(self, articles: 'Articles'): # if self.verbose: # log('') - #except DanmicholoParseError as e: - # log(" >> DanmicholoParser failed to parse " + article_key) - # parentid = firstrev.parentid - # args = { 'article': article_key, 'prevrev': firstrev.parentid, 'rev': lastrev.revid, 'error': e.msg } - # article.site().errors.append(_('Could not analyze the article %(article)s because one of the revisions %(prevrev)d or %(rev)d could not be parsed: %(error)s') % args) + # except DanmicholoParseError as e: + # log(" >> DanmicholoParser failed to parse " + article_key) + # parentid = firstrev.parentid + # args = { 'article': article_key, 'prevrev': firstrev.parentid, 'rev': lastrev.revid, 'error': e.msg } + # article.site().errors.append(_('Could not analyze the article %(article)s because one of the revisions %(prevrev)d or %(rev)d could not be parsed: %(error)s') % args) # log(" [+] Applying stub filter: %d -> %d" % (len(articles), len(out))) @@ -228,6 +229,8 @@ def get_ignore_list(tpl: 'FilterTemplate', page_name: str): try: m = re.search(r'
(.*?)
', catignore_txt, flags=re.DOTALL) + if m is None: + return [] return m.group(1).strip().splitlines() except (IndexError, KeyError): raise RuntimeError(_('Could not parse the catignore page')) @@ -276,7 +279,7 @@ def __init__(self, sites: 'SiteManager', categories: List[Union['Page', Wildcard if not isinstance(page, WildcardPage) ] - self.categories_cache = {site_key: {} for site_key in self.sites.keys()} + self.categories_cache: Dict[str, Dict[str, Set[str]]] = {site_key: {} for site_key in self.sites.keys()} # Sites for which we should accept all contributions self.wildcard_include = [ @@ -331,8 +334,8 @@ def add_to_category_cache(self, articles: 'Articles'): logger.debug('CatFilter [%s, level %d]: Cache hits: %d, cache misses: %d', site_key, level, len(titles0) - len(cache_misses), len(cache_misses)) for s0 in range(0, len(cache_misses), requestlimit): - logger.debug('CatFilter [%s, level %d] Fetching categories for %d pages. Batch %d to %d', site_key, level, len(cache_misses), s0, s0+requestlimit) - ids = '|'.join(cache_misses[s0:s0+requestlimit]) + logger.debug('CatFilter [%s, level %d] Fetching categories for %d pages. Batch %d to %d', site_key, level, len(cache_misses), s0, s0 + requestlimit) + ids = '|'.join(cache_misses[s0:s0 + requestlimit]) cont = True clcont = {'continue': ''} @@ -428,9 +431,9 @@ def filter(self, articles: 'Articles') -> 'Articles': # It should be slightly more efficient to first populate a list, and then # convert it a set for uniqueness, rather than working with a set all the way. - categories = set(categories) + categories_set = set(categories) - matching_category = self.get_first_matching_category(categories) + matching_category = self.get_first_matching_category(categories_set) if matching_category is not None: out[article_key] = article try: @@ -457,11 +460,6 @@ def get_first_matching_category(self, article_cats: set) -> Optional[str]: @staticmethod def get_category_path(members: dict, category_key: str, article_key: str) -> List[str]: - """ - Try to find a path from a category to an article: - - [category_key, ..., article_key] - """ cat_path = [category_key] i = 0 while category_key != article_key: @@ -489,7 +487,7 @@ def make(cls, tpl, sites, **kwargs): def __init__(self, sites, bytelimit): """ - The ByteFilter keeps pages with a byte size above a certain threshold limit. + The ByteFilter keeps pages with a byte size above a certain threshold limit. Args: sites (SiteManager): References to the sites part of this contest @@ -522,7 +520,7 @@ def __init__(self, sites, contest, redirects=False): """ The NewPageFilter keeps pages that was created within the timeframe of a contest. In order to encourage collaboration on articles, the filter does not discriminate - on which user created the page. + on which user created the page. Args: sites (SiteManager): References to the sites part of this contest @@ -593,7 +591,7 @@ def __init__(self, sites, pages, include_langlinks=False): Args: sites (SiteManager): References to the sites part of this contest pages(list): List of Page objects to extract links from. - include_langlinks(bool): Whether to include langlinked pages as well. This is useful for + include_langlinks(bool): Whether to include langlinked pages as well. This is useful for multi-language contests, but we can save some time by not checking them. """ Filter.__init__(self, sites) @@ -667,7 +665,7 @@ def make(cls, tpl, **kwargs): def __init__(self, sites, pages): """ Arguments: - sites (SiteManager): References to the sites part of this contest + sites (SiteManager): References to the sites part of this contest pages (list): list of Page objects """ Filter.__init__(self, sites) @@ -830,11 +828,9 @@ def add_wikidata_items(self, site, item_var): SELECT ?%(item)s WHERE { { %(query)s } - } - """ % { + } """ % { 'item': item_var, 'query': self.query, - 'site': site, } logger.debug('SparqlFilter: %s', query) diff --git a/ukbot/revision.py b/ukbot/revision.py index 97008de..6496c83 100644 --- a/ukbot/revision.py +++ b/ukbot/revision.py @@ -106,7 +106,7 @@ def words(self): return 0 try: return self._wordcount - except: + except AttributeError: pass mt1 = get_body_text(re.sub('', '', self.text)) @@ -148,8 +148,8 @@ def words(self): logger.warning(w) self.errors.append(w) - #s = _('A problem encountered with revision %(revid)d may have influenced the word count for this revision: %(problems)s ') - #s = _('Et problem med revisjon %d kan ha påvirket ordtellingen for denne: %s ') + # s = _('A problem encountered with revision %(revid)d may have influenced the word count for this revision: %(problems)s ') + # s = _('Et problem med revisjon %d kan ha påvirket ordtellingen for denne: %s ') del mt1 del mt0 # except DanmicholoParseError as e: @@ -174,7 +174,6 @@ def get_link(self, homesite): """ returns a link to revision """ iw_prefix = '' if self.article().site().host != homesite.host: - homelang = homesite.host.split('.')[0] homefam = homesite.host.split('.')[1] targetlang = self.article().site().host.split('.')[0] targetfam = self.article().site().host.split('.')[1] diff --git a/ukbot/rules/__init__.py b/ukbot/rules/__init__.py index db19db1..9207eba 100644 --- a/ukbot/rules/__init__.py +++ b/ukbot/rules/__init__.py @@ -32,4 +32,3 @@ WordRule, WordBonusRule, ] - diff --git a/ukbot/rules/rule.py b/ukbot/rules/rule.py index b5c8162..c7c13d7 100644 --- a/ukbot/rules/rule.py +++ b/ukbot/rules/rule.py @@ -6,8 +6,7 @@ class Rule(object): - - rule_name = None + rule_name: str = "" def __init__(self, sites, params, trans=None): self.sites = sites @@ -74,4 +73,3 @@ def test(self, current_rev): if total >= self.limit and this_rev is True: yield UserContribution(rev=current_rev, points=self.points, rule=self, description=_('bonus %(words)d words') % {'words': self.limit}) - diff --git a/ukbot/rules/templateremoval.py b/ukbot/rules/templateremoval.py index c729eda..d0d123a 100644 --- a/ukbot/rules/templateremoval.py +++ b/ukbot/rules/templateremoval.py @@ -86,4 +86,3 @@ def test(self, rev): template['total'] += removed yield UserContribution(rev=rev, rule=self, points=removed * self.points, description=_('removal of {{tl|%(template)s}}') % {'template': template['name']}) - diff --git a/ukbot/server.py b/ukbot/server.py index e1ed41e..d9004b6 100755 --- a/ukbot/server.py +++ b/ukbot/server.py @@ -15,4 +15,3 @@ app.logger.addHandler(fh) # app.debug = True # reload on each code change - diff --git a/ukbot/site.py b/ukbot/site.py index 4890d32..e9edbc0 100644 --- a/ukbot/site.py +++ b/ukbot/site.py @@ -6,7 +6,7 @@ import mwclient from requests import Session from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry +from urllib3.util.retry import Retry from requests_oauthlib import OAuth1 diff --git a/ukbot/ukbot.py b/ukbot/ukbot.py index 3539826..02ca53d 100644 --- a/ukbot/ukbot.py +++ b/ukbot/ukbot.py @@ -1,34 +1,33 @@ # encoding=utf-8 # vim: fenc=utf-8 et sw=4 ts=4 sts=4 ai +from .sites import init_sites +from .contests import discover_contest_pages +from .contest import Contest +from .util import load_config +from .common import get_mem_usage, Localization, _, STATE_NORMAL, InvalidContestPage +from importlib.metadata import version +from dotenv import load_dotenv +import platform +from mwtemplates import TemplateEditor +import mwclient +import argparse +import os +import json +from datetime import datetime +import matplotlib +import logging +import sys import time runstart_s = time.time() print('Loading') -import sys -import logging -import matplotlib -from datetime import datetime -import pytz -import json -import os -import argparse -import mwclient -from mwtemplates import TemplateEditor -import platform -from dotenv import load_dotenv -from importlib.metadata import version - -from .common import get_mem_usage, Localization, _, STATE_NORMAL, InvalidContestPage -from .util import load_config -from .contest import Contest -from .contests import discover_contest_pages -from .sites import init_sites matplotlib.use('svg') __version__ = version('ukbot') + class AppFilter(logging.Filter): @staticmethod diff --git a/ukbot/user.py b/ukbot/user.py index 27fd70a..2392243 100644 --- a/ukbot/user.py +++ b/ukbot/user.py @@ -1,7 +1,6 @@ # encoding=utf-8 # vim: fenc=utf-8 et sw=4 ts=4 sts=4 ai import logging -import re import time from collections import OrderedDict from copy import copy @@ -11,7 +10,6 @@ import numpy as np from datetime import datetime import pytz -import pymysql from more_itertools import first from .contributions import UserContributions @@ -170,12 +168,12 @@ def add_contribs_from_wiki(self, site, start, end, fulltext=False, **kwargs): # 2) Check if pages are redirects (this information can not be cached, because other users may make the page a redirect) # If we fail to notice a redirect, the contributions to the page will be double-counted, so lets check - #titles = [a.name for a in self.articles.values() if a.site.key == site_key] - #for s0 in range(0, len(titles), apilim): - # ids = '|'.join(titles[s0:s0+apilim]) - # for page in site.api('query', prop = 'info', titles = ids)['query']['pages'].values(): - # article_key = site_key + ':' + page['title'] - # self.articles[article_key].redirect = ('redirect' in page.keys()) + # titles = [a.name for a in self.articles.values() if a.site.key == site_key] + # for s0 in range(0, len(titles), apilim): + # ids = '|'.join(titles[s0:s0+apilim]) + # for page in site.api('query', prop = 'info', titles = ids)['query']['pages'].values(): + # article_key = site_key + ':' + page['title'] + # self.articles[article_key].redirect = ('redirect' in page.keys()) # 3) Fetch info about the new revisions: diff size, possibly content @@ -340,7 +338,7 @@ def save_contribs_to_db(self, sql): # Insert all revisions chunk_size = 1000 for n in range(0, len(contribs_query_params), chunk_size): - data = contribs_query_params[n:n+chunk_size] + data = contribs_query_params[n:n + chunk_size] # logger.info('Adding %d contributions to database', len(data)) t0 = time.time() @@ -348,13 +346,13 @@ def save_contribs_to_db(self, sql): insert into contribs (revid, site, parentid, user, page, timestamp, size, parentsize, parsedcomment, ns) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) """, data - ) + ) dt = time.time() - t0 logger.info('Added %d contributions to database in %.2f secs', len(data), dt) chunk_size = 100 for n in range(0, len(fulltexts_query_params), chunk_size): - data = fulltexts_query_params[n:n+chunk_size] + data = fulltexts_query_params[n:n + chunk_size] # logger.info('Adding %d fulltexts to database', len(data)) t0 = time.time() @@ -363,7 +361,7 @@ def save_contribs_to_db(self, sql): values (%s,%s,%s) on duplicate key update revtxt=values(revtxt); """, data - ) + ) dt = time.time() - t0 logger.info('Added %d fulltexts to database in %.2f secs', len(data), dt) @@ -457,7 +455,7 @@ def add_contribs_from_db(self, sql, start, end, sites): continue # Add article if not present - if not article_key in self.articles: + if article_key not in self.articles: narts += 1 self.articles[article_key] = Article(sites[site_key], self, article_title, ns) if article_key in self.disqualified_articles: @@ -465,10 +463,10 @@ def add_contribs_from_db(self, sql, start, end, sites): article = self.articles[article_key] # Add revision if not present - if not rev_id in self.revisions: + if rev_id not in self.revisions: nrevs += 1 article.add_revision(rev_id, timestamp=ts, parentid=parent_id, size=size, parentsize=parentsize, - username=self.name, parsedcomment=parsedcomment, text=rev_text, parenttext=parent_rev_txt) + username=self.name, parsedcomment=parsedcomment, text=rev_text, parenttext=parent_rev_txt) rev = self.revisions[rev_id] rev.saved = True @@ -602,41 +600,40 @@ def analyze(self, rules): o = np.argsort(x) x = x[o] y = y[o] - #pl = np.array(pl, dtype=float) - #pl.sort(axis = 0) + # pl = np.array(pl, dtype=float) + # pl.sort(axis = 0) y2 = np.array([np.sum(y[:q + 1]) for q in range(len(y))]) self.plotdata = np.column_stack((x, y2)) - #np.savetxt('user-%s'%self.name, np.column_stack((x,y,y2))) + # np.savetxt('user-%s'%self.name, np.column_stack((x,y,y2))) def format_result(self): logger.debug('Formatting results for user %s', self.name) entries = [] - - ## WIP + # WIP # <<<<<<< HEAD - # dt = utc.localize(datetime.fromtimestamp(rev.timestamp)) - # dt_str = dt.astimezone(self.contest().wiki_tz).strftime(_('%d.%m, %H:%M')) - # out = '[%s %s]: %s' % (rev.get_link(), dt_str, descr) - # if self.suspended_since is not None and dt > self.suspended_since: - # out = '' + out + '' - # if len(rev.errors) > 0: - # out = '[[File:Ambox warning yellow.svg|12px|%s]] ' % (', '.join(rev.errors)) + out - # revs.append(out) - - # titletxt = '' - # try: - # cat_path = [x.split(':')[-1] for x in article.cat_path] - # titletxt = ' : '.join(cat_path) + '
' - # except AttributeError: - # pass - # titletxt += '
'.join(revs) - # # if len(article.point_deductions) > 0: - # # pds = [] - # # for points, reason in article.point_deductions: - # # pds.append('%.f p: %s' % (-points, reason)) - # # titletxt += '
\'\'' + _('Notes') + ':\'\'
%s
' % '
'.join(pds) + # dt = utc.localize(datetime.fromtimestamp(rev.timestamp)) + # dt_str = dt.astimezone(self.contest().wiki_tz).strftime(_('%d.%m, %H:%M')) + # out = '[%s %s]: %s' % (rev.get_link(), dt_str, descr) + # if self.suspended_since is not None and dt > self.suspended_since: + # out = '' + out + '' + # if len(rev.errors) > 0: + # out = '[[File:Ambox warning yellow.svg|12px|%s]] ' % (', '.join(rev.errors)) + out + # revs.append(out) + + # titletxt = '' + # try: + # cat_path = [x.split(':')[-1] for x in article.cat_path] + # titletxt = ' : '.join(cat_path) + '
' + # except AttributeError: + # pass + # titletxt += '
'.join(revs) + # # if len(article.point_deductions) > 0: + # # pds = [] + # # for points, reason in article.point_deductions: + # # pds.append('%.f p: %s' % (-points, reason)) + # # titletxt += '
\'\'' + _('Notes') + ':\'\'
%s
' % '
'.join(pds) # titletxt += '
' + _('Total {{formatnum:%(bytecount)d}} bytes, %(wordcount)d words') % {'bytecount': article.bytes, 'wordcount': article.words} + '
' diff --git a/ukbot/util.py b/ukbot/util.py index 42477fb..b92842e 100644 --- a/ukbot/util.py +++ b/ukbot/util.py @@ -11,7 +11,7 @@ from datetime import datetime from datetime import time as dt_time from isoweek import Week # Sort-of necessary until datetime supports %V, see http://bugs.python.org/issue12006 - # and http://stackoverflow.com/questions/5882405/get-date-from-iso-week-number-in-python +# and http://stackoverflow.com/questions/5882405/get-date-from-iso-week-number-in-python from mwtemplates import TemplateEditor logger = logging.getLogger(__name__) diff --git a/ukbot/webinterface/app.py b/ukbot/webinterface/app.py index 3681ab7..e3b016f 100644 --- a/ukbot/webinterface/app.py +++ b/ukbot/webinterface/app.py @@ -93,7 +93,7 @@ def read_status(fname): status = json.load(fp) except IOError: logger.error('Could not read status file: %s', fname) - return 'Could not read status'; + return 'Could not read status' args = { 'job_status': status.get('status'), @@ -310,7 +310,7 @@ def update_contest(): config_file = rows[0][0] page_name = rows[0][1] - last_job_id = rows[0][2] + # last_job_id = rows[0][2] if config_file is None: return redirect('contests?%s' % urllib.parse.urlencode({ diff --git a/ukbot/webinterface/static/.htaccess b/ukbot/webinterface/static/.htaccess index ee4ef8f..dbf8627 100644 --- a/ukbot/webinterface/static/.htaccess +++ b/ukbot/webinterface/static/.htaccess @@ -10,4 +10,3 @@ RewriteRule ^(.*)$ /ukbot/webfront.py/$1 [L] #RewriteEngine On #RewriteCond %{REQUEST_FILENAME} !-f # Don't interfere with static files #RewriteRule ^(.*)$ /ukbot/application.cgi/$1 [L] - diff --git a/ukbot/webinterface/static/assets/d3.v3.js b/ukbot/webinterface/static/assets/d3.v3.js index a098b9a..d9a7398 100644 --- a/ukbot/webinterface/static/assets/d3.v3.js +++ b/ukbot/webinterface/static/assets/d3.v3.js @@ -1683,7 +1683,7 @@ d3 = function() { return xhr; }; d3.rebind(xhr, dispatch, "on"); - if (arguments.length === 2 && typeof mimeType === "function") callback = mimeType, + if (arguments.length === 2 && typeof mimeType === "function") callback = mimeType, mimeType = null; return callback == null ? xhr : xhr.get(d3_xhr_fixCallback(callback)); }; @@ -1997,7 +1997,7 @@ d3 = function() { if (d3_format_grouping) { var d3_format_groupingLength = d3_format_grouping.length; d3_format_group = function(value) { - var i = value.lastIndexOf("."), f = i >= 0 ? "." + value.substring(i + 1) : (i = value.length, + var i = value.lastIndexOf("."), f = i >= 0 ? "." + value.substring(i + 1) : (i = value.length, ""), t = [], j = 0, g = d3_format_grouping[0]; while (i > 0 && g > 0) { t.push(value.substring(i -= g, i + g)); @@ -2098,7 +2098,7 @@ d3 = function() { var λ00, φ00, λ0, cosφ0, sinφ0; d3_geo_area.point = function(λ, φ) { d3_geo_area.point = nextPoint; - λ0 = (λ00 = λ) * d3_radians, cosφ0 = Math.cos(φ = (φ00 = φ) * d3_radians / 2 + π / 4), + λ0 = (λ00 = λ) * d3_radians, cosφ0 = Math.cos(φ = (φ00 = φ) * d3_radians / 2 + π / 4), sinφ0 = Math.sin(φ); }; function nextPoint(λ, φ) { @@ -3887,7 +3887,7 @@ d3 = function() { return points.length < 4 ? d3_svg_lineLinear(points) : points[1] + d3_svg_lineHermite(points.slice(1, points.length - 1), d3_svg_lineCardinalTangents(points, tension)); } function d3_svg_lineCardinalClosed(points, tension) { - return points.length < 3 ? d3_svg_lineLinear(points) : points[0] + d3_svg_lineHermite((points.push(points[0]), + return points.length < 3 ? d3_svg_lineLinear(points) : points[0] + d3_svg_lineHermite((points.push(points[0]), points), d3_svg_lineCardinalTangents([ points[points.length - 2] ].concat(points, [ points[1] ]), tension)); } function d3_svg_lineCardinal(points, tension) { @@ -4055,7 +4055,7 @@ d3 = function() { function hull(data) { if (data.length < 3) return []; var fx = d3_functor(x), fy = d3_functor(y), n = data.length, vertices, plen = n - 1, points = [], stack = [], d, i, j, h = 0, x1, y1, x2, y2, u, v, a, sp; - if (fx === d3_svg_lineX && y === d3_svg_lineY) vertices = data; else for (i = 0, + if (fx === d3_svg_lineX && y === d3_svg_lineY) vertices = data; else for (i = 0, vertices = []; i < n; ++i) { vertices.push([ +fx.call(this, d = data[i], i), +fy.call(this, d, i) ]); } @@ -4216,7 +4216,7 @@ d3 = function() { var points, polygons = data.map(function() { return []; }), fx = d3_functor(x), fy = d3_functor(y), d, i, n = data.length, Z = 1e6; - if (fx === d3_svg_lineX && fy === d3_svg_lineY) points = data; else for (points = [], + if (fx === d3_svg_lineX && fy === d3_svg_lineY) points = data; else for (points = [], i = 0; i < n; ++i) { points.push([ +fx.call(this, d = data[i], i), +fy.call(this, d, i) ]); } @@ -6383,7 +6383,7 @@ d3 = function() { return d3_layout_treemapPad(node, x); } var type; - pad = (padding = x) == null ? d3_layout_treemapPadNull : (type = typeof x) === "function" ? padFunction : type === "number" ? (x = [ x, x, x, x ], + pad = (padding = x) == null ? d3_layout_treemapPadNull : (type = typeof x) === "function" ? padFunction : type === "number" ? (x = [ x, x, x, x ], padConstant) : padConstant; return treemap; }; @@ -6602,7 +6602,7 @@ d3 = function() { }; scale.domain = function(x) { if (!arguments.length) return domain; - if (x[0] < 0) log = d3_scale_logn, pow = d3_scale_pown; else log = d3_scale_logp, + if (x[0] < 0) log = d3_scale_logn, pow = d3_scale_pown; else log = d3_scale_logp, pow = d3_scale_powp; linear.domain((domain = x.map(Number)).map(log)); return scale; @@ -6636,7 +6636,7 @@ d3 = function() { scale.tickFormat = function(n, format) { if (arguments.length < 2) format = d3_scale_logFormat; if (!arguments.length) return format; - var b = Math.log(base), k = Math.max(.1, n / scale.ticks().length), f = log === d3_scale_logn ? (e = -1e-12, + var b = Math.log(base), k = Math.max(.1, n / scale.ticks().length), f = log === d3_scale_logn ? (e = -1e-12, Math.floor) : (e = 1e-12, Math.ceil), e; return function(d) { return d / pow(b * f(log(d) / b + e)) <= k ? format(d) : ""; @@ -6935,7 +6935,7 @@ d3 = function() { d3.svg.arc = function() { var innerRadius = d3_svg_arcInnerRadius, outerRadius = d3_svg_arcOuterRadius, startAngle = d3_svg_arcStartAngle, endAngle = d3_svg_arcEndAngle; function arc() { - var r0 = innerRadius.apply(this, arguments), r1 = outerRadius.apply(this, arguments), a0 = startAngle.apply(this, arguments) + d3_svg_arcOffset, a1 = endAngle.apply(this, arguments) + d3_svg_arcOffset, da = (a1 < a0 && (da = a0, + var r0 = innerRadius.apply(this, arguments), r1 = outerRadius.apply(this, arguments), a0 = startAngle.apply(this, arguments) + d3_svg_arcOffset, a1 = endAngle.apply(this, arguments) + d3_svg_arcOffset, da = (a1 < a0 && (da = a0, a0 = a1, a1 = da), a1 - a0), df = da < π ? "0" : "1", c0 = Math.cos(a0), s0 = Math.sin(a0), c1 = Math.cos(a1), s1 = Math.sin(a1); return da >= d3_svg_arcMax ? r0 ? "M0," + r1 + "A" + r1 + "," + r1 + " 0 1,1 0," + -r1 + "A" + r1 + "," + r1 + " 0 1,1 0," + r1 + "M0," + r0 + "A" + r0 + "," + r0 + " 0 1,0 0," + -r0 + "A" + r0 + "," + r0 + " 0 1,0 0," + r0 + "Z" : "M0," + r1 + "A" + r1 + "," + r1 + " 0 1,1 0," + -r1 + "A" + r1 + "," + r1 + " 0 1,1 0," + r1 + "Z" : r0 ? "M" + r1 * c0 + "," + r1 * s0 + "A" + r1 + "," + r1 + " 0 " + df + ",1 " + r1 * c1 + "," + r1 * s1 + "L" + r0 * c1 + "," + r0 * s1 + "A" + r0 + "," + r0 + " 0 " + df + ",0 " + r0 * c0 + "," + r0 * s0 + "Z" : "M" + r1 * c0 + "," + r1 * s0 + "A" + r1 + "," + r1 + " 0 " + df + ",1 " + r1 * c1 + "," + r1 * s1 + "L0,0" + "Z"; } @@ -7529,7 +7529,7 @@ d3 = function() { var ticks = tickValues == null ? scale.ticks ? scale.ticks.apply(scale, tickArguments_) : scale.domain() : tickValues, tickFormat = tickFormat_ == null ? scale.tickFormat ? scale.tickFormat.apply(scale, tickArguments_) : String : tickFormat_; var subticks = d3_svg_axisSubdivide(scale, ticks, tickSubdivide), subtick = g.selectAll(".tick.minor").data(subticks, String), subtickEnter = subtick.enter().insert("line", ".tick").attr("class", "tick minor").style("opacity", 1e-6), subtickExit = d3.transition(subtick.exit()).style("opacity", 1e-6).remove(), subtickUpdate = d3.transition(subtick).style("opacity", 1); var tick = g.selectAll(".tick.major").data(ticks, String), tickEnter = tick.enter().insert("g", "path").attr("class", "tick major").style("opacity", 1e-6), tickExit = d3.transition(tick.exit()).style("opacity", 1e-6).remove(), tickUpdate = d3.transition(tick).style("opacity", 1), tickTransform; - var range = d3_scaleRange(scale), path = g.selectAll(".domain").data([ 0 ]), pathUpdate = (path.enter().append("path").attr("class", "domain"), + var range = d3_scaleRange(scale), path = g.selectAll(".domain").data([ 0 ]), pathUpdate = (path.enter().append("path").attr("class", "domain"), d3.transition(path)); var scale1 = scale.copy(), scale0 = this.__chart__ || scale1; this.__chart__ = scale1; @@ -8549,4 +8549,4 @@ d3 = function() { return request.responseXML; } return d3; -}(); \ No newline at end of file +}(); diff --git a/ukbot/webinterface/static/plots b/ukbot/webinterface/static/plots index eb50f7d..7fca0f2 120000 --- a/ukbot/webinterface/static/plots +++ b/ukbot/webinterface/static/plots @@ -1 +1 @@ -../../../plots \ No newline at end of file +../../../plots diff --git a/ukbot/webinterface/templates/contests.html b/ukbot/webinterface/templates/contests.html index effc39b..6283ed2 100644 --- a/ukbot/webinterface/templates/contests.html +++ b/ukbot/webinterface/templates/contests.html @@ -76,5 +76,3 @@ {% endblock %} - - diff --git a/ukbot/webinterface/templates/home.html b/ukbot/webinterface/templates/home.html index 2b3204d..59ae46b 100644 --- a/ukbot/webinterface/templates/home.html +++ b/ukbot/webinterface/templates/home.html @@ -71,5 +71,3 @@

*/ {% endblock %} - - diff --git a/ukbot/webinterface/templates/layout.html b/ukbot/webinterface/templates/layout.html index ea9461d..94f3a9f 100644 --- a/ukbot/webinterface/templates/layout.html +++ b/ukbot/webinterface/templates/layout.html @@ -62,4 +62,3 @@ - diff --git a/ukbot/webinterface/templates/uk.html b/ukbot/webinterface/templates/uk.html index b15b40b..024aefa 100644 --- a/ukbot/webinterface/templates/uk.html +++ b/ukbot/webinterface/templates/uk.html @@ -74,7 +74,7 @@ var g = vis.append("svg:g") .attr("transform", "translate(0, 200)"); - + var line = d3.svg.line(); w = 400, h = 200, @@ -89,13 +89,13 @@ var g = vis.append("svg:g") .attr("transform", "translate(0, 200)"); - + var line = d3.svg.line() .x(function(d,i) { return x(i); }) .y(function(d) { return -1 * y(d); }) - + g.append("svg:path").attr("d", line(data)); - + g.append("svg:line") .attr("x1", x(0)) .attr("y1", -1 * y(0)) @@ -107,7 +107,7 @@ .attr("y1", -1 * y(0)) .attr("x2", x(0)) .attr("y2", -1 * y(d3.max(data))) - + g.selectAll(".xLabel") .data(x.ticks(5)) .enter().append("svg:text") @@ -126,7 +126,7 @@ .attr("y", function(d) { return -1 * y(d) }) .attr("text-anchor", "right") .attr("dy", 4) - + g.selectAll(".xTicks") .data(x.ticks(5)) .enter().append("svg:line") @@ -145,9 +145,9 @@ .attr("y2", function(d) { return -1 * y(d); }) .attr("x2", x(0)).x(function(d,i) { return x(i); }) .y(function(d) { return -1 * y(d); }) - + g.append("svg:path").attr("d", line(data)); - + g.append("svg:line") .attr("x1", x(0)) .attr("y1", -1 * y(0)) @@ -159,7 +159,7 @@ .attr("y1", -1 * y(0)) .attr("x2", x(0)) .attr("y2", -1 * y(d3.max(data))) - + g.selectAll(".xLabel") .data(x.ticks(5)) .enter().append("svg:text") @@ -178,7 +178,7 @@ .attr("y", function(d) { return -1 * y(d) }) .attr("text-anchor", "right") .attr("dy", 4) - + g.selectAll(".xTicks") .data(x.ticks(5)) .enter().append("svg:line") diff --git a/ukbot/webinterface/templates/uk_user_details.html b/ukbot/webinterface/templates/uk_user_details.html index 9b35798..13e6a93 100644 --- a/ukbot/webinterface/templates/uk_user_details.html +++ b/ukbot/webinterface/templates/uk_user_details.html @@ -32,7 +32,7 @@

{{ name }}

{{ contest['name'] }} - {{ contest['pos'] }}. plass av + {{ contest['pos'] }}. plass av {{ contest['cnt'] }} diff --git a/www/python/src/app.py b/www/python/src/app.py index 2f19d21..5effff7 100644 --- a/www/python/src/app.py +++ b/www/python/src/app.py @@ -1 +1,3 @@ from ukbot.server import app + +application = app # for WSGI servers diff --git a/www/python/src/websocket.py b/www/python/src/websocket.py index 51da6a3..f3bbf9a 100644 --- a/www/python/src/websocket.py +++ b/www/python/src/websocket.py @@ -42,7 +42,7 @@ def app(env, start_response): new_data = stream.read() if new_data != '': uwsgi.websocket_send(new_data) - #if 'Job finished contest' in new_data: + # if 'Job finished contest' in new_data: # close_next_time = True if close_next_time: logger.info(f"Completed streaming log: {log_file}") @@ -61,4 +61,3 @@ def app(env, start_response): logger.info(f"Status file {status_file} not found") n += 1 time.sleep(1) -