diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e59ae0f..6ea760e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: "actions/checkout@v3" - uses: "actions/setup-python@v4" diff --git a/.gitignore b/.gitignore index daef807..218d2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,5 @@ docs/_build/ # PyBuilder target/ + +.venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ff61df7..15266b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,13 +2,14 @@ default_language_version: python: python3 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v5.0.0 hooks: - id: check-toml - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/PyCQA/flake8 - rev: "3.9.2" + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.0 hooks: - - id: flake8 + - id: ruff + args: [--fix] diff --git a/README.rst b/README.rst index 4a74110..d4fd885 100644 --- a/README.rst +++ b/README.rst @@ -43,6 +43,27 @@ Create a ``repos.yaml`` or ``repos.yml`` file: fetch_all: - oca + ./gitlab-repo-example: + remotes: + gitlab_1: https://giltab.com/sample-org/sample-repo.git + gitlab_2: https://giltab.com/another-sample-org/sample-repo-fork.git + merges: + - gitlab_1 main + - gitlab_2 merge-requests/123/head + target: gitlab_1 + + # FYI: Bitbucket doesn't support fetching PR's + ./bitbucket-repo-example: + remotes: + bitbucket_1: https://bitbucket.org/sample-org/sample-repo.git + bitbucket_2: https://bitbucker.org/another-sample-org/sample-repo-fork.git + merges: + - bitbucket_1 main + - bitbucket_2 dev + target: bitbucket_1 + + + Environment variables inside of this file will be expanded if the proper option is selected. All the ``merges`` are combined into a single branch. By default this branch is called ``_git_aggregated`` but another name may be given in the ``target`` section. @@ -107,6 +128,38 @@ the ``dict`` alternate construction. If you need to disable a default in ref: refs/pull/14859/head target: acsone 9.0 + ./gitlab-repo-example: + defaults: + depth: 20 + remotes: + gitlab_1: https://giltab.com/sample-org/sample-repo.git + gitlab_2: https://giltab.com/another-sample-org/sample-repo-fork.git + merges: + - + remote: gitlab_1 + ref: main + depth: 20 + - + remote: gitlab_2 + ref: merge-requests/123/head + target: gitlab_1 + + # FYI: Bitbucket doesn't support fetching PR's + ./bitbucket-repo-example: + remotes: + bitbucket_1: https://bitbucket.org/sample-org/sample-repo.git + bitbucket_2: https://bitbucker.org/another-sample-org/sample-repo-fork.git + merges: + - + remote: bitbucket_1 + ref: main + depth: 3 + - + remote: bitbucket_2 + ref: dev + target: bitbucket_1 + + Remember that you need to fetch at least the common ancestor of all merges for it to succeed. @@ -224,6 +277,23 @@ To work around API limitation, you must first generate a Changes ======= +4.1 (2025-03-25) +---------------- + +* Update README for other forges than GitHub (`#88 `_) +* Internal API change (`#91 `_) +* Drop support for unsupported Python versions (`#92 `_) + +4.0.2 (2024-10-23) +------------------ + +* target_dir may be an empty directory (`#83 `_) + +4.0.1 (2024-06-04) +------------------ + +* fix: git remote: change url instead of rm / add (`#81 `_`) + 4.0 (2023-07-22) ---------------- diff --git a/git_aggregator/__init__.py b/git_aggregator/__init__.py index 77ab94a..b868204 100644 --- a/git_aggregator/__init__.py +++ b/git_aggregator/__init__.py @@ -1,3 +1,2 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) diff --git a/git_aggregator/_compat.py b/git_aggregator/_compat.py index 9a4f757..678ac5e 100644 --- a/git_aggregator/_compat.py +++ b/git_aggregator/_compat.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) import sys diff --git a/git_aggregator/config.py b/git_aggregator/config.py index 878ad9f..266d284 100644 --- a/git_aggregator/config.py +++ b/git_aggregator/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) @@ -8,10 +7,9 @@ import yaml -from .exception import ConfigException from ._compat import string_types from .repo import Repo, ishex - +from .exception import ConfigException log = logging.getLogger(__name__) @@ -37,6 +35,8 @@ def get_repos(config, force=False, skip_merge_check=False): 'apply_patch': repo_data.get('apply_patch', False), 'skip_repo_init': repo_data.get('skip_repo_init', False), } + if 'retry' in repo_data: + repo_dict['retry'] = repo_data['retry'] remote_names = set() if 'remotes' in repo_data: repo_dict['remotes'] = [] @@ -57,7 +57,10 @@ def get_repos(config, force=False, skip_merge_check=False): '%s: You should at least define one remote.' % directory) else: try: - tmp_repo = Repo(repo_dict['cwd'], [], [], None) + tmp_repo = Repo( + repo_dict['cwd'], [], [], None, + retry={"max_retries": 0}, + ) remotes = tmp_repo._get_remotes() repo_dict['remotes'] = [] for remote_name, url in remotes.items(): @@ -73,7 +76,10 @@ def get_repos(config, force=False, skip_merge_check=False): merge_data = repo_data.get('merges') or [] tmp_repo = None if not skip_merge_check: - tmp_repo = Repo(repo_dict['cwd'], [], [], None) + tmp_repo = Repo( + repo_dict['cwd'], [], [], None, + retry={"max_retries": 0}, + ) if os.path.exists(tmp_repo.cwd): # Set remotes for remote in repo_dict['remotes']: @@ -207,11 +213,11 @@ def load_config( key, value = line.split('=') environment.update({key.strip(): value.strip()}) environment.update(os.environ) - with open(config, 'r') as file_handler: + with open(config) as file_handler: config = Template(file_handler.read()) config = config.substitute(environment) else: - config = open(config, 'r').read() + config = open(config).read() conf = yaml.load(config, Loader=yaml.SafeLoader) diff --git a/git_aggregator/exception.py b/git_aggregator/exception.py index 86d6a94..0f25940 100644 --- a/git_aggregator/exception.py +++ b/git_aggregator/exception.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) diff --git a/git_aggregator/log.py b/git_aggregator/log.py index c1cfe0e..5219ce0 100644 --- a/git_aggregator/log.py +++ b/git_aggregator/log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) diff --git a/git_aggregator/main.py b/git_aggregator/main.py index 22291ad..696bdca 100644 --- a/git_aggregator/main.py +++ b/git_aggregator/main.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015-2019 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) @@ -7,22 +6,24 @@ import sys import threading import traceback + try: - from Queue import Queue, Empty as EmptyQueue + from Queue import Empty as EmptyQueue + from Queue import Queue except ImportError: - from queue import Queue, Empty as EmptyQueue + from queue import Empty as EmptyQueue + from queue import Queue import argparse +import fnmatch + import argcomplete import colorama -import fnmatch -from .utils import ThreadNameKeeper -from .log import DebugLogFormatter -from .log import LogFormatter from .config import load_config +from .log import DebugLogFormatter, LogFormatter from .repo import Repo - +from .utils import ThreadNameKeeper logger = logging.getLogger(__name__) @@ -31,8 +32,7 @@ def _log_level_string_to_int(log_level_string): if log_level_string not in _LOG_LEVEL_STRINGS: - message = 'invalid choice: {0} (choose from {1})'.format( - log_level_string, _LOG_LEVEL_STRINGS) + message = f'invalid choice: {log_level_string} (choose from {_LOG_LEVEL_STRINGS})' raise argparse.ArgumentTypeError(message) log_level_int = getattr(logging, log_level_string, logging.INFO) @@ -99,7 +99,7 @@ def get_parser(): dest='log_level', type=_log_level_string_to_int, nargs='?', - help='Set the logging output level. {0}'.format(_LOG_LEVEL_STRINGS)) + help=f'Set the logging output level. {_LOG_LEVEL_STRINGS}') main_parser.add_argument( '-e', '--expand-env', diff --git a/git_aggregator/repo.py b/git_aggregator/repo.py index 46609e0..aeccdd5 100644 --- a/git_aggregator/repo.py +++ b/git_aggregator/repo.py @@ -1,24 +1,33 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) # Parts of the code comes from ANYBOX # https://github.com/anybox/anybox.recipe.odoo -from __future__ import unicode_literals -import os import logging +import os import re import shutil import subprocess +import time import requests -from .exception import DirtyException, GitAggregatorException from ._compat import console_to_str +from .exception import DirtyException, GitAggregatorException FETCH_DEFAULTS = ("depth", "shallow-since", "shallow-exclude") +NETWORK_GIT_COMMANDS = ("fetch", "pull", "push", "ls-remote", "clone") +DEFAULT_RETRY = {"max_retries": 3, "delay": 5, "backoff_factor": 2} logger = logging.getLogger(__name__) +def _is_network_command(cmd): + """Detect if a git command is a network operation.""" + for part in cmd: + if part in NETWORK_GIT_COMMANDS: + return True + return False + + def ishex(s): """True iff given string is a valid hexadecimal number. >>> ishex('deadbeef') @@ -33,14 +42,14 @@ def ishex(s): return True -class Repo(object): +class Repo: _git_version = None def __init__(self, cwd, remotes, merges, target, shell_command_after=None, fetch_all=False, defaults=None, force=False, skip_dry_run=False, apply_patch=False, - skip_repo_init=False): + skip_repo_init=False, retry=None): """Initialize a git repository aggregator :param cwd: path to the directory where to initialize the repository @@ -81,6 +90,10 @@ def __init__(self, cwd, remotes, merges, target, self.skip_dry_run = skip_dry_run self.apply_patch = apply_patch self.skip_repo_init = skip_repo_init + if retry is None: + self.retry = dict(DEFAULT_RETRY) + else: + self.retry = retry @property def git_version(self): @@ -170,14 +183,42 @@ def log_call(self, cmd, callwith=subprocess.check_call, :param meth: the calling method to use. """ logger.log(log_level, "%s> call %r", self.cwd, cmd) - try: - ret = callwith(cmd, **kw) - except Exception: - logger.error("%s> error calling %r", self.cwd, cmd) - raise - if callwith == subprocess.check_output: - ret = console_to_str(ret) - return ret + max_retries = self.retry.get("max_retries", 0) + if max_retries > 0 and _is_network_command(cmd): + delay = self.retry.get("delay", DEFAULT_RETRY["delay"]) + backoff_factor = self.retry.get( + "backoff_factor", DEFAULT_RETRY["backoff_factor"] + ) + last_exception = None + for attempt in range(max_retries + 1): + try: + ret = callwith(cmd, **kw) + if callwith == subprocess.check_output: + ret = console_to_str(ret) + return ret + except Exception as exc: + last_exception = exc + if attempt < max_retries: + wait = delay * backoff_factor ** attempt + logger.warning( + "%s> Retry %d/%d for %r in %ds...", + self.cwd, attempt + 1, max_retries, cmd, wait, + ) + time.sleep(wait) + else: + logger.error( + "%s> error calling %r", self.cwd, cmd + ) + raise last_exception + else: + try: + ret = callwith(cmd, **kw) + except Exception: + logger.error("%s> error calling %r", self.cwd, cmd) + raise + if callwith == subprocess.check_output: + ret = console_to_str(ret) + return ret def _dry_run_cleanup(self, target_dir): if os.path.exists(target_dir): @@ -201,7 +242,7 @@ def aggregate(self, dry_run=False): target_dir = "%s-%s" % (target_dir, "dry-run") self.cwd = target_dir - is_new = not os.path.exists(target_dir) + is_new = not os.path.exists(target_dir) or os.listdir(target_dir) == [] if is_new: if self.skip_repo_init: logger.info( @@ -397,8 +438,8 @@ def _set_remote(self, name, url): else: logger.info('Updating remote %s <%s> -> <%s>', name, exising_url, url) - self.log_call(['git', 'remote', 'rm', name], cwd=self.cwd) - self.log_call(['git', 'remote', 'add', name, url], cwd=self.cwd) + self.log_call( + ['git', 'remote', 'set-url', name, url], cwd=self.cwd) def _github_api_get(self, path): url = 'https://api.github.com' + path @@ -408,7 +449,7 @@ def _github_api_get(self, path): headers = {'Authorization': 'token %s' % token} return requests.get(url, headers=headers) - def collect_prs_info(self): + def collect_prs_info(self, merges=None): """Collect all pending merge PRs info. :returns: mapping of PRs by state @@ -420,7 +461,7 @@ def collect_prs_info(self): '^(refs/)?pull/(?P[0-9]+)/head$') remotes = {r['name']: r['url'] for r in self.remotes} all_prs = {} - for merge in self.merges: + for merge in (merges or self.merges): remote = merge['remote'] ref = merge['ref'] repo_url = remotes[remote] diff --git a/git_aggregator/utils.py b/git_aggregator/utils.py index 063e787..1ecbb89 100644 --- a/git_aggregator/utils.py +++ b/git_aggregator/utils.py @@ -1,14 +1,14 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # © ANYBOX https://github.com/anybox/anybox.recipe.odoo # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) +import logging import os import threading -import logging + logger = logging.getLogger(__name__) -class WorkingDirectoryKeeper(object): # DEPRECATED +class WorkingDirectoryKeeper: # DEPRECATED """A context manager to get back the working directory as it was before. If you want to stack working directory keepers, you need a new instance for each stage. @@ -30,7 +30,7 @@ def __exit__(self, *exc_args): working_directory_keeper = WorkingDirectoryKeeper() -class ThreadNameKeeper(object): +class ThreadNameKeeper: """A contect manager to get back the thread name as it was before. It is meant to be used when modifying the 'MainThread' tread. """ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5c98ac7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.ruff] +fix = true +line-length = 79 + +[tool.ruff.lint] +extend-select = [ + "I", + "UP", +] +ignore = [ + "UP031", # % formatting +] + +[tool.ruff.lint.isort] +known-first-party = ["git_aggregator"] diff --git a/setup.py b/setup.py index d02c992..0256cf9 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) @@ -17,11 +16,11 @@ 'GNU Affero General Public License v3 or later (AGPLv3+)', 'Operating System :: POSIX', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', "Topic :: Utilities", "Topic :: System :: Shells", ], diff --git a/tests/__init__.py b/tests/__init__.py index 77ab94a..b868204 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,2 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) diff --git a/tests/test_config.py b/tests/test_config.py index 0c9fa50..ab2eba2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) import os @@ -9,8 +8,8 @@ import yaml from git_aggregator import config -from git_aggregator.exception import ConfigException from git_aggregator._compat import PY2 +from git_aggregator.exception import ConfigException class TestConfig(unittest.TestCase): diff --git a/tests/test_log.py b/tests/test_log.py index b588e2a..ef4a849 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) import logging @@ -20,7 +19,7 @@ class TestLog(unittest.TestCase): def setUp(self): """ Setup """ - super(TestLog, self).setUp() + super().setUp() reset_logger() def test_info(self): diff --git a/tests/test_repo.py b/tests/test_repo.py index 99939ad..91c303e 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) # Parts of the code comes from ANYBOX @@ -6,12 +5,14 @@ import argparse import os import shutil -import unittest import subprocess +import unittest + try: # Py 2 - from urlparse import urljoin from urllib import pathname2url + + from urlparse import urljoin except ImportError: # PY 3 from urllib.parse import urljoin @@ -20,11 +21,12 @@ from tempfile import mkdtemp from textwrap import dedent - -from git_aggregator.utils import WorkingDirectoryKeeper,\ - working_directory_keeper -from git_aggregator.repo import Repo from git_aggregator import exception, main +from git_aggregator.repo import Repo +from git_aggregator.utils import ( + WorkingDirectoryKeeper, + working_directory_keeper, +) def git_get_last_rev(repo_dir): @@ -65,7 +67,7 @@ class TestRepo(unittest.TestCase): @classmethod def setUpClass(cls): main.setup_logger(level=logging.DEBUG) - super(TestRepo, cls).setUpClass() + super().setUpClass() def setUp(self): """ Setup @@ -79,7 +81,7 @@ def setUp(self): commit 3 branch b2 """ - super(TestRepo, self).setUp() + super().setUp() sandbox = self.sandbox = mkdtemp('test_repo') with working_directory_keeper: os.chdir(sandbox) @@ -124,6 +126,28 @@ def test_minimal(self): last_rev = git_get_last_rev(self.cwd) self.assertEqual(last_rev, self.commit_1_sha) + def test_empty_dir(self): + # ensure git clone in empty directory works + remotes = [{ + 'name': 'r1', + 'url': self.url_remote1 + }] + merges = [{ + 'remote': 'r1', + 'ref': 'tag1' + }] + target = { + 'remote': 'r1', + 'branch': 'agg1' + } + # self.cwd should not exist yet + self.assertFalse(os.path.exists(self.cwd)) + os.mkdir(self.cwd) + repo = Repo(self.cwd, remotes, merges, target) + repo.aggregate() + last_rev = git_get_last_rev(self.cwd) + self.assertEqual(last_rev, self.commit_1_sha) + def test_annotated_tag(self): remotes = [{ 'name': 'r1', diff --git a/tox.ini b/tox.ini index f9d55ce..95a70de 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,13 @@ [gh-actions] python = - 3.7: py37 - 3.8: py38 3.9: py39 3.10: py310 3.11: py311 + 3.12: py312 + 3.13: py313 [tox] -envlist = py37, py38, py39, py310, py311 +envlist = py39, py310, py311, py312, py313 [testenv] passenv =