From 14cc1756fd22b195704b5528d58240473ecb8b83 Mon Sep 17 00:00:00 2001 From: hparfr Date: Tue, 9 Apr 2024 15:16:02 +0200 Subject: [PATCH 01/10] fix: git remote: change url instead of rm / add With some repos, there is an issue when the origin url is changed to a fork in repo.yml. Errors look like `unable to read sha1 file of`, `index-pack failed` It's occuring when the remote is removed (git remote rm origin) then added back with another url (git remote add origin ). With this fix, we change the url inplace. `git remote set-url origin ` --- git_aggregator/repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git_aggregator/repo.py b/git_aggregator/repo.py index 677334c..625e5ba 100644 --- a/git_aggregator/repo.py +++ b/git_aggregator/repo.py @@ -350,8 +350,8 @@ def _set_remote(self, name, url): else: logger.info('Updating remote %s <%s> -> <%s>', name, exising_url, url) - self.log_call(['git', 'remote', 'rm', name], cwd=self.cwd) - self.log_call(['git', 'remote', 'add', name, url], cwd=self.cwd) + self.log_call( + ['git', 'remote', 'set-url', name, url], cwd=self.cwd) def _github_api_get(self, path): url = 'https://api.github.com' + path From d8ccdaf2779558518f764f720016d9a26d53e4ad Mon Sep 17 00:00:00 2001 From: hparfr Date: Mon, 7 Oct 2024 10:52:55 +0200 Subject: [PATCH 02/10] target_dir may be an empty directory git clone do not care if the dir exist or is empty And an existing target_dir/.git is expected in the rest of this function --- git_aggregator/repo.py | 2 +- tests/test_repo.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/git_aggregator/repo.py b/git_aggregator/repo.py index 625e5ba..27d623d 100644 --- a/git_aggregator/repo.py +++ b/git_aggregator/repo.py @@ -173,7 +173,7 @@ def aggregate(self): logger.info('Start aggregation of %s', self.cwd) target_dir = self.cwd - is_new = not os.path.exists(target_dir) + is_new = not os.path.exists(target_dir) or os.listdir(target_dir) == [] if is_new: cloned = self.init_repository(target_dir) diff --git a/tests/test_repo.py b/tests/test_repo.py index 99939ad..c974449 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -124,6 +124,28 @@ def test_minimal(self): last_rev = git_get_last_rev(self.cwd) self.assertEqual(last_rev, self.commit_1_sha) + def test_empty_dir(self): + # ensure git clone in empty directory works + remotes = [{ + 'name': 'r1', + 'url': self.url_remote1 + }] + merges = [{ + 'remote': 'r1', + 'ref': 'tag1' + }] + target = { + 'remote': 'r1', + 'branch': 'agg1' + } + # self.cwd should not exist yet + self.assertFalse(os.path.exists(self.cwd)) + os.mkdir(self.cwd) + repo = Repo(self.cwd, remotes, merges, target) + repo.aggregate() + last_rev = git_get_last_rev(self.cwd) + self.assertEqual(last_rev, self.commit_1_sha) + def test_annotated_tag(self): remotes = [{ 'name': 'r1', From c9aeaa2805a801a8f65351a757235963557ea4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 23 Oct 2024 14:44:37 +0200 Subject: [PATCH 03/10] ci: use ruff instead of flake8 --- .pre-commit-config.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ff61df7..15266b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,13 +2,14 @@ default_language_version: python: python3 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v5.0.0 hooks: - id: check-toml - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/PyCQA/flake8 - rev: "3.9.2" + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.0 hooks: - - id: flake8 + - id: ruff + args: [--fix] From a0dcb3fe62358576aa9b4f03e604889134962cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 23 Oct 2024 14:47:38 +0200 Subject: [PATCH 04/10] lint: let ruff modernize the code a bit --- git_aggregator/__init__.py | 1 - git_aggregator/_compat.py | 1 - git_aggregator/config.py | 8 +++----- git_aggregator/exception.py | 1 - git_aggregator/log.py | 1 - git_aggregator/main.py | 22 +++++++++++----------- git_aggregator/repo.py | 8 +++----- git_aggregator/utils.py | 8 ++++---- pyproject.toml | 15 +++++++++++++++ setup.py | 1 - tests/__init__.py | 1 - tests/test_config.py | 3 +-- tests/test_log.py | 3 +-- tests/test_repo.py | 20 +++++++++++--------- 14 files changed, 49 insertions(+), 44 deletions(-) create mode 100644 pyproject.toml diff --git a/git_aggregator/__init__.py b/git_aggregator/__init__.py index 77ab94a..b868204 100644 --- a/git_aggregator/__init__.py +++ b/git_aggregator/__init__.py @@ -1,3 +1,2 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) diff --git a/git_aggregator/_compat.py b/git_aggregator/_compat.py index 9a4f757..678ac5e 100644 --- a/git_aggregator/_compat.py +++ b/git_aggregator/_compat.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) import sys diff --git a/git_aggregator/config.py b/git_aggregator/config.py index 0043e84..1f95b66 100644 --- a/git_aggregator/config.py +++ b/git_aggregator/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) @@ -8,9 +7,8 @@ import yaml -from .exception import ConfigException from ._compat import string_types - +from .exception import ConfigException log = logging.getLogger(__name__) @@ -167,11 +165,11 @@ def load_config(config, expand_env=False, env_file=None, force=False): key, value = line.split('=') environment.update({key.strip(): value.strip()}) environment.update(os.environ) - with open(config, 'r') as file_handler: + with open(config) as file_handler: config = Template(file_handler.read()) config = config.substitute(environment) else: - config = open(config, 'r').read() + config = open(config).read() conf = yaml.load(config, Loader=yaml.SafeLoader) diff --git a/git_aggregator/exception.py b/git_aggregator/exception.py index 86d6a94..0f25940 100644 --- a/git_aggregator/exception.py +++ b/git_aggregator/exception.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) diff --git a/git_aggregator/log.py b/git_aggregator/log.py index c1cfe0e..5219ce0 100644 --- a/git_aggregator/log.py +++ b/git_aggregator/log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) diff --git a/git_aggregator/main.py b/git_aggregator/main.py index 146ddcf..1c922a6 100644 --- a/git_aggregator/main.py +++ b/git_aggregator/main.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015-2019 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) @@ -7,22 +6,24 @@ import sys import threading import traceback + try: - from Queue import Queue, Empty as EmptyQueue + from Queue import Empty as EmptyQueue + from Queue import Queue except ImportError: - from queue import Queue, Empty as EmptyQueue + from queue import Empty as EmptyQueue + from queue import Queue import argparse +import fnmatch + import argcomplete import colorama -import fnmatch -from .utils import ThreadNameKeeper -from .log import DebugLogFormatter -from .log import LogFormatter from .config import load_config +from .log import DebugLogFormatter, LogFormatter from .repo import Repo - +from .utils import ThreadNameKeeper logger = logging.getLogger(__name__) @@ -31,8 +32,7 @@ def _log_level_string_to_int(log_level_string): if log_level_string not in _LOG_LEVEL_STRINGS: - message = 'invalid choice: {0} (choose from {1})'.format( - log_level_string, _LOG_LEVEL_STRINGS) + message = f'invalid choice: {log_level_string} (choose from {_LOG_LEVEL_STRINGS})' raise argparse.ArgumentTypeError(message) log_level_int = getattr(logging, log_level_string, logging.INFO) @@ -99,7 +99,7 @@ def get_parser(): dest='log_level', type=_log_level_string_to_int, nargs='?', - help='Set the logging output level. {0}'.format(_LOG_LEVEL_STRINGS)) + help=f'Set the logging output level. {_LOG_LEVEL_STRINGS}') main_parser.add_argument( '-e', '--expand-env', diff --git a/git_aggregator/repo.py b/git_aggregator/repo.py index 27d623d..0656b89 100644 --- a/git_aggregator/repo.py +++ b/git_aggregator/repo.py @@ -1,18 +1,16 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) # Parts of the code comes from ANYBOX # https://github.com/anybox/anybox.recipe.odoo -from __future__ import unicode_literals -import os import logging +import os import re import subprocess import requests -from .exception import DirtyException, GitAggregatorException from ._compat import console_to_str +from .exception import DirtyException, GitAggregatorException FETCH_DEFAULTS = ("depth", "shallow-since", "shallow-exclude") logger = logging.getLogger(__name__) @@ -32,7 +30,7 @@ def ishex(s): return True -class Repo(object): +class Repo: _git_version = None diff --git a/git_aggregator/utils.py b/git_aggregator/utils.py index 063e787..1ecbb89 100644 --- a/git_aggregator/utils.py +++ b/git_aggregator/utils.py @@ -1,14 +1,14 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # © ANYBOX https://github.com/anybox/anybox.recipe.odoo # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) +import logging import os import threading -import logging + logger = logging.getLogger(__name__) -class WorkingDirectoryKeeper(object): # DEPRECATED +class WorkingDirectoryKeeper: # DEPRECATED """A context manager to get back the working directory as it was before. If you want to stack working directory keepers, you need a new instance for each stage. @@ -30,7 +30,7 @@ def __exit__(self, *exc_args): working_directory_keeper = WorkingDirectoryKeeper() -class ThreadNameKeeper(object): +class ThreadNameKeeper: """A contect manager to get back the thread name as it was before. It is meant to be used when modifying the 'MainThread' tread. """ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5c98ac7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.ruff] +fix = true +line-length = 79 + +[tool.ruff.lint] +extend-select = [ + "I", + "UP", +] +ignore = [ + "UP031", # % formatting +] + +[tool.ruff.lint.isort] +known-first-party = ["git_aggregator"] diff --git a/setup.py b/setup.py index d02c992..2013ddd 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) diff --git a/tests/__init__.py b/tests/__init__.py index 77ab94a..b868204 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,2 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) diff --git a/tests/test_config.py b/tests/test_config.py index 0c9fa50..ab2eba2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) import os @@ -9,8 +8,8 @@ import yaml from git_aggregator import config -from git_aggregator.exception import ConfigException from git_aggregator._compat import PY2 +from git_aggregator.exception import ConfigException class TestConfig(unittest.TestCase): diff --git a/tests/test_log.py b/tests/test_log.py index b588e2a..ef4a849 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) import logging @@ -20,7 +19,7 @@ class TestLog(unittest.TestCase): def setUp(self): """ Setup """ - super(TestLog, self).setUp() + super().setUp() reset_logger() def test_info(self): diff --git a/tests/test_repo.py b/tests/test_repo.py index c974449..91c303e 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # © 2015 ACSONE SA/NV # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) # Parts of the code comes from ANYBOX @@ -6,12 +5,14 @@ import argparse import os import shutil -import unittest import subprocess +import unittest + try: # Py 2 - from urlparse import urljoin from urllib import pathname2url + + from urlparse import urljoin except ImportError: # PY 3 from urllib.parse import urljoin @@ -20,11 +21,12 @@ from tempfile import mkdtemp from textwrap import dedent - -from git_aggregator.utils import WorkingDirectoryKeeper,\ - working_directory_keeper -from git_aggregator.repo import Repo from git_aggregator import exception, main +from git_aggregator.repo import Repo +from git_aggregator.utils import ( + WorkingDirectoryKeeper, + working_directory_keeper, +) def git_get_last_rev(repo_dir): @@ -65,7 +67,7 @@ class TestRepo(unittest.TestCase): @classmethod def setUpClass(cls): main.setup_logger(level=logging.DEBUG) - super(TestRepo, cls).setUpClass() + super().setUpClass() def setUp(self): """ Setup @@ -79,7 +81,7 @@ def setUp(self): commit 3 branch b2 """ - super(TestRepo, self).setUp() + super().setUp() sandbox = self.sandbox = mkdtemp('test_repo') with working_directory_keeper: os.chdir(sandbox) From b2f19f8ff663e6c12614ee063c091287ab0116af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 23 Oct 2024 14:55:10 +0200 Subject: [PATCH 05/10] Prepare 4.0.2 --- README.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.rst b/README.rst index 4a74110..a5d9173 100644 --- a/README.rst +++ b/README.rst @@ -224,6 +224,16 @@ To work around API limitation, you must first generate a Changes ======= +4.0.2 (2024-10-23) +------------------ + +* target_dir may be an empty directory (`#83 `_) + +4.0.1 (2024-06-04) +------------------ + +* fix: git remote: change url instead of rm / add (`#81 `_`) + 4.0 (2023-07-22) ---------------- From 0d6cc2b914d088813d87d1bd1c6053aa17e8eb7d Mon Sep 17 00:00:00 2001 From: Ivan Sokolov Date: Tue, 7 Jan 2025 22:23:18 +0100 Subject: [PATCH 06/10] [DOC] Update README Add usage examples for Gitlab and Bitbucket repositories. --- README.rst | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/README.rst b/README.rst index a5d9173..525186b 100644 --- a/README.rst +++ b/README.rst @@ -43,6 +43,27 @@ Create a ``repos.yaml`` or ``repos.yml`` file: fetch_all: - oca + ./gitlab-repo-example: + remotes: + gitlab_1: https://giltab.com/sample-org/sample-repo.git + gitlab_2: https://giltab.com/another-sample-org/sample-repo-fork.git + merges: + - gitlab_1 main + - gitlab_2 merge-requests/123/head + target: gitlab_1 + + # FYI: Bitbucket doesn't support fetching PR's + ./bitbucket-repo-example: + remotes: + bitbucket_1: https://bitbucket.org/sample-org/sample-repo.git + bitbucket_2: https://bitbucker.org/another-sample-org/sample-repo-fork.git + merges: + - bitbucket_1 main + - bitbucket_2 dev + target: bitbucket_1 + + + Environment variables inside of this file will be expanded if the proper option is selected. All the ``merges`` are combined into a single branch. By default this branch is called ``_git_aggregated`` but another name may be given in the ``target`` section. @@ -107,6 +128,38 @@ the ``dict`` alternate construction. If you need to disable a default in ref: refs/pull/14859/head target: acsone 9.0 + ./gitlab-repo-example: + defaults: + depth: 20 + remotes: + gitlab_1: https://giltab.com/sample-org/sample-repo.git + gitlab_2: https://giltab.com/another-sample-org/sample-repo-fork.git + merges: + - + remote: gitlab_1 + ref: main + depth: 20 + - + remote: gitlab_2 + ref: merge-requests/123/head + target: gitlab_1 + + # FYI: Bitbucket doesn't support fetching PR's + ./bitbucket-repo-example: + remotes: + bitbucket_1: https://bitbucket.org/sample-org/sample-repo.git + bitbucket_2: https://bitbucker.org/another-sample-org/sample-repo-fork.git + merges: + - + remote: bitbucket_1 + ref: main + depth: 3 + - + remote: bitbucket_2 + ref: dev + target: bitbucket_1 + + Remember that you need to fetch at least the common ancestor of all merges for it to succeed. From 32e93a342536a85be6f0b075d23bba4d7c7b0307 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 21 Mar 2025 11:36:38 +0100 Subject: [PATCH 07/10] Update CI py versions --- .github/workflows/ci.yml | 2 +- setup.py | 4 ++-- tox.ini | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e59ae0f..6ea760e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: "actions/checkout@v3" - uses: "actions/setup-python@v4" diff --git a/setup.py b/setup.py index 2013ddd..0256cf9 100644 --- a/setup.py +++ b/setup.py @@ -16,11 +16,11 @@ 'GNU Affero General Public License v3 or later (AGPLv3+)', 'Operating System :: POSIX', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', "Topic :: Utilities", "Topic :: System :: Shells", ], diff --git a/tox.ini b/tox.ini index f9d55ce..95a70de 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,13 @@ [gh-actions] python = - 3.7: py37 - 3.8: py38 3.9: py39 3.10: py310 3.11: py311 + 3.12: py312 + 3.13: py313 [tox] -envlist = py37, py38, py39, py310, py311 +envlist = py39, py310, py311, py312, py313 [testenv] passenv = From c6a51fc5fe7a071425feef4d3b0d04c52c292055 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 21 Mar 2025 10:55:40 +0100 Subject: [PATCH 08/10] [imp] repo.collect_prs_info allow to pass custom merges Very handy to be able to feed it with custom PRs info. Example: provide a parsed version of shell_after_commands containing PR patches. --- git_aggregator/repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git_aggregator/repo.py b/git_aggregator/repo.py index 0656b89..3801434 100644 --- a/git_aggregator/repo.py +++ b/git_aggregator/repo.py @@ -359,7 +359,7 @@ def _github_api_get(self, path): headers = {'Authorization': 'token %s' % token} return requests.get(url, headers=headers) - def collect_prs_info(self): + def collect_prs_info(self, merges=None): """Collect all pending merge PRs info. :returns: mapping of PRs by state @@ -371,7 +371,7 @@ def collect_prs_info(self): '^(refs/)?pull/(?P[0-9]+)/head$') remotes = {r['name']: r['url'] for r in self.remotes} all_prs = {} - for merge in self.merges: + for merge in (merges or self.merges): remote = merge['remote'] ref = merge['ref'] repo_url = remotes[remote] From 223afe312880d1e1f7dd16daf4d659309710818c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 25 Mar 2025 18:12:15 +0100 Subject: [PATCH 09/10] Prepare 4.1 --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 525186b..d4fd885 100644 --- a/README.rst +++ b/README.rst @@ -277,6 +277,13 @@ To work around API limitation, you must first generate a Changes ======= +4.1 (2025-03-25) +---------------- + +* Update README for other forges than GitHub (`#88 `_) +* Internal API change (`#91 `_) +* Drop support for unsupported Python versions (`#92 `_) + 4.0.2 (2024-10-23) ------------------ From c850cb7b26a41721784a0564c79b5310335117a2 Mon Sep 17 00:00:00 2001 From: Hugo Santos Date: Thu, 19 Feb 2026 11:26:20 +0100 Subject: [PATCH 10/10] [IMP] Retry network connection errors on gitaggregate --- .gitignore | 2 ++ git_aggregator/config.py | 12 ++++++-- git_aggregator/repo.py | 61 ++++++++++++++++++++++++++++++++++------ 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index daef807..218d2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,5 @@ docs/_build/ # PyBuilder target/ + +.venv/ diff --git a/git_aggregator/config.py b/git_aggregator/config.py index f2b9794..266d284 100644 --- a/git_aggregator/config.py +++ b/git_aggregator/config.py @@ -35,6 +35,8 @@ def get_repos(config, force=False, skip_merge_check=False): 'apply_patch': repo_data.get('apply_patch', False), 'skip_repo_init': repo_data.get('skip_repo_init', False), } + if 'retry' in repo_data: + repo_dict['retry'] = repo_data['retry'] remote_names = set() if 'remotes' in repo_data: repo_dict['remotes'] = [] @@ -55,7 +57,10 @@ def get_repos(config, force=False, skip_merge_check=False): '%s: You should at least define one remote.' % directory) else: try: - tmp_repo = Repo(repo_dict['cwd'], [], [], None) + tmp_repo = Repo( + repo_dict['cwd'], [], [], None, + retry={"max_retries": 0}, + ) remotes = tmp_repo._get_remotes() repo_dict['remotes'] = [] for remote_name, url in remotes.items(): @@ -71,7 +76,10 @@ def get_repos(config, force=False, skip_merge_check=False): merge_data = repo_data.get('merges') or [] tmp_repo = None if not skip_merge_check: - tmp_repo = Repo(repo_dict['cwd'], [], [], None) + tmp_repo = Repo( + repo_dict['cwd'], [], [], None, + retry={"max_retries": 0}, + ) if os.path.exists(tmp_repo.cwd): # Set remotes for remote in repo_dict['remotes']: diff --git a/git_aggregator/repo.py b/git_aggregator/repo.py index 6ec1744..aeccdd5 100644 --- a/git_aggregator/repo.py +++ b/git_aggregator/repo.py @@ -7,6 +7,7 @@ import re import shutil import subprocess +import time import requests @@ -14,9 +15,19 @@ from .exception import DirtyException, GitAggregatorException FETCH_DEFAULTS = ("depth", "shallow-since", "shallow-exclude") +NETWORK_GIT_COMMANDS = ("fetch", "pull", "push", "ls-remote", "clone") +DEFAULT_RETRY = {"max_retries": 3, "delay": 5, "backoff_factor": 2} logger = logging.getLogger(__name__) +def _is_network_command(cmd): + """Detect if a git command is a network operation.""" + for part in cmd: + if part in NETWORK_GIT_COMMANDS: + return True + return False + + def ishex(s): """True iff given string is a valid hexadecimal number. >>> ishex('deadbeef') @@ -38,7 +49,7 @@ class Repo: def __init__(self, cwd, remotes, merges, target, shell_command_after=None, fetch_all=False, defaults=None, force=False, skip_dry_run=False, apply_patch=False, - skip_repo_init=False): + skip_repo_init=False, retry=None): """Initialize a git repository aggregator :param cwd: path to the directory where to initialize the repository @@ -79,6 +90,10 @@ def __init__(self, cwd, remotes, merges, target, self.skip_dry_run = skip_dry_run self.apply_patch = apply_patch self.skip_repo_init = skip_repo_init + if retry is None: + self.retry = dict(DEFAULT_RETRY) + else: + self.retry = retry @property def git_version(self): @@ -168,14 +183,42 @@ def log_call(self, cmd, callwith=subprocess.check_call, :param meth: the calling method to use. """ logger.log(log_level, "%s> call %r", self.cwd, cmd) - try: - ret = callwith(cmd, **kw) - except Exception: - logger.error("%s> error calling %r", self.cwd, cmd) - raise - if callwith == subprocess.check_output: - ret = console_to_str(ret) - return ret + max_retries = self.retry.get("max_retries", 0) + if max_retries > 0 and _is_network_command(cmd): + delay = self.retry.get("delay", DEFAULT_RETRY["delay"]) + backoff_factor = self.retry.get( + "backoff_factor", DEFAULT_RETRY["backoff_factor"] + ) + last_exception = None + for attempt in range(max_retries + 1): + try: + ret = callwith(cmd, **kw) + if callwith == subprocess.check_output: + ret = console_to_str(ret) + return ret + except Exception as exc: + last_exception = exc + if attempt < max_retries: + wait = delay * backoff_factor ** attempt + logger.warning( + "%s> Retry %d/%d for %r in %ds...", + self.cwd, attempt + 1, max_retries, cmd, wait, + ) + time.sleep(wait) + else: + logger.error( + "%s> error calling %r", self.cwd, cmd + ) + raise last_exception + else: + try: + ret = callwith(cmd, **kw) + except Exception: + logger.error("%s> error calling %r", self.cwd, cmd) + raise + if callwith == subprocess.check_output: + ret = console_to_str(ret) + return ret def _dry_run_cleanup(self, target_dir): if os.path.exists(target_dir):