diff --git a/test/import_blobless.txt b/test/import_blobless.txt new file mode 100644 index 0000000..b461059 --- /dev/null +++ b/test/import_blobless.txt @@ -0,0 +1,53 @@ +...... +=== ./immutable/hash (git) === +Cloning into '.'... +warning: filtering not recognized by server, ignoring +Note: switching to '5b3504594f7354121cf024dc734bf79e270cffd3'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at 5b35045... update changelog +=== ./immutable/hash_tar (tar) === +Downloaded tarball from 'file:///vcstmp/archive.tar.gz' and unpacked it +=== ./immutable/hash_zip (zip) === +Downloaded zipfile from 'file:///vcstmp/archive.zip' and unpacked it +=== ./immutable/tag (git) === +Cloning into '.'... +warning: filtering not recognized by server, ignoring +Note: switching to 'tags/0.1.27'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at 8087b72... 0.1.27 +=== ./vcs2l (git) === +Cloning into '.'... +warning: filtering not recognized by server, ignoring +=== ./without_version (git) === +Cloning into '.'... +warning: filtering not recognized by server, ignoring diff --git a/test/test_commands.py b/test/test_commands.py index 151deef..249ce00 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -270,6 +270,45 @@ def test_import_shallow(self): finally: rmtree(workdir) + def test_import_blobless(self): + workdir = os.path.join(TEST_WORKSPACE, 'import-blobless') + os.makedirs(workdir) + try: + output = run_command( + 'import', + ['--blobless-clone', '--input', self.repos_file_path, '.'], + subfolder='import-blobless', + ) + # the actual output contains absolute paths + output = output.replace( + b'repository in ' + workdir.encode() + b'/', b'repository in ./' + ) + expected = get_expected_output('import_blobless') + # newer git versions don't append ... after the commit hash + assert output == expected or output == expected.replace(b'... ', b' ') + + git_repos = [ + os.path.join(workdir, 'immutable', 'hash'), + os.path.join(workdir, 'immutable', 'tag'), + os.path.join(workdir, 'vcs2l'), + os.path.join(workdir, 'without_version'), + ] + for repo_path in git_repos: + partial_filter = subprocess.check_output( + ['git', 'config', '--get', 'remote.origin.partialclonefilter'], + stderr=subprocess.STDOUT, + cwd=repo_path, + ).strip() + self.assertEqual(partial_filter, b'blob:none') + promisor = subprocess.check_output( + ['git', 'config', '--get', 'remote.origin.promisor'], + stderr=subprocess.STDOUT, + cwd=repo_path, + ).strip() + self.assertEqual(promisor, b'true') + finally: + rmtree(workdir) + def test_import_url(self): workdir = os.path.join(TEST_WORKSPACE, 'import-url') os.makedirs(workdir) diff --git a/vcs2l/clients/git.py b/vcs2l/clients/git.py index b8022a3..ffbea5a 100644 --- a/vcs2l/clients/git.py +++ b/vcs2l/clients/git.py @@ -326,6 +326,8 @@ def import_(self, command): # fetch updates for existing repo cmd_fetch = [GitClient._executable, 'fetch', remote] + if command.blobless_clone: + cmd_fetch.append('--filter=blob:none') if command.shallow: result_version_type, version_name = self._check_version_type( command.url, checkout_version, command.retry @@ -409,7 +411,17 @@ def import_(self, command): if not command.shallow or version_type in (None, 'branch'): cmd_clone = [GitClient._executable, 'clone', command.url, '.'] - if version_type == 'branch': + if command.blobless_clone: + cmd_clone += ['--filter=blob:none'] + if version_type == 'branch': + cmd_clone += ['-b', version_name] + checkout_version = None + elif command.version: + cmd_clone.append('--no-checkout') + checkout_version = command.version + else: + checkout_version = None + elif version_type == 'branch': cmd_clone += ['-b', version_name] checkout_version = None else: diff --git a/vcs2l/commands/import_.py b/vcs2l/commands/import_.py index 54af290..2c19e47 100644 --- a/vcs2l/commands/import_.py +++ b/vcs2l/commands/import_.py @@ -19,7 +19,15 @@ class ImportCommand(Command): command = 'import' help = 'Import the list of repositories' - def __init__(self, args, url, version=None, recursive=False, shallow=False): + def __init__( + self, + args, + url, + version=None, + recursive=False, + shallow=False, + blobless_clone=False, + ): super(ImportCommand, self).__init__(args) self.url = url self.version = version @@ -28,6 +36,7 @@ def __init__(self, args, url, version=None, recursive=False, shallow=False): self.skip_existing = args.skip_existing self.recursive = recursive self.shallow = shallow + self.blobless_clone = blobless_clone def get_parser(): @@ -49,12 +58,19 @@ def get_parser(): help="Delete existing directories if they don't contain the " 'repository being imported', ) - group.add_argument( + clone_type_group = group.add_mutually_exclusive_group() + clone_type_group.add_argument( '--shallow', action='store_true', default=False, help='Create a shallow clone without a history', ) + clone_type_group.add_argument( + '--blobless-clone', + action='store_true', + default=False, + help='Only clone the commit history first, then checkout to the target version to obtain files', + ) group.add_argument( '--recursive', action='store_true', @@ -207,6 +223,7 @@ def generate_jobs(repos, args): str(repo['version']) if 'version' in repo else None, recursive=args.recursive, shallow=args.shallow, + blobless_clone=args.blobless_clone, ) job = {'client': client, 'command': command} jobs.append(job) @@ -247,6 +264,7 @@ def main(args=None, stdout=None, stderr=None): except (RuntimeError, request.URLError) as e: print(ansi('redf') + str(e) + ansi('reset'), file=sys.stderr) return 1 + jobs = generate_jobs(repos, args) add_dependencies(jobs)