From f175b349a579e19cde1afe87d3be0d1f8358853c Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 8 Aug 2025 11:59:42 +0200 Subject: [PATCH 01/25] t0450: fix test for out-of-tree builds When using Meson, builds are out-of-tree and $GIT_BUILD_DIR gets set to the path where the build output is landing. To locate the Documentation sources, test 't0450' was using that path. Modify test 't0450' to use `$GIT_SOURCE_DIR/Documentation` to find the documentation sources. Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- t/t0450-txt-doc-vs-help.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t0450-txt-doc-vs-help.sh b/t/t0450-txt-doc-vs-help.sh index 2f7504ae7e9090..da2d0af5b07810 100755 --- a/t/t0450-txt-doc-vs-help.sh +++ b/t/t0450-txt-doc-vs-help.sh @@ -41,7 +41,7 @@ help_to_synopsis () { } builtin_to_adoc () { - echo "$GIT_BUILD_DIR/Documentation/git-$1.adoc" + echo "$GIT_SOURCE_DIR/Documentation/git-$1.adoc" } adoc_to_synopsis () { From 595bef7180b57889a4dec4b675a7fc6084c863ac Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:41 +0200 Subject: [PATCH 02/25] odb: store locality in object database sources Object database sources are classified either as: - Local, which means that the source is the repository's primary source. This is typically ".git/objects". - Non-local, which is everything else. Most importantly this includes alternates and quarantine directories. This locality is often computed ad-hoc by checking whether a given object source is the first one. This works, but it is quite roundabout. Refactor the code so that we store locality when creating the sources in the first place. This makes it both more accessible and robust. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- midx.c | 5 +++-- midx.h | 2 +- odb.c | 1 + odb.h | 8 ++++++++ packfile.c | 9 ++++----- repository.c | 1 + 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/midx.c b/midx.c index 7d407682e60a6f..b9ca0915a67f10 100644 --- a/midx.c +++ b/midx.c @@ -723,7 +723,7 @@ int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) return 0; } -int prepare_multi_pack_index_one(struct odb_source *source, int local) +int prepare_multi_pack_index_one(struct odb_source *source) { struct repository *r = source->odb->repo; @@ -734,7 +734,8 @@ int prepare_multi_pack_index_one(struct odb_source *source, int local) if (source->midx) return 1; - source->midx = load_multi_pack_index(r, source->path, local); + source->midx = load_multi_pack_index(r, source->path, + source->local); return !!source->midx; } diff --git a/midx.h b/midx.h index 076382de8acd26..28c426a8232997 100644 --- a/midx.h +++ b/midx.h @@ -122,7 +122,7 @@ int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pa int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id); -int prepare_multi_pack_index_one(struct odb_source *source, int local); +int prepare_multi_pack_index_one(struct odb_source *source); /* * Variant of write_midx_file which writes a MIDX containing only the packs diff --git a/odb.c b/odb.c index 1f48a0448e398a..1761a50840ddf8 100644 --- a/odb.c +++ b/odb.c @@ -176,6 +176,7 @@ static int link_alt_odb_entry(struct object_database *odb, CALLOC_ARRAY(alternate, 1); alternate->odb = odb; + alternate->local = false; /* pathbuf.buf is already in r->objects->source_by_path */ alternate->path = strbuf_detach(&pathbuf, NULL); diff --git a/odb.h b/odb.h index 09177bf430dc38..f9300439bab3af 100644 --- a/odb.h +++ b/odb.h @@ -63,6 +63,14 @@ struct odb_source { */ struct multi_pack_index *midx; + /* + * Figure out whether this is the local source of the owning + * repository, which would typically be its ".git/objects" directory. + * This local object directory is usually where objects would be + * written to. + */ + bool local; + /* * This is a temporary object store created by the tmp_objdir * facility. Disable ref updates since the objects in the store diff --git a/packfile.c b/packfile.c index 5d73932f50ce68..a38544b87bf2f1 100644 --- a/packfile.c +++ b/packfile.c @@ -935,14 +935,14 @@ static void prepare_pack(const char *full_name, size_t full_name_len, report_garbage(PACKDIR_FILE_GARBAGE, full_name); } -static void prepare_packed_git_one(struct odb_source *source, int local) +static void prepare_packed_git_one(struct odb_source *source) { struct string_list garbage = STRING_LIST_INIT_DUP; struct prepare_pack_data data = { .m = source->midx, .r = source->odb->repo, .garbage = &garbage, - .local = local, + .local = source->local, }; for_each_file_in_pack_dir(source->path, prepare_pack, &data); @@ -1037,9 +1037,8 @@ static void prepare_packed_git(struct repository *r) odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { - int local = (source == r->objects->sources); - prepare_multi_pack_index_one(source, local); - prepare_packed_git_one(source, local); + prepare_multi_pack_index_one(source); + prepare_packed_git_one(source); } rearrange_packed_git(r); diff --git a/repository.c b/repository.c index ecd691181fc97d..97f0578381d894 100644 --- a/repository.c +++ b/repository.c @@ -168,6 +168,7 @@ void repo_set_gitdir(struct repository *repo, if (!repo->objects->sources) { CALLOC_ARRAY(repo->objects->sources, 1); repo->objects->sources->odb = repo->objects; + repo->objects->sources->local = true; repo->objects->sources_tail = &repo->objects->sources->next; } expand_base_dir(&repo->objects->sources->path, o->object_dir, From 0d61933b8f9a0392310196578e1374283496843c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:42 +0200 Subject: [PATCH 03/25] odb: allow `odb_find_source()` to fail When trying to locate a source for an unknown object directory we will die right away. In subsequent patches we will add new callsites though that want to handle this situation gracefully instead. Refactor the function to return a `NULL` pointer if the source could not be found and adapt the callsites to die instead. Introduce a new wrapper `odb_find_source_or_die()` that continues to die in case the source could not be found. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/commit-graph.c | 4 ++-- midx-write.c | 2 +- odb.c | 6 ++++++ odb.h | 7 +++++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 25018a0b9df464..33fb7a5145c694 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -101,7 +101,7 @@ static int graph_verify(int argc, const char **argv, const char *prefix, if (opts.progress) flags |= COMMIT_GRAPH_WRITE_PROGRESS; - source = odb_find_source(the_repository->objects, opts.obj_dir); + source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); graph_name = get_commit_graph_filename(source); chain_name = get_commit_graph_chain_filename(source); if (open_commit_graph(graph_name, &fd, &st)) @@ -289,7 +289,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags |= COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - source = odb_find_source(the_repository->objects, opts.obj_dir); + source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); if (opts.reachable) { if (write_commit_graph_reachable(source, flags, &write_opts)) diff --git a/midx-write.c b/midx-write.c index c1ae62d3549425..d38caceadb4fa4 100644 --- a/midx-write.c +++ b/midx-write.c @@ -916,7 +916,7 @@ static int write_midx_bitmap(struct write_midx_context *ctx, static struct multi_pack_index *lookup_multi_pack_index(struct repository *r, const char *object_dir) { - struct odb_source *source = odb_find_source(r->objects, object_dir); + struct odb_source *source = odb_find_source_or_die(r->objects, object_dir); return get_multi_pack_index(source); } diff --git a/odb.c b/odb.c index 1761a50840ddf8..4e7f14be4a004c 100644 --- a/odb.c +++ b/odb.c @@ -464,6 +464,12 @@ struct odb_source *odb_find_source(struct object_database *odb, const char *obj_ free(obj_dir_real); strbuf_release(&odb_path_real); + return source; +} + +struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir) +{ + struct odb_source *source = odb_find_source(odb, obj_dir); if (!source) die(_("could not find object directory matching %s"), obj_dir); return source; diff --git a/odb.h b/odb.h index f9300439bab3af..312921077b857a 100644 --- a/odb.h +++ b/odb.h @@ -186,11 +186,14 @@ struct object_database *odb_new(struct repository *repo); void odb_clear(struct object_database *o); /* - * Find source by its object directory path. Dies in case the source couldn't - * be found. + * Find source by its object directory path. Returns a `NULL` pointer in case + * the source could not be found. */ struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir); +/* Same as `odb_find_source()`, but dies in case the source doesn't exist. */ +struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir); + /* * Replace the current writable object directory with the specified temporary * object directory; returns the former primary source. From 25c532f6e0797ef501ce43835fb4af4bd9c33de5 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:43 +0200 Subject: [PATCH 04/25] odb: consistently use "dir" to refer to alternate's directory The functions that add an alternate object directory to the object database are somewhat inconsistent in how they call the paramater that refers to the directory path: in our headers we refer to it as "dir", whereas in the implementation we often call it "reference" or "entry". Unify this and consistently call the parameter "dir". While at it, refactor `link_alt_odb_entry()` to accept a C string instead of a `struct strbuf` as parameter to clarify that we really only need the path and nothing else. Suggested-by: Taylor Blau Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/odb.c b/odb.c index 4e7f14be4a004c..e41e3952ea0a28 100644 --- a/odb.c +++ b/odb.c @@ -140,7 +140,7 @@ static void read_info_alternates(struct object_database *odb, int depth); static int link_alt_odb_entry(struct object_database *odb, - const struct strbuf *entry, + const char *dir, const char *relative_base, int depth, const char *normalized_objdir) @@ -151,11 +151,11 @@ static int link_alt_odb_entry(struct object_database *odb, khiter_t pos; int ret = -1; - if (!is_absolute_path(entry->buf) && relative_base) { + if (!is_absolute_path(dir) && relative_base) { strbuf_realpath(&pathbuf, relative_base, 1); strbuf_addch(&pathbuf, '/'); } - strbuf_addbuf(&pathbuf, entry); + strbuf_addstr(&pathbuf, dir); if (!strbuf_realpath(&tmp, pathbuf.buf, 0)) { error(_("unable to normalize alternate object path: %s"), @@ -229,7 +229,7 @@ static void link_alt_odb_entries(struct object_database *odb, const char *alt, int sep, const char *relative_base, int depth) { struct strbuf objdirbuf = STRBUF_INIT; - struct strbuf entry = STRBUF_INIT; + struct strbuf dir = STRBUF_INIT; if (!alt || !*alt) return; @@ -243,13 +243,13 @@ static void link_alt_odb_entries(struct object_database *odb, const char *alt, strbuf_realpath(&objdirbuf, odb->sources->path, 1); while (*alt) { - alt = parse_alt_odb_entry(alt, sep, &entry); - if (!entry.len) + alt = parse_alt_odb_entry(alt, sep, &dir); + if (!dir.len) continue; - link_alt_odb_entry(odb, &entry, + link_alt_odb_entry(odb, dir.buf, relative_base, depth, objdirbuf.buf); } - strbuf_release(&entry); + strbuf_release(&dir); strbuf_release(&objdirbuf); } @@ -273,7 +273,7 @@ static void read_info_alternates(struct object_database *odb, } void odb_add_to_alternates_file(struct object_database *odb, - const char *reference) + const char *dir) { struct lock_file lock = LOCK_INIT; char *alts = repo_git_path(odb->repo, "objects/info/alternates"); @@ -290,7 +290,7 @@ void odb_add_to_alternates_file(struct object_database *odb, struct strbuf line = STRBUF_INIT; while (strbuf_getline(&line, in) != EOF) { - if (!strcmp(reference, line.buf)) { + if (!strcmp(dir, line.buf)) { found = 1; break; } @@ -306,18 +306,17 @@ void odb_add_to_alternates_file(struct object_database *odb, if (found) { rollback_lock_file(&lock); } else { - fprintf_or_die(out, "%s\n", reference); + fprintf_or_die(out, "%s\n", dir); if (commit_lock_file(&lock)) die_errno(_("unable to move new alternates file into place")); if (odb->loaded_alternates) - link_alt_odb_entries(odb, reference, - '\n', NULL, 0); + link_alt_odb_entries(odb, dir, '\n', NULL, 0); } free(alts); } void odb_add_to_alternates_memory(struct object_database *odb, - const char *reference) + const char *dir) { /* * Make sure alternates are initialized, or else our entry may be @@ -325,8 +324,7 @@ void odb_add_to_alternates_memory(struct object_database *odb, */ odb_prepare_alternates(odb); - link_alt_odb_entries(odb, reference, - '\n', NULL, 0); + link_alt_odb_entries(odb, dir, '\n', NULL, 0); } struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, From a59d44ff3f0f308f9577b05c858c063d2466b061 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:44 +0200 Subject: [PATCH 05/25] odb: return newly created in-memory sources Callers have no trivial way to obtain the newly created object database source when adding it to the in-memory list of alternates. While not yet needed anywhere, a subsequent commit will want to obtain that pointer. Refactor the function to return the source to make it easily accessible. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 30 ++++++++++++++++++------------ odb.h | 4 ++-- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/odb.c b/odb.c index e41e3952ea0a28..0c808bb288f168 100644 --- a/odb.c +++ b/odb.c @@ -139,17 +139,16 @@ static void read_info_alternates(struct object_database *odb, const char *relative_base, int depth); -static int link_alt_odb_entry(struct object_database *odb, - const char *dir, - const char *relative_base, - int depth, - const char *normalized_objdir) +static struct odb_source *link_alt_odb_entry(struct object_database *odb, + const char *dir, + const char *relative_base, + int depth, + const char *normalized_objdir) { - struct odb_source *alternate; + struct odb_source *alternate = NULL; struct strbuf pathbuf = STRBUF_INIT; struct strbuf tmp = STRBUF_INIT; khiter_t pos; - int ret = -1; if (!is_absolute_path(dir) && relative_base) { strbuf_realpath(&pathbuf, relative_base, 1); @@ -189,11 +188,11 @@ static int link_alt_odb_entry(struct object_database *odb, /* recursively add alternates */ read_info_alternates(odb, alternate->path, depth + 1); - ret = 0; + error: strbuf_release(&tmp); strbuf_release(&pathbuf); - return ret; + return alternate; } static const char *parse_alt_odb_entry(const char *string, @@ -315,16 +314,23 @@ void odb_add_to_alternates_file(struct object_database *odb, free(alts); } -void odb_add_to_alternates_memory(struct object_database *odb, - const char *dir) +struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, + const char *dir) { + struct odb_source *alternate; + char *objdir; + /* * Make sure alternates are initialized, or else our entry may be * overwritten when they are. */ odb_prepare_alternates(odb); - link_alt_odb_entries(odb, dir, '\n', NULL, 0); + objdir = real_pathdup(odb->sources->path, 1); + alternate = link_alt_odb_entry(odb, dir, NULL, 0, objdir); + + free(objdir); + return alternate; } struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, diff --git a/odb.h b/odb.h index 312921077b857a..d7691326997947 100644 --- a/odb.h +++ b/odb.h @@ -268,8 +268,8 @@ void odb_add_to_alternates_file(struct object_database *odb, * recursive alternates it points to), but do not modify the on-disk alternates * file. */ -void odb_add_to_alternates_memory(struct object_database *odb, - const char *dir); +struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, + const char *dir); /* * Read an object from the database. Returns the object data and assigns object From 57363dfa0dce05aac735d5cfd626e6aac8cb706c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:45 +0200 Subject: [PATCH 06/25] odb: simplify calling `link_alt_odb_entry()` Callers of `link_alt_odb_entry()` are expected to pass in three different paths: - The (potentially relative) path of the object directory that we're about to add. - The base that should be used to resolve a relative object directory path. - The resolved path to the object database's objects directory. Juggling those three paths makes the calling convention somewhat hard to grok at first. As it turns out, the third parameter is redundant: we always pass in the resolved path of the object database's primary source, and we already pass in the database itself. So instead, we can resolve that path in the function itself. One downside of this is that one caller of `link_alt_odb_entry()` calls this function in a loop, so we were able to resolve the directory a single time, only. But ultimately, we only ever end up with a rather limited number of alternates anyway, so the extra couple of cycles we save feels more like a micro optimization. Refactor the code accordingly. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/odb.c b/odb.c index 0c808bb288f168..4f884e3b509a59 100644 --- a/odb.c +++ b/odb.c @@ -142,8 +142,7 @@ static void read_info_alternates(struct object_database *odb, static struct odb_source *link_alt_odb_entry(struct object_database *odb, const char *dir, const char *relative_base, - int depth, - const char *normalized_objdir) + int depth) { struct odb_source *alternate = NULL; struct strbuf pathbuf = STRBUF_INIT; @@ -170,7 +169,10 @@ static struct odb_source *link_alt_odb_entry(struct object_database *odb, while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/') strbuf_setlen(&pathbuf, pathbuf.len - 1); - if (!alt_odb_usable(odb, &pathbuf, normalized_objdir, &pos)) + strbuf_reset(&tmp); + strbuf_realpath(&tmp, odb->sources->path, 1); + + if (!alt_odb_usable(odb, &pathbuf, tmp.buf, &pos)) goto error; CALLOC_ARRAY(alternate, 1); @@ -227,7 +229,6 @@ static const char *parse_alt_odb_entry(const char *string, static void link_alt_odb_entries(struct object_database *odb, const char *alt, int sep, const char *relative_base, int depth) { - struct strbuf objdirbuf = STRBUF_INIT; struct strbuf dir = STRBUF_INIT; if (!alt || !*alt) @@ -239,17 +240,13 @@ static void link_alt_odb_entries(struct object_database *odb, const char *alt, return; } - strbuf_realpath(&objdirbuf, odb->sources->path, 1); - while (*alt) { alt = parse_alt_odb_entry(alt, sep, &dir); if (!dir.len) continue; - link_alt_odb_entry(odb, dir.buf, - relative_base, depth, objdirbuf.buf); + link_alt_odb_entry(odb, dir.buf, relative_base, depth); } strbuf_release(&dir); - strbuf_release(&objdirbuf); } static void read_info_alternates(struct object_database *odb, @@ -317,20 +314,12 @@ void odb_add_to_alternates_file(struct object_database *odb, struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, const char *dir) { - struct odb_source *alternate; - char *objdir; - /* * Make sure alternates are initialized, or else our entry may be * overwritten when they are. */ odb_prepare_alternates(odb); - - objdir = real_pathdup(odb->sources->path, 1); - alternate = link_alt_odb_entry(odb, dir, NULL, 0, objdir); - - free(objdir); - return alternate; + return link_alt_odb_entry(odb, dir, NULL, 0); } struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, From 9ff212961506679c1e2c1541b17ab2bd8563ff15 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:46 +0200 Subject: [PATCH 07/25] midx: drop redundant `struct repository` parameter There are a couple of functions that take both a `struct repository` and a `struct multi_pack_index`. This provides redundant information though without much benefit given that the multi-pack index already has a pointer to its owning repository. Drop the `struct repository` parameter from such functions. While at it, reorder the list of parameters of `fill_midx_entry()` so that the MIDX comes first to better align with our coding guidelines. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 2 +- midx-write.c | 16 +++++++--------- midx.c | 18 +++++++++--------- midx.h | 6 +++--- pack-bitmap.c | 4 ++-- packfile.c | 4 ++-- t/helper/test-read-midx.c | 4 ++-- 7 files changed, 26 insertions(+), 28 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 3dd84495b869e0..b9fd685b8fcc8f 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1733,7 +1733,7 @@ static int want_object_in_pack_mtime(const struct object_id *oid, struct multi_pack_index *m = get_multi_pack_index(source); struct pack_entry e; - if (m && fill_midx_entry(the_repository, oid, &e, m)) { + if (m && fill_midx_entry(m, oid, &e)) { want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset, found_mtime); if (want != -1) return want; diff --git a/midx-write.c b/midx-write.c index d38caceadb4fa4..b858be475fc3fa 100644 --- a/midx-write.c +++ b/midx-write.c @@ -942,8 +942,7 @@ static int fill_packs_from_midx(struct write_midx_context *ctx, */ if (flags & MIDX_WRITE_REV_INDEX || preferred_pack_name) { - if (prepare_midx_pack(ctx->repo, m, - m->num_packs_in_base + i)) { + if (prepare_midx_pack(m, m->num_packs_in_base + i)) { error(_("could not load pack")); return 1; } @@ -1566,7 +1565,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla if (count[i]) continue; - if (prepare_midx_pack(r, m, i)) + if (prepare_midx_pack(m, i)) continue; if (m->packs[i]->pack_keep || m->packs[i]->is_cruft) @@ -1612,13 +1611,12 @@ static int compare_by_mtime(const void *a_, const void *b_) return 0; } -static int want_included_pack(struct repository *r, - struct multi_pack_index *m, +static int want_included_pack(struct multi_pack_index *m, int pack_kept_objects, uint32_t pack_int_id) { struct packed_git *p; - if (prepare_midx_pack(r, m, pack_int_id)) + if (prepare_midx_pack(m, pack_int_id)) return 0; p = m->packs[pack_int_id]; if (!pack_kept_objects && p->pack_keep) @@ -1640,7 +1638,7 @@ static void fill_included_packs_all(struct repository *r, repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects); for (i = 0; i < m->num_packs; i++) { - if (!want_included_pack(r, m, pack_kept_objects, i)) + if (!want_included_pack(m, pack_kept_objects, i)) continue; include_pack[i] = 1; @@ -1664,7 +1662,7 @@ static void fill_included_packs_batch(struct repository *r, for (i = 0; i < m->num_packs; i++) { pack_info[i].pack_int_id = i; - if (prepare_midx_pack(r, m, i)) + if (prepare_midx_pack(m, i)) continue; pack_info[i].mtime = m->packs[i]->mtime; @@ -1683,7 +1681,7 @@ static void fill_included_packs_batch(struct repository *r, struct packed_git *p = m->packs[pack_int_id]; uint64_t expected_size; - if (!want_included_pack(r, m, pack_kept_objects, pack_int_id)) + if (!want_included_pack(m, pack_kept_objects, pack_int_id)) continue; /* diff --git a/midx.c b/midx.c index b9ca0915a67f10..8459dda8c9e810 100644 --- a/midx.c +++ b/midx.c @@ -450,9 +450,10 @@ static uint32_t midx_for_pack(struct multi_pack_index **_m, return pack_int_id - m->num_packs_in_base; } -int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, +int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id) { + struct repository *r = m->repo; struct strbuf pack_name = STRBUF_INIT; struct strbuf key = STRBUF_INIT; struct packed_git *p; @@ -507,7 +508,7 @@ struct packed_git *nth_midxed_pack(struct multi_pack_index *m, #define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t)) -int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, +int nth_bitmapped_pack(struct multi_pack_index *m, struct bitmapped_pack *bp, uint32_t pack_int_id) { uint32_t local_pack_int_id = midx_for_pack(&m, pack_int_id); @@ -515,7 +516,7 @@ int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, if (!m->chunk_bitmapped_packs) return error(_("MIDX does not contain the BTMP chunk")); - if (prepare_midx_pack(r, m, pack_int_id)) + if (prepare_midx_pack(m, pack_int_id)) return error(_("could not load bitmapped pack %"PRIu32), pack_int_id); bp->p = m->packs[local_pack_int_id]; @@ -600,10 +601,9 @@ uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos) (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH); } -int fill_midx_entry(struct repository *r, +int fill_midx_entry(struct multi_pack_index *m, const struct object_id *oid, - struct pack_entry *e, - struct multi_pack_index *m) + struct pack_entry *e) { uint32_t pos; uint32_t pack_int_id; @@ -615,7 +615,7 @@ int fill_midx_entry(struct repository *r, midx_for_object(&m, pos); pack_int_id = nth_midxed_pack_int_id(m, pos); - if (prepare_midx_pack(r, m, pack_int_id)) + if (prepare_midx_pack(m, pack_int_id)) return 0; p = m->packs[pack_int_id - m->num_packs_in_base]; @@ -912,7 +912,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag _("Looking for referenced packfiles"), m->num_packs + m->num_packs_in_base); for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) { - if (prepare_midx_pack(r, m, i)) + if (prepare_midx_pack(m, i)) midx_report("failed to load pack in position %d", i); display_progress(progress, i + 1); @@ -989,7 +989,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag nth_midxed_object_oid(&oid, m, pairs[i].pos); - if (!fill_midx_entry(r, &oid, &e, m)) { + if (!fill_midx_entry(m, &oid, &e)) { midx_report(_("failed to load pack entry for oid[%d] = %s"), pairs[i].pos, oid_to_hex(&oid)); continue; diff --git a/midx.h b/midx.h index 28c426a8232997..f7e07083e1f9de 100644 --- a/midx.h +++ b/midx.h @@ -103,10 +103,10 @@ void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, struct multi_pack_index *load_multi_pack_index(struct repository *r, const char *object_dir, int local); -int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id); +int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id); struct packed_git *nth_midxed_pack(struct multi_pack_index *m, uint32_t pack_int_id); -int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, +int nth_bitmapped_pack(struct multi_pack_index *m, struct bitmapped_pack *bp, uint32_t pack_int_id); int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result); @@ -118,7 +118,7 @@ uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos); struct object_id *nth_midxed_object_oid(struct object_id *oid, struct multi_pack_index *m, uint32_t n); -int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m); +int fill_midx_entry(struct multi_pack_index *m, const struct object_id *oid, struct pack_entry *e); int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id); diff --git a/pack-bitmap.c b/pack-bitmap.c index d14421ee204414..fb0b11ca073856 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -493,7 +493,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, } for (i = 0; i < bitmap_git->midx->num_packs + bitmap_git->midx->num_packs_in_base; i++) { - if (prepare_midx_pack(bitmap_repo(bitmap_git), bitmap_git->midx, i)) { + if (prepare_midx_pack(bitmap_git->midx, i)) { warning(_("could not open pack %s"), bitmap_git->midx->pack_names[i]); goto cleanup; @@ -2466,7 +2466,7 @@ void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, struct multi_pack_index *m = bitmap_git->midx; for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) { struct bitmapped_pack pack; - if (nth_bitmapped_pack(r, bitmap_git->midx, &pack, i) < 0) { + if (nth_bitmapped_pack(bitmap_git->midx, &pack, i) < 0) { warning(_("unable to load pack: '%s', disabling pack-reuse"), bitmap_git->midx->pack_names[i]); free(packs); diff --git a/packfile.c b/packfile.c index a38544b87bf2f1..acb680966dacf9 100644 --- a/packfile.c +++ b/packfile.c @@ -1091,7 +1091,7 @@ struct packed_git *get_all_packs(struct repository *r) if (!m) continue; for (uint32_t i = 0; i < m->num_packs + m->num_packs_in_base; i++) - prepare_midx_pack(r, m, i); + prepare_midx_pack(m, i); } return r->objects->packed_git; @@ -2077,7 +2077,7 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa prepare_packed_git(r); for (struct odb_source *source = r->objects->sources; source; source = source->next) - if (source->midx && fill_midx_entry(r, oid, e, source->midx)) + if (source->midx && fill_midx_entry(source->midx, oid, e)) return 1; if (!r->objects->packed_git) diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index da2aa036b57ef6..e430aa247c6f34 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -65,7 +65,7 @@ static int read_midx_file(const char *object_dir, const char *checksum, for (i = 0; i < m->num_objects; i++) { nth_midxed_object_oid(&oid, m, i + m->num_objects_in_base); - fill_midx_entry(the_repository, &oid, &e, m); + fill_midx_entry(m, &oid, &e); printf("%s %"PRIu64"\t%s\n", oid_to_hex(&oid), e.offset, e.p->pack_name); @@ -126,7 +126,7 @@ static int read_midx_bitmapped_packs(const char *object_dir) return 1; for (i = 0; i < midx->num_packs + midx->num_packs_in_base; i++) { - if (nth_bitmapped_pack(the_repository, midx, &pack, i) < 0) { + if (nth_bitmapped_pack(midx, &pack, i) < 0) { close_midx(midx); return 1; } From 017db7bb14246dea55b678fc20e34ce91c28968a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:47 +0200 Subject: [PATCH 08/25] midx: load multi-pack indices via their source To load a multi-pack index the caller is expected to pass both the repository and the object directory where the multi-pack index is located. While this works, this layout has a couple of downsides: - We need to pass in information reduntant with the owning source, namely its object directory and whether the source is local or not. - We don't have access to the source when loading the multi-pack index. If we had that access, we could store a pointer to the owning source in the MIDX and thus deduplicate some information. - Multi-pack indices are inherently specific to the object source and its format. With the goal of pluggable object backends in mind we will eventually want the backends to own the logic of reading and writing multi-pack indices. Making the logic work on top of object sources is a step into that direction. Refactor loading of multi-pack indices accordingly. This surfaces one small problem though: git-multi-pack-index(1) and our MIDX test helper both know to read and write multi-pack-indices located in a different object directory. This issue is addressed by adding the user-provided object directory as an in-memory alternate. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/multi-pack-index.c | 18 ++++++++++-- midx.c | 57 ++++++++++++++++--------------------- midx.h | 6 ++-- t/helper/test-read-midx.c | 25 +++++++++------- t/t5319-multi-pack-index.sh | 8 +++--- 5 files changed, 62 insertions(+), 52 deletions(-) diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index aa25b06f9d0f89..e4a9305af3ab75 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -64,12 +64,20 @@ static int parse_object_dir(const struct option *opt, const char *arg, char **value = opt->value; free(*value); if (unset) - *value = xstrdup(repo_get_object_directory(the_repository)); + *value = xstrdup(the_repository->objects->sources->path); else *value = real_pathdup(arg, 1); return 0; } +static struct odb_source *handle_object_dir_option(struct repository *repo) +{ + struct odb_source *source = odb_find_source(repo->objects, opts.object_dir); + if (!source) + source = odb_add_to_alternates_memory(repo->objects, opts.object_dir); + return source; +} + static struct option common_opts[] = { OPT_CALLBACK(0, "object-dir", &opts.object_dir, N_("directory"), @@ -157,6 +165,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_write_usage, options); + handle_object_dir_option(repo); FREE_AND_NULL(options); @@ -193,6 +202,8 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, N_("force progress reporting"), MIDX_PROGRESS), OPT_END(), }; + struct odb_source *source; + options = add_common_options(builtin_multi_pack_index_verify_options); trace2_cmd_mode(argv[0]); @@ -205,10 +216,11 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_verify_usage, options); + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return verify_midx_file(the_repository, opts.object_dir, opts.flags); + return verify_midx_file(source, opts.flags); } static int cmd_multi_pack_index_expire(int argc, const char **argv, @@ -233,6 +245,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_expire_usage, options); + handle_object_dir_option(the_repository); FREE_AND_NULL(options); @@ -265,6 +278,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_repack_usage, options); + handle_object_dir_option(the_repository); FREE_AND_NULL(options); diff --git a/midx.c b/midx.c index 8459dda8c9e810..831a7e9b5f2c08 100644 --- a/midx.c +++ b/midx.c @@ -95,11 +95,10 @@ static int midx_read_object_offsets(const unsigned char *chunk_start, return 0; } -static struct multi_pack_index *load_multi_pack_index_one(struct repository *r, - const char *object_dir, - const char *midx_name, - int local) +static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source, + const char *midx_name) { + struct repository *r = source->odb->repo; struct multi_pack_index *m = NULL; int fd; struct stat st; @@ -129,10 +128,10 @@ static struct multi_pack_index *load_multi_pack_index_one(struct repository *r, midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - FLEX_ALLOC_STR(m, object_dir, object_dir); + FLEX_ALLOC_STR(m, object_dir, source->path); m->data = midx_map; m->data_len = midx_size; - m->local = local; + m->local = source->local; m->repo = r; m->signature = get_be32(m->data); @@ -297,19 +296,18 @@ static int add_midx_to_chain(struct multi_pack_index *midx, return 1; } -static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, - const char *object_dir, - int local, +static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source, int fd, struct stat *st, int *incomplete_chain) { + const struct git_hash_algo *hash_algo = source->odb->repo->hash_algo; struct multi_pack_index *midx_chain = NULL; struct strbuf buf = STRBUF_INIT; int valid = 1; uint32_t i, count; FILE *fp = xfdopen(fd, "r"); - count = st->st_size / (r->hash_algo->hexsz + 1); + count = st->st_size / (hash_algo->hexsz + 1); for (i = 0; i < count; i++) { struct multi_pack_index *m; @@ -318,7 +316,7 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, if (strbuf_getline_lf(&buf, fp) == EOF) break; - if (get_oid_hex_algop(buf.buf, &layer, r->hash_algo)) { + if (get_oid_hex_algop(buf.buf, &layer, hash_algo)) { warning(_("invalid multi-pack-index chain: line '%s' " "not a hash"), buf.buf); @@ -329,9 +327,9 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, valid = 0; strbuf_reset(&buf); - get_split_midx_filename_ext(r->hash_algo, &buf, object_dir, + get_split_midx_filename_ext(hash_algo, &buf, source->path, layer.hash, MIDX_EXT_MIDX); - m = load_multi_pack_index_one(r, object_dir, buf.buf, local); + m = load_multi_pack_index_one(source, buf.buf); if (m) { if (add_midx_to_chain(m, midx_chain)) { @@ -354,40 +352,35 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, return midx_chain; } -static struct multi_pack_index *load_multi_pack_index_chain(struct repository *r, - const char *object_dir, - int local) +static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *source) { struct strbuf chain_file = STRBUF_INIT; struct stat st; int fd; struct multi_pack_index *m = NULL; - get_midx_chain_filename(&chain_file, object_dir); - if (open_multi_pack_index_chain(r->hash_algo, chain_file.buf, &fd, &st)) { + get_midx_chain_filename(&chain_file, source->path); + if (open_multi_pack_index_chain(source->odb->repo->hash_algo, chain_file.buf, &fd, &st)) { int incomplete; /* ownership of fd is taken over by load function */ - m = load_midx_chain_fd_st(r, object_dir, local, fd, &st, - &incomplete); + m = load_midx_chain_fd_st(source, fd, &st, &incomplete); } strbuf_release(&chain_file); return m; } -struct multi_pack_index *load_multi_pack_index(struct repository *r, - const char *object_dir, - int local) +struct multi_pack_index *load_multi_pack_index(struct odb_source *source) { struct strbuf midx_name = STRBUF_INIT; struct multi_pack_index *m; - get_midx_filename(r->hash_algo, &midx_name, object_dir); + get_midx_filename(source->odb->repo->hash_algo, &midx_name, + source->path); - m = load_multi_pack_index_one(r, object_dir, - midx_name.buf, local); + m = load_multi_pack_index_one(source, midx_name.buf); if (!m) - m = load_multi_pack_index_chain(r, object_dir, local); + m = load_multi_pack_index_chain(source); strbuf_release(&midx_name); @@ -734,8 +727,7 @@ int prepare_multi_pack_index_one(struct odb_source *source) if (source->midx) return 1; - source->midx = load_multi_pack_index(r, source->path, - source->local); + source->midx = load_multi_pack_index(source); return !!source->midx; } @@ -880,12 +872,13 @@ static int compare_pair_pos_vs_id(const void *_a, const void *_b) display_progress(progress, _n); \ } while (0) -int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags) +int verify_midx_file(struct odb_source *source, unsigned flags) { + struct repository *r = source->odb->repo; struct pair_pos_vs_id *pairs = NULL; uint32_t i; struct progress *progress = NULL; - struct multi_pack_index *m = load_multi_pack_index(r, object_dir, 1); + struct multi_pack_index *m = load_multi_pack_index(source); struct multi_pack_index *curr; verify_midx_error = 0; @@ -894,7 +887,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag struct stat sb; struct strbuf filename = STRBUF_INIT; - get_midx_filename(r->hash_algo, &filename, object_dir); + get_midx_filename(r->hash_algo, &filename, source->path); if (!stat(filename.buf, &sb)) { error(_("multi-pack-index file exists, but failed to parse")); diff --git a/midx.h b/midx.h index f7e07083e1f9de..970d043989a5b0 100644 --- a/midx.h +++ b/midx.h @@ -100,9 +100,7 @@ void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, struct strbuf *buf, const char *object_dir, const unsigned char *hash, const char *ext); -struct multi_pack_index *load_multi_pack_index(struct repository *r, - const char *object_dir, - int local); +struct multi_pack_index *load_multi_pack_index(struct odb_source *source); int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id); struct packed_git *nth_midxed_pack(struct multi_pack_index *m, uint32_t pack_int_id); @@ -136,7 +134,7 @@ int write_midx_file_only(struct repository *r, const char *object_dir, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags); void clear_midx_file(struct repository *r); -int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags); +int verify_midx_file(struct odb_source *source, unsigned flags); int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags); int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags); diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index e430aa247c6f34..bcb8ea767179a6 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -11,14 +11,24 @@ #include "gettext.h" #include "pack-revindex.h" +static struct multi_pack_index *setup_midx(const char *object_dir) +{ + struct odb_source *source; + setup_git_directory(); + source = odb_find_source(the_repository->objects, object_dir); + if (!source) + source = odb_add_to_alternates_memory(the_repository->objects, + object_dir); + return load_multi_pack_index(source); +} + static int read_midx_file(const char *object_dir, const char *checksum, int show_objects) { uint32_t i; struct multi_pack_index *m; - setup_git_directory(); - m = load_multi_pack_index(the_repository, object_dir, 1); + m = setup_midx(object_dir); if (!m) return 1; @@ -81,8 +91,7 @@ static int read_midx_checksum(const char *object_dir) { struct multi_pack_index *m; - setup_git_directory(); - m = load_multi_pack_index(the_repository, object_dir, 1); + m = setup_midx(object_dir); if (!m) return 1; printf("%s\n", hash_to_hex(get_midx_checksum(m))); @@ -96,9 +105,7 @@ static int read_midx_preferred_pack(const char *object_dir) struct multi_pack_index *midx = NULL; uint32_t preferred_pack; - setup_git_directory(); - - midx = load_multi_pack_index(the_repository, object_dir, 1); + midx = setup_midx(object_dir); if (!midx) return 1; @@ -119,9 +126,7 @@ static int read_midx_bitmapped_packs(const char *object_dir) struct bitmapped_pack pack; uint32_t i; - setup_git_directory(); - - midx = load_multi_pack_index(the_repository, object_dir, 1); + midx = setup_midx(object_dir); if (!midx) return 1; diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index bd75dea9501ed7..4e5e882989fc45 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -28,11 +28,11 @@ midx_read_expect () { EOF if test $NUM_PACKS -ge 1 then - ls $OBJECT_DIR/pack/ | grep idx | sort + ls "$OBJECT_DIR"/pack/ | grep idx | sort fi && printf "object-dir: $OBJECT_DIR\n" } >expect && - test-tool read-midx $OBJECT_DIR >actual && + test-tool read-midx "$OBJECT_DIR" >actual && test_cmp expect actual } @@ -305,7 +305,7 @@ test_expect_success 'midx picks objects from preferred pack' ' ofs=$(git show-index expect && grep ^$b out >actual && @@ -639,7 +639,7 @@ test_expect_success 'force some 64-bit offsets with pack-objects' ' ( cd ../objects64 && pwd ) >.git/objects/info/alternates && midx64=$(git multi-pack-index --object-dir=../objects64 write) ) && - midx_read_expect 1 63 5 objects64 " large-offsets" + midx_read_expect 1 63 5 "$(pwd)/objects64" " large-offsets" ' test_expect_success 'verify multi-pack-index with 64-bit offsets' ' From c3f5d251469525a52074b0373671a588f0e5b972 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:48 +0200 Subject: [PATCH 09/25] midx: write multi-pack indices via their source Similar to the preceding commit, refactor the writing side of multi-pack indices so that we pass in the object database source where the index should be written to. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/multi-pack-index.c | 19 ++++++----- builtin/repack.c | 2 +- midx-write.c | 67 ++++++++++++++++++-------------------- midx.h | 8 ++--- 4 files changed, 47 insertions(+), 49 deletions(-) diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index e4a9305af3ab75..b1e971e535d20e 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -147,6 +147,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, N_("refs snapshot for selecting bitmap commits")), OPT_END(), }; + struct odb_source *source; int ret; opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE; @@ -165,7 +166,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_write_usage, options); - handle_object_dir_option(repo); + source = handle_object_dir_option(repo); FREE_AND_NULL(options); @@ -174,7 +175,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, read_packs_from_stdin(&packs); - ret = write_midx_file_only(repo, opts.object_dir, &packs, + ret = write_midx_file_only(source, &packs, opts.preferred_pack, opts.refs_snapshot, opts.flags); @@ -185,7 +186,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } - ret = write_midx_file(repo, opts.object_dir, opts.preferred_pack, + ret = write_midx_file(source, opts.preferred_pack, opts.refs_snapshot, opts.flags); free(opts.refs_snapshot); @@ -233,6 +234,8 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, N_("force progress reporting"), MIDX_PROGRESS), OPT_END(), }; + struct odb_source *source; + options = add_common_options(builtin_multi_pack_index_expire_options); trace2_cmd_mode(argv[0]); @@ -245,11 +248,11 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_expire_usage, options); - handle_object_dir_option(the_repository); + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return expire_midx_packs(the_repository, opts.object_dir, opts.flags); + return expire_midx_packs(source, opts.flags); } static int cmd_multi_pack_index_repack(int argc, const char **argv, @@ -264,6 +267,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, N_("force progress reporting"), MIDX_PROGRESS), OPT_END(), }; + struct odb_source *source; options = add_common_options(builtin_multi_pack_index_repack_options); @@ -278,12 +282,11 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_repack_usage, options); - handle_object_dir_option(the_repository); + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return midx_repack(the_repository, opts.object_dir, - (size_t)opts.batch_size, opts.flags); + return midx_repack(source, (size_t)opts.batch_size, opts.flags); } int cmd_multi_pack_index(int argc, diff --git a/builtin/repack.c b/builtin/repack.c index 21723866b9cdeb..94dec26f185be8 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -1711,7 +1711,7 @@ int cmd_repack(int argc, unsigned flags = 0; if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(the_repository, repo_get_object_directory(the_repository), + write_midx_file(the_repository->objects->sources, NULL, NULL, flags); } diff --git a/midx-write.c b/midx-write.c index b858be475fc3fa..bf7c01d4b1d67c 100644 --- a/midx-write.c +++ b/midx-write.c @@ -913,13 +913,6 @@ static int write_midx_bitmap(struct write_midx_context *ctx, return ret; } -static struct multi_pack_index *lookup_multi_pack_index(struct repository *r, - const char *object_dir) -{ - struct odb_source *source = odb_find_source_or_die(r->objects, object_dir); - return get_multi_pack_index(source); -} - static int fill_packs_from_midx(struct write_midx_context *ctx, const char *preferred_pack_name, uint32_t flags) { @@ -1010,7 +1003,7 @@ static int link_midx_to_chain(struct multi_pack_index *m) return ret; } -static void clear_midx_files(struct repository *r, const char *object_dir, +static void clear_midx_files(struct odb_source *source, const char **hashes, uint32_t hashes_nr, unsigned incremental) { @@ -1029,16 +1022,16 @@ static void clear_midx_files(struct repository *r, const char *object_dir, uint32_t i, j; for (i = 0; i < ARRAY_SIZE(exts); i++) { - clear_incremental_midx_files_ext(object_dir, exts[i], + clear_incremental_midx_files_ext(source->path, exts[i], hashes, hashes_nr); for (j = 0; j < hashes_nr; j++) - clear_midx_files_ext(object_dir, exts[i], hashes[j]); + clear_midx_files_ext(source->path, exts[i], hashes[j]); } if (incremental) - get_midx_filename(r->hash_algo, &buf, object_dir); + get_midx_filename(source->odb->repo->hash_algo, &buf, source->path); else - get_midx_chain_filename(&buf, object_dir); + get_midx_chain_filename(&buf, source->path); if (unlink(buf.buf) && errno != ENOENT) die_errno(_("failed to clear multi-pack-index at %s"), buf.buf); @@ -1046,13 +1039,14 @@ static void clear_midx_files(struct repository *r, const char *object_dir, strbuf_release(&buf); } -static int write_midx_internal(struct repository *r, const char *object_dir, +static int write_midx_internal(struct odb_source *source, struct string_list *packs_to_include, struct string_list *packs_to_drop, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) { + struct repository *r = source->odb->repo; struct strbuf midx_name = STRBUF_INIT; unsigned char midx_hash[GIT_MAX_RAWSZ]; uint32_t i, start_pack; @@ -1076,15 +1070,15 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (ctx.incremental) strbuf_addf(&midx_name, "%s/pack/multi-pack-index.d/tmp_midx_XXXXXX", - object_dir); + source->path); else - get_midx_filename(r->hash_algo, &midx_name, object_dir); + get_midx_filename(r->hash_algo, &midx_name, source->path); if (safe_create_leading_directories(r, midx_name.buf)) die_errno(_("unable to create leading directories of %s"), midx_name.buf); if (!packs_to_include || ctx.incremental) { - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); + struct multi_pack_index *m = get_multi_pack_index(source); if (m && !midx_checksum_valid(m)) { warning(_("ignoring existing multi-pack-index; checksum mismatch")); m = NULL; @@ -1138,7 +1132,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, ctx.to_include = packs_to_include; - for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx); + for_each_file_in_pack_dir(source->path, add_pack_to_midx, &ctx); stop_progress(&ctx.progress); if ((ctx.m && ctx.nr == ctx.m->num_packs + ctx.m->num_packs_in_base) && @@ -1158,7 +1152,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, * corresponding bitmap (or one wasn't requested). */ if (!want_bitmap) - clear_midx_files_ext(object_dir, "bitmap", NULL); + clear_midx_files_ext(source->path, "bitmap", NULL); goto cleanup; } } @@ -1326,7 +1320,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (ctx.incremental) { struct strbuf lock_name = STRBUF_INIT; - get_midx_chain_filename(&lock_name, object_dir); + get_midx_chain_filename(&lock_name, source->path); hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR); strbuf_release(&lock_name); @@ -1389,7 +1383,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (flags & MIDX_WRITE_REV_INDEX && git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0)) - write_midx_reverse_index(&ctx, object_dir, midx_hash); + write_midx_reverse_index(&ctx, source->path, midx_hash); if (flags & MIDX_WRITE_BITMAP) { struct packing_data pdata; @@ -1412,7 +1406,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, FREE_AND_NULL(ctx.entries); ctx.entries_nr = 0; - if (write_midx_bitmap(&ctx, object_dir, + if (write_midx_bitmap(&ctx, source->path, midx_hash, &pdata, commits, commits_nr, flags) < 0) { error(_("could not write multi-pack bitmap")); @@ -1446,7 +1440,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, return -1; get_split_midx_filename_ext(r->hash_algo, &final_midx_name, - object_dir, midx_hash, MIDX_EXT_MIDX); + source->path, midx_hash, MIDX_EXT_MIDX); if (rename_tempfile(&incr, final_midx_name.buf) < 0) { error_errno(_("unable to rename new multi-pack-index layer")); @@ -1479,7 +1473,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (commit_lock_file(&lk) < 0) die_errno(_("could not write multi-pack-index")); - clear_midx_files(r, object_dir, keep_hashes, + clear_midx_files(source, keep_hashes, ctx.num_multi_pack_indexes_before + 1, ctx.incremental); @@ -1508,29 +1502,29 @@ static int write_midx_internal(struct repository *r, const char *object_dir, return result; } -int write_midx_file(struct repository *r, const char *object_dir, +int write_midx_file(struct odb_source *source, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) { - return write_midx_internal(r, object_dir, NULL, NULL, + return write_midx_internal(source, NULL, NULL, preferred_pack_name, refs_snapshot, flags); } -int write_midx_file_only(struct repository *r, const char *object_dir, +int write_midx_file_only(struct odb_source *source, struct string_list *packs_to_include, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) { - return write_midx_internal(r, object_dir, packs_to_include, NULL, + return write_midx_internal(source, packs_to_include, NULL, preferred_pack_name, refs_snapshot, flags); } -int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags) +int expire_midx_packs(struct odb_source *source, unsigned flags) { uint32_t i, *count, result = 0; struct string_list packs_to_drop = STRING_LIST_INIT_DUP; - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); + struct multi_pack_index *m = get_multi_pack_index(source); struct progress *progress = NULL; if (!m) @@ -1543,7 +1537,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla if (flags & MIDX_PROGRESS) progress = start_delayed_progress( - r, + source->odb->repo, _("Counting referenced objects"), m->num_objects); for (i = 0; i < m->num_objects; i++) { @@ -1555,7 +1549,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla if (flags & MIDX_PROGRESS) progress = start_delayed_progress( - r, + source->odb->repo, _("Finding and deleting unreferenced packfiles"), m->num_packs); for (i = 0; i < m->num_packs; i++) { @@ -1583,7 +1577,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla free(count); if (packs_to_drop.nr) - result = write_midx_internal(r, object_dir, NULL, + result = write_midx_internal(source, NULL, &packs_to_drop, NULL, NULL, flags); string_list_clear(&packs_to_drop, 0); @@ -1708,14 +1702,15 @@ static void fill_included_packs_batch(struct repository *r, free(pack_info); } -int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags) +int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags) { + struct repository *r = source->odb->repo; int result = 0; uint32_t i, packs_to_repack = 0; unsigned char *include_pack; struct child_process cmd = CHILD_PROCESS_INIT; FILE *cmd_in; - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); + struct multi_pack_index *m = get_multi_pack_index(source); /* * When updating the default for these configuration @@ -1749,7 +1744,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, strvec_push(&cmd.args, "pack-objects"); - strvec_pushf(&cmd.args, "%s/pack/pack", object_dir); + strvec_pushf(&cmd.args, "%s/pack/pack", source->path); if (delta_base_offset) strvec_push(&cmd.args, "--delta-base-offset"); @@ -1790,7 +1785,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, goto cleanup; } - result = write_midx_internal(r, object_dir, NULL, NULL, NULL, NULL, + result = write_midx_internal(source, NULL, NULL, NULL, NULL, flags); cleanup: diff --git a/midx.h b/midx.h index 970d043989a5b0..d162001fbbe2b3 100644 --- a/midx.h +++ b/midx.h @@ -126,17 +126,17 @@ int prepare_multi_pack_index_one(struct odb_source *source); * Variant of write_midx_file which writes a MIDX containing only the packs * specified in packs_to_include. */ -int write_midx_file(struct repository *r, const char *object_dir, +int write_midx_file(struct odb_source *source, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags); -int write_midx_file_only(struct repository *r, const char *object_dir, +int write_midx_file_only(struct odb_source *source, struct string_list *packs_to_include, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags); void clear_midx_file(struct repository *r); int verify_midx_file(struct odb_source *source, unsigned flags); -int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags); -int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags); +int expire_midx_packs(struct odb_source *source, unsigned flags); +int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags); void close_midx(struct multi_pack_index *m); From 7744936f374308d6fa3c6e317fb8fe0b685d0ef2 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:49 +0200 Subject: [PATCH 10/25] midx: stop duplicating info redundant with its owning source Multi-pack indices store some information that is redundant with their owning source: - The locality bit that tracks whether the source is the primary object source or an alternate. - The object directory path the multi-pack index is located in. - The pointer to the owning parent directory. All of this information is already contained in `struct odb_source`. So now that we always have that struct available when loading a multi-pack index we have it readily accessible. Drop the redundant information and instead store a pointer to the object source. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/repack.c | 5 +++-- midx-write.c | 9 +++++---- midx.c | 21 +++++++++++---------- midx.h | 7 ++----- pack-bitmap.c | 13 +++++++------ pack-revindex.c | 14 +++++++------- t/helper/test-read-midx.c | 2 +- 7 files changed, 36 insertions(+), 35 deletions(-) diff --git a/builtin/repack.c b/builtin/repack.c index 94dec26f185be8..5af3e27357c3ec 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -223,9 +223,10 @@ static void mark_packs_for_deletion(struct existing_packs *existing, static void remove_redundant_pack(const char *dir_name, const char *base_name) { struct strbuf buf = STRBUF_INIT; - struct multi_pack_index *m = get_multi_pack_index(the_repository->objects->sources); + struct odb_source *source = the_repository->objects->sources; + struct multi_pack_index *m = get_multi_pack_index(source); strbuf_addf(&buf, "%s.pack", base_name); - if (m && m->local && midx_contains_pack(m, buf.buf)) + if (m && source->local && midx_contains_pack(m, buf.buf)) clear_midx_file(the_repository); strbuf_insertf(&buf, 0, "%s/", dir_name); unlink_pack_path(buf.buf, 1); diff --git a/midx-write.c b/midx-write.c index bf7c01d4b1d67c..84f76856d67ee9 100644 --- a/midx-write.c +++ b/midx-write.c @@ -981,10 +981,11 @@ static int link_midx_to_chain(struct multi_pack_index *m) for (i = 0; i < ARRAY_SIZE(midx_exts); i++) { const unsigned char *hash = get_midx_checksum(m); - get_midx_filename_ext(m->repo->hash_algo, &from, m->object_dir, + get_midx_filename_ext(m->source->odb->repo->hash_algo, &from, + m->source->path, hash, midx_exts[i].non_split); - get_split_midx_filename_ext(m->repo->hash_algo, &to, - m->object_dir, hash, + get_split_midx_filename_ext(m->source->odb->repo->hash_algo, &to, + m->source->path, hash, midx_exts[i].split); if (link(from.buf, to.buf) < 0 && errno != ENOENT) { @@ -1109,7 +1110,7 @@ static int write_midx_internal(struct odb_source *source, if (flags & MIDX_WRITE_BITMAP && load_midx_revindex(m)) { error(_("could not load reverse index for MIDX %s"), hash_to_hex_algop(get_midx_checksum(m), - m->repo->hash_algo)); + m->source->odb->repo->hash_algo)); result = 1; goto cleanup; } diff --git a/midx.c b/midx.c index 831a7e9b5f2c08..81bf3c4d5f351e 100644 --- a/midx.c +++ b/midx.c @@ -26,7 +26,7 @@ int cmp_idx_or_pack_name(const char *idx_or_pack_name, const unsigned char *get_midx_checksum(struct multi_pack_index *m) { - return m->data + m->data_len - m->repo->hash_algo->rawsz; + return m->data + m->data_len - m->source->odb->repo->hash_algo->rawsz; } void get_midx_filename(const struct git_hash_algo *hash_algo, @@ -128,11 +128,10 @@ static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *sou midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - FLEX_ALLOC_STR(m, object_dir, source->path); + CALLOC_ARRAY(m, 1); m->data = midx_map; m->data_len = midx_size; - m->local = source->local; - m->repo = r; + m->source = source; m->signature = get_be32(m->data); if (m->signature != MIDX_SIGNATURE) @@ -446,7 +445,7 @@ static uint32_t midx_for_pack(struct multi_pack_index **_m, int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id) { - struct repository *r = m->repo; + struct repository *r = m->source->odb->repo; struct strbuf pack_name = STRBUF_INIT; struct strbuf key = STRBUF_INIT; struct packed_git *p; @@ -458,7 +457,7 @@ int prepare_midx_pack(struct multi_pack_index *m, if (m->packs[pack_int_id]) return 0; - strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir, + strbuf_addf(&pack_name, "%s/pack/%s", m->source->path, m->pack_names[pack_int_id]); /* pack_map holds the ".pack" name, but we have the .idx */ @@ -469,7 +468,8 @@ int prepare_midx_pack(struct multi_pack_index *m, strhash(key.buf), key.buf, struct packed_git, packmap_ent); if (!p) { - p = add_packed_git(r, pack_name.buf, pack_name.len, m->local); + p = add_packed_git(r, pack_name.buf, pack_name.len, + m->source->local); if (p) { install_packed_git(r, p); list_add_tail(&p->mru, &r->objects->packed_git_mru); @@ -528,7 +528,8 @@ int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result) { int ret = bsearch_hash(oid->hash, m->chunk_oid_fanout, - m->chunk_oid_lookup, m->repo->hash_algo->rawsz, + m->chunk_oid_lookup, + m->source->odb->repo->hash_algo->rawsz, result); if (result) *result += m->num_objects_in_base; @@ -559,7 +560,7 @@ struct object_id *nth_midxed_object_oid(struct object_id *oid, n = midx_for_object(&m, n); oidread(oid, m->chunk_oid_lookup + st_mult(m->hash_len, n), - m->repo->hash_algo); + m->source->odb->repo->hash_algo); return oid; } @@ -734,7 +735,7 @@ int prepare_multi_pack_index_one(struct odb_source *source) int midx_checksum_valid(struct multi_pack_index *m) { - return hashfile_checksum_valid(m->repo->hash_algo, + return hashfile_checksum_valid(m->source->odb->repo->hash_algo, m->data, m->data_len); } diff --git a/midx.h b/midx.h index d162001fbbe2b3..71dbdec66ef618 100644 --- a/midx.h +++ b/midx.h @@ -35,6 +35,8 @@ struct odb_source; "GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL" struct multi_pack_index { + struct odb_source *source; + const unsigned char *data; size_t data_len; @@ -50,7 +52,6 @@ struct multi_pack_index { uint32_t num_objects; int preferred_pack_idx; - int local; int has_chain; const unsigned char *chunk_pack_names; @@ -71,10 +72,6 @@ struct multi_pack_index { const char **pack_names; struct packed_git **packs; - - struct repository *repo; - - char object_dir[FLEX_ARRAY]; }; #define MIDX_PROGRESS (1 << 0) diff --git a/pack-bitmap.c b/pack-bitmap.c index fb0b11ca073856..01e14c34bd0cac 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -216,7 +216,7 @@ static uint32_t bitmap_num_objects(struct bitmap_index *index) static struct repository *bitmap_repo(struct bitmap_index *bitmap_git) { if (bitmap_is_midx(bitmap_git)) - return bitmap_git->midx->repo; + return bitmap_git->midx->source->odb->repo; return bitmap_git->pack->repo; } @@ -418,13 +418,13 @@ char *midx_bitmap_filename(struct multi_pack_index *midx) { struct strbuf buf = STRBUF_INIT; if (midx->has_chain) - get_split_midx_filename_ext(midx->repo->hash_algo, &buf, - midx->object_dir, + get_split_midx_filename_ext(midx->source->odb->repo->hash_algo, &buf, + midx->source->path, get_midx_checksum(midx), MIDX_EXT_BITMAP); else - get_midx_filename_ext(midx->repo->hash_algo, &buf, - midx->object_dir, get_midx_checksum(midx), + get_midx_filename_ext(midx->source->odb->repo->hash_algo, &buf, + midx->source->path, get_midx_checksum(midx), MIDX_EXT_BITMAP); return strbuf_detach(&buf, NULL); @@ -463,7 +463,8 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, if (bitmap_git->pack || bitmap_git->midx) { struct strbuf buf = STRBUF_INIT; - get_midx_filename(midx->repo->hash_algo, &buf, midx->object_dir); + get_midx_filename(midx->source->odb->repo->hash_algo, &buf, + midx->source->path); trace2_data_string("bitmap", bitmap_repo(bitmap_git), "ignoring extra midx bitmap file", buf.buf); close(fd); diff --git a/pack-revindex.c b/pack-revindex.c index 0cc422a1e67bc8..b206518dcb51dd 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -379,25 +379,25 @@ int load_midx_revindex(struct multi_pack_index *m) * not want to accidentally call munmap() in the middle of the * MIDX. */ - trace2_data_string("load_midx_revindex", m->repo, + trace2_data_string("load_midx_revindex", m->source->odb->repo, "source", "midx"); m->revindex_data = (const uint32_t *)m->chunk_revindex; return 0; } - trace2_data_string("load_midx_revindex", m->repo, + trace2_data_string("load_midx_revindex", m->source->odb->repo, "source", "rev"); if (m->has_chain) - get_split_midx_filename_ext(m->repo->hash_algo, &revindex_name, - m->object_dir, get_midx_checksum(m), + get_split_midx_filename_ext(m->source->odb->repo->hash_algo, &revindex_name, + m->source->path, get_midx_checksum(m), MIDX_EXT_REV); else - get_midx_filename_ext(m->repo->hash_algo, &revindex_name, - m->object_dir, get_midx_checksum(m), + get_midx_filename_ext(m->source->odb->repo->hash_algo, &revindex_name, + m->source->path, get_midx_checksum(m), MIDX_EXT_REV); - ret = load_revindex_from_disk(m->repo->hash_algo, + ret = load_revindex_from_disk(m->source->odb->repo->hash_algo, revindex_name.buf, m->num_objects, &m->revindex_map, diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index bcb8ea767179a6..6de5d1665afbfc 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -66,7 +66,7 @@ static int read_midx_file(const char *object_dir, const char *checksum, for (i = 0; i < m->num_packs; i++) printf("%s\n", m->pack_names[i]); - printf("object-dir: %s\n", m->object_dir); + printf("object-dir: %s\n", m->source->path); if (show_objects) { struct object_id oid; From 13296ac909d53e14712f89a7f4fda94dd0465479 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:50 +0200 Subject: [PATCH 11/25] midx: compute paths via their source With the preceding commits we started to always have the object database source available when we load, write or access multi-pack indices. With this in place we can change how MIDX paths are computed so that we don't have to pass in the combination of a hash algorithm and object directory anymore, but only the object database source. Refactor the code accordingly. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- midx-write.c | 52 ++++++++++++++++++++++------------------------- midx.c | 54 +++++++++++++++++++++++-------------------------- midx.h | 13 +++++------- pack-bitmap.c | 10 ++++----- pack-revindex.c | 8 ++++---- 5 files changed, 62 insertions(+), 75 deletions(-) diff --git a/midx-write.c b/midx-write.c index 84f76856d67ee9..1dcdf3dc0f151b 100644 --- a/midx-write.c +++ b/midx-write.c @@ -26,9 +26,9 @@ #define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t)) extern int midx_checksum_valid(struct multi_pack_index *m); -extern void clear_midx_files_ext(const char *object_dir, const char *ext, +extern void clear_midx_files_ext(struct odb_source *source, const char *ext, const char *keep_hash); -extern void clear_incremental_midx_files_ext(const char *object_dir, +extern void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, const char **keep_hashes, uint32_t hashes_nr); @@ -112,6 +112,7 @@ struct write_midx_context { struct string_list *to_include; struct repository *repo; + struct odb_source *source; }; static int should_include_pack(const struct write_midx_context *ctx, @@ -648,7 +649,6 @@ static uint32_t *midx_pack_order(struct write_midx_context *ctx) } static void write_midx_reverse_index(struct write_midx_context *ctx, - const char *object_dir, unsigned char *midx_hash) { struct strbuf buf = STRBUF_INIT; @@ -657,11 +657,10 @@ static void write_midx_reverse_index(struct write_midx_context *ctx, trace2_region_enter("midx", "write_midx_reverse_index", ctx->repo); if (ctx->incremental) - get_split_midx_filename_ext(ctx->repo->hash_algo, &buf, - object_dir, midx_hash, - MIDX_EXT_REV); + get_split_midx_filename_ext(ctx->source, &buf, + midx_hash, MIDX_EXT_REV); else - get_midx_filename_ext(ctx->repo->hash_algo, &buf, object_dir, + get_midx_filename_ext(ctx->source, &buf, midx_hash, MIDX_EXT_REV); tmp_file = write_rev_file_order(ctx->repo, NULL, ctx->pack_order, @@ -836,7 +835,6 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr } static int write_midx_bitmap(struct write_midx_context *ctx, - const char *object_dir, const unsigned char *midx_hash, struct packing_data *pdata, struct commit **commits, @@ -852,12 +850,11 @@ static int write_midx_bitmap(struct write_midx_context *ctx, trace2_region_enter("midx", "write_midx_bitmap", ctx->repo); if (ctx->incremental) - get_split_midx_filename_ext(ctx->repo->hash_algo, &bitmap_name, - object_dir, midx_hash, - MIDX_EXT_BITMAP); + get_split_midx_filename_ext(ctx->source, &bitmap_name, + midx_hash, MIDX_EXT_BITMAP); else - get_midx_filename_ext(ctx->repo->hash_algo, &bitmap_name, - object_dir, midx_hash, MIDX_EXT_BITMAP); + get_midx_filename_ext(ctx->source, &bitmap_name, + midx_hash, MIDX_EXT_BITMAP); if (flags & MIDX_WRITE_BITMAP_HASH_CACHE) options |= BITMAP_OPT_HASH_CACHE; @@ -981,11 +978,9 @@ static int link_midx_to_chain(struct multi_pack_index *m) for (i = 0; i < ARRAY_SIZE(midx_exts); i++) { const unsigned char *hash = get_midx_checksum(m); - get_midx_filename_ext(m->source->odb->repo->hash_algo, &from, - m->source->path, + get_midx_filename_ext(m->source, &from, hash, midx_exts[i].non_split); - get_split_midx_filename_ext(m->source->odb->repo->hash_algo, &to, - m->source->path, hash, + get_split_midx_filename_ext(m->source, &to, hash, midx_exts[i].split); if (link(from.buf, to.buf) < 0 && errno != ENOENT) { @@ -1023,16 +1018,16 @@ static void clear_midx_files(struct odb_source *source, uint32_t i, j; for (i = 0; i < ARRAY_SIZE(exts); i++) { - clear_incremental_midx_files_ext(source->path, exts[i], + clear_incremental_midx_files_ext(source, exts[i], hashes, hashes_nr); for (j = 0; j < hashes_nr; j++) - clear_midx_files_ext(source->path, exts[i], hashes[j]); + clear_midx_files_ext(source, exts[i], hashes[j]); } if (incremental) - get_midx_filename(source->odb->repo->hash_algo, &buf, source->path); + get_midx_filename(source, &buf); else - get_midx_chain_filename(&buf, source->path); + get_midx_chain_filename(source, &buf); if (unlink(buf.buf) && errno != ENOENT) die_errno(_("failed to clear multi-pack-index at %s"), buf.buf); @@ -1065,6 +1060,7 @@ static int write_midx_internal(struct odb_source *source, trace2_region_enter("midx", "write_midx_internal", r); ctx.repo = r; + ctx.source = source; ctx.incremental = !!(flags & MIDX_WRITE_INCREMENTAL); @@ -1073,7 +1069,7 @@ static int write_midx_internal(struct odb_source *source, "%s/pack/multi-pack-index.d/tmp_midx_XXXXXX", source->path); else - get_midx_filename(r->hash_algo, &midx_name, source->path); + get_midx_filename(source, &midx_name); if (safe_create_leading_directories(r, midx_name.buf)) die_errno(_("unable to create leading directories of %s"), midx_name.buf); @@ -1153,7 +1149,7 @@ static int write_midx_internal(struct odb_source *source, * corresponding bitmap (or one wasn't requested). */ if (!want_bitmap) - clear_midx_files_ext(source->path, "bitmap", NULL); + clear_midx_files_ext(source, "bitmap", NULL); goto cleanup; } } @@ -1321,7 +1317,7 @@ static int write_midx_internal(struct odb_source *source, if (ctx.incremental) { struct strbuf lock_name = STRBUF_INIT; - get_midx_chain_filename(&lock_name, source->path); + get_midx_chain_filename(source, &lock_name); hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR); strbuf_release(&lock_name); @@ -1384,7 +1380,7 @@ static int write_midx_internal(struct odb_source *source, if (flags & MIDX_WRITE_REV_INDEX && git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0)) - write_midx_reverse_index(&ctx, source->path, midx_hash); + write_midx_reverse_index(&ctx, midx_hash); if (flags & MIDX_WRITE_BITMAP) { struct packing_data pdata; @@ -1407,7 +1403,7 @@ static int write_midx_internal(struct odb_source *source, FREE_AND_NULL(ctx.entries); ctx.entries_nr = 0; - if (write_midx_bitmap(&ctx, source->path, + if (write_midx_bitmap(&ctx, midx_hash, &pdata, commits, commits_nr, flags) < 0) { error(_("could not write multi-pack bitmap")); @@ -1440,8 +1436,8 @@ static int write_midx_internal(struct odb_source *source, if (link_midx_to_chain(ctx.base_midx) < 0) return -1; - get_split_midx_filename_ext(r->hash_algo, &final_midx_name, - source->path, midx_hash, MIDX_EXT_MIDX); + get_split_midx_filename_ext(source, &final_midx_name, + midx_hash, MIDX_EXT_MIDX); if (rename_tempfile(&incr, final_midx_name.buf) < 0) { error_errno(_("unable to rename new multi-pack-index layer")); diff --git a/midx.c b/midx.c index 81bf3c4d5f351e..7726c13d7e7bc0 100644 --- a/midx.c +++ b/midx.c @@ -16,9 +16,9 @@ #define MIDX_PACK_ERROR ((void *)(intptr_t)-1) int midx_checksum_valid(struct multi_pack_index *m); -void clear_midx_files_ext(const char *object_dir, const char *ext, +void clear_midx_files_ext(struct odb_source *source, const char *ext, const char *keep_hash); -void clear_incremental_midx_files_ext(const char *object_dir, const char *ext, +void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, char **keep_hashes, uint32_t hashes_nr); int cmp_idx_or_pack_name(const char *idx_or_pack_name, @@ -29,19 +29,17 @@ const unsigned char *get_midx_checksum(struct multi_pack_index *m) return m->data + m->data_len - m->source->odb->repo->hash_algo->rawsz; } -void get_midx_filename(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir) +void get_midx_filename(struct odb_source *source, struct strbuf *out) { - get_midx_filename_ext(hash_algo, out, object_dir, NULL, NULL); + get_midx_filename_ext(source, out, NULL, NULL); } -void get_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir, +void get_midx_filename_ext(struct odb_source *source, struct strbuf *out, const unsigned char *hash, const char *ext) { - strbuf_addf(out, "%s/pack/multi-pack-index", object_dir); + strbuf_addf(out, "%s/pack/multi-pack-index", source->path); if (ext) - strbuf_addf(out, "-%s.%s", hash_to_hex_algop(hash, hash_algo), ext); + strbuf_addf(out, "-%s.%s", hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext); } static int midx_read_oid_fanout(const unsigned char *chunk_start, @@ -222,24 +220,23 @@ static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *sou return NULL; } -void get_midx_chain_dirname(struct strbuf *buf, const char *object_dir) +void get_midx_chain_dirname(struct odb_source *source, struct strbuf *buf) { - strbuf_addf(buf, "%s/pack/multi-pack-index.d", object_dir); + strbuf_addf(buf, "%s/pack/multi-pack-index.d", source->path); } -void get_midx_chain_filename(struct strbuf *buf, const char *object_dir) +void get_midx_chain_filename(struct odb_source *source, struct strbuf *buf) { - get_midx_chain_dirname(buf, object_dir); + get_midx_chain_dirname(source, buf); strbuf_addstr(buf, "/multi-pack-index-chain"); } -void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *buf, const char *object_dir, +void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf, const unsigned char *hash, const char *ext) { - get_midx_chain_dirname(buf, object_dir); + get_midx_chain_dirname(source, buf); strbuf_addf(buf, "/multi-pack-index-%s.%s", - hash_to_hex_algop(hash, hash_algo), ext); + hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext); } static int open_multi_pack_index_chain(const struct git_hash_algo *hash_algo, @@ -326,7 +323,7 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source, valid = 0; strbuf_reset(&buf); - get_split_midx_filename_ext(hash_algo, &buf, source->path, + get_split_midx_filename_ext(source, &buf, layer.hash, MIDX_EXT_MIDX); m = load_multi_pack_index_one(source, buf.buf); @@ -358,7 +355,7 @@ static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *s int fd; struct multi_pack_index *m = NULL; - get_midx_chain_filename(&chain_file, source->path); + get_midx_chain_filename(source, &chain_file); if (open_multi_pack_index_chain(source->odb->repo->hash_algo, chain_file.buf, &fd, &st)) { int incomplete; /* ownership of fd is taken over by load function */ @@ -374,8 +371,7 @@ struct multi_pack_index *load_multi_pack_index(struct odb_source *source) struct strbuf midx_name = STRBUF_INIT; struct multi_pack_index *m; - get_midx_filename(source->odb->repo->hash_algo, &midx_name, - source->path); + get_midx_filename(source, &midx_name); m = load_multi_pack_index_one(source, midx_name.buf); if (!m) @@ -762,7 +758,7 @@ static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUS die_errno(_("failed to remove %s"), full_path); } -void clear_midx_files_ext(const char *object_dir, const char *ext, +void clear_midx_files_ext(struct odb_source *source, const char *ext, const char *keep_hash) { struct clear_midx_data data; @@ -776,7 +772,7 @@ void clear_midx_files_ext(const char *object_dir, const char *ext, } data.ext = ext; - for_each_file_in_pack_dir(object_dir, + for_each_file_in_pack_dir(source->path, clear_midx_file_ext, &data); @@ -785,7 +781,7 @@ void clear_midx_files_ext(const char *object_dir, const char *ext, free(data.keep); } -void clear_incremental_midx_files_ext(const char *object_dir, const char *ext, +void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, char **keep_hashes, uint32_t hashes_nr) { @@ -801,7 +797,7 @@ void clear_incremental_midx_files_ext(const char *object_dir, const char *ext, data.keep_nr = hashes_nr; data.ext = ext; - for_each_file_in_pack_subdir(object_dir, "multi-pack-index.d", + for_each_file_in_pack_subdir(source->path, "multi-pack-index.d", clear_midx_file_ext, &data); for (i = 0; i < hashes_nr; i++) @@ -813,7 +809,7 @@ void clear_midx_file(struct repository *r) { struct strbuf midx = STRBUF_INIT; - get_midx_filename(r->hash_algo, &midx, r->objects->sources->path); + get_midx_filename(r->objects->sources, &midx); if (r->objects) { struct odb_source *source; @@ -828,8 +824,8 @@ void clear_midx_file(struct repository *r) if (remove_path(midx.buf)) die(_("failed to clear multi-pack-index at %s"), midx.buf); - clear_midx_files_ext(r->objects->sources->path, MIDX_EXT_BITMAP, NULL); - clear_midx_files_ext(r->objects->sources->path, MIDX_EXT_REV, NULL); + clear_midx_files_ext(r->objects->sources, MIDX_EXT_BITMAP, NULL); + clear_midx_files_ext(r->objects->sources, MIDX_EXT_REV, NULL); strbuf_release(&midx); } @@ -888,7 +884,7 @@ int verify_midx_file(struct odb_source *source, unsigned flags) struct stat sb; struct strbuf filename = STRBUF_INIT; - get_midx_filename(r->hash_algo, &filename, source->path); + get_midx_filename(source, &filename); if (!stat(filename.buf, &sb)) { error(_("multi-pack-index file exists, but failed to parse")); diff --git a/midx.h b/midx.h index 71dbdec66ef618..e241d2d6900bc3 100644 --- a/midx.h +++ b/midx.h @@ -86,15 +86,12 @@ struct multi_pack_index { #define MIDX_EXT_MIDX "midx" const unsigned char *get_midx_checksum(struct multi_pack_index *m); -void get_midx_filename(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir); -void get_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir, +void get_midx_filename(struct odb_source *source, struct strbuf *out); +void get_midx_filename_ext(struct odb_source *source, struct strbuf *out, const unsigned char *hash, const char *ext); -void get_midx_chain_dirname(struct strbuf *buf, const char *object_dir); -void get_midx_chain_filename(struct strbuf *buf, const char *object_dir); -void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *buf, const char *object_dir, +void get_midx_chain_dirname(struct odb_source *source, struct strbuf *out); +void get_midx_chain_filename(struct odb_source *source, struct strbuf *out); +void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf, const unsigned char *hash, const char *ext); struct multi_pack_index *load_multi_pack_index(struct odb_source *source); diff --git a/pack-bitmap.c b/pack-bitmap.c index 01e14c34bd0cac..058bdb5d7ded0b 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -418,13 +418,12 @@ char *midx_bitmap_filename(struct multi_pack_index *midx) { struct strbuf buf = STRBUF_INIT; if (midx->has_chain) - get_split_midx_filename_ext(midx->source->odb->repo->hash_algo, &buf, - midx->source->path, + get_split_midx_filename_ext(midx->source, &buf, get_midx_checksum(midx), MIDX_EXT_BITMAP); else - get_midx_filename_ext(midx->source->odb->repo->hash_algo, &buf, - midx->source->path, get_midx_checksum(midx), + get_midx_filename_ext(midx->source, &buf, + get_midx_checksum(midx), MIDX_EXT_BITMAP); return strbuf_detach(&buf, NULL); @@ -463,8 +462,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, if (bitmap_git->pack || bitmap_git->midx) { struct strbuf buf = STRBUF_INIT; - get_midx_filename(midx->source->odb->repo->hash_algo, &buf, - midx->source->path); + get_midx_filename(midx->source, &buf); trace2_data_string("bitmap", bitmap_repo(bitmap_git), "ignoring extra midx bitmap file", buf.buf); close(fd); diff --git a/pack-revindex.c b/pack-revindex.c index b206518dcb51dd..d0791cc4938fa2 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -389,12 +389,12 @@ int load_midx_revindex(struct multi_pack_index *m) "source", "rev"); if (m->has_chain) - get_split_midx_filename_ext(m->source->odb->repo->hash_algo, &revindex_name, - m->source->path, get_midx_checksum(m), + get_split_midx_filename_ext(m->source, &revindex_name, + get_midx_checksum(m), MIDX_EXT_REV); else - get_midx_filename_ext(m->source->odb->repo->hash_algo, &revindex_name, - m->source->path, get_midx_checksum(m), + get_midx_filename_ext(m->source, &revindex_name, + get_midx_checksum(m), MIDX_EXT_REV); ret = load_revindex_from_disk(m->source->odb->repo->hash_algo, From d14147c0ab84bf4d08adedb4d1a4e99511c56375 Mon Sep 17 00:00:00 2001 From: Julia Evans Date: Tue, 19 Aug 2025 20:46:09 +0000 Subject: [PATCH 12/25] doc: git-add: clarify intro & add an example - Add a basic example of how "git add" is normally used - It's not technically true that you *must* use the `add` command to add changes before running `git commit`, because `git commit -a` exists. Instead say that you *can* use the `add` command. - Mention early on that "index" is another word for "staging area", since Git very rarely uses the word "index" in its output (`git status`) uses the term "staged", and many Git users are unfamiliar with the term "index" - Remove "It typically adds" (it's not clear what "typically" means), and instead mention that `git add -p` can be used to add partial contents - Currently the introduction is somewhat repetitive ("to prepare the content staged for the next commit" ... "this snapshot that is taken as the contents of the next commit."), replace with a single sentence ("The "index" [...] is where Git stores the contents of the next commit.") Signed-off-by: Julia Evans Signed-off-by: Junio C Hamano --- Documentation/git-add.adoc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Documentation/git-add.adoc b/Documentation/git-add.adoc index b7a735824d6ce0..19f99b0e7f6f09 100644 --- a/Documentation/git-add.adoc +++ b/Documentation/git-add.adoc @@ -16,18 +16,18 @@ git add [--verbose | -v] [--dry-run | -n] [--force | -f] [--interactive | -i] [- DESCRIPTION ----------- -This command updates the index using the current content found in -the working tree, to prepare the content staged for the next commit. -It typically adds the current content of existing paths as a whole, -but with some options it can also be used to add content with -only part of the changes made to the working tree files applied, or -remove paths that do not exist in the working tree anymore. - -The "index" holds a snapshot of the content of the working tree, and it -is this snapshot that is taken as the contents of the next commit. Thus -after making any changes to the working tree, and before running -the commit command, you must use the `add` command to add any new or -modified files to the index. +Add contents of new or changed files to the index. The "index" (also +known as "staging area") is where Git stores the contents of the next +commit. + +When you run `git commit` without any other arguments, it will only +commit staged changes. For example, if you've edited `file.c` and want +to commit your changes to that file, you can run: + + git add file.c + git commit + +You can also add only part of your changes to a file with `git add -p`. This command can be performed multiple times before a commit. It only adds the content of the specified file(s) at the time the add command is From 929e112481b4632a6664d7b583bd99b8c590eb1a Mon Sep 17 00:00:00 2001 From: Julia Evans Date: Tue, 19 Aug 2025 20:46:10 +0000 Subject: [PATCH 13/25] doc: git-add: simplify discussion of ignored files - Mention the --force option earlier - Remove the explanation of shell globbing vs git's internal glob system, since users are confused by it and there's a clearer discussion in the EXAMPLES section. Signed-off-by: Julia Evans Signed-off-by: Junio C Hamano --- Documentation/git-add.adoc | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Documentation/git-add.adoc b/Documentation/git-add.adoc index 19f99b0e7f6f09..bf793d28949339 100644 --- a/Documentation/git-add.adoc +++ b/Documentation/git-add.adoc @@ -37,12 +37,10 @@ you must run `git add` again to add the new content to the index. The `git status` command can be used to obtain a summary of which files have changes that are staged for the next commit. -The `git add` command will not add ignored files by default. If any -ignored files were explicitly specified on the command line, `git add` -will fail with a list of ignored files. Ignored files reached by -directory recursion or filename globbing performed by Git (quote your -globs before the shell) will be silently ignored. The `git add` command can -be used to add ignored files with the `-f` (force) option. +The `git add` command will not add ignored files by default. You can +use the `--force` option to add ignored files. If you specify the exact +filename of an ignored file, `git add` will fail with a list of ignored +files. Otherwise it will silently ignore the file. Please see linkgit:git-commit[1] for alternative ways to add content to a commit. From e5c27bd3d82c558f4f8ced1f61c28a466232ee21 Mon Sep 17 00:00:00 2001 From: Julia Evans Date: Fri, 29 Aug 2025 11:55:02 +0000 Subject: [PATCH 14/25] doc: rephrase the purpose of the staging area Git does not really "store the contents of the next commit" anywhere; rather, you the user use the index to prepare it. Signed-off-by: Julia Evans [jc; made the change relative to what is already in 'next'] Signed-off-by: Junio C Hamano --- Documentation/git-add.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/git-add.adoc b/Documentation/git-add.adoc index bf793d28949339..ad629c46c5f39a 100644 --- a/Documentation/git-add.adoc +++ b/Documentation/git-add.adoc @@ -17,8 +17,8 @@ git add [--verbose | -v] [--dry-run | -n] [--force | -f] [--interactive | -i] [- DESCRIPTION ----------- Add contents of new or changed files to the index. The "index" (also -known as "staging area") is where Git stores the contents of the next -commit. +known as the "staging area") is what you use to prepare the contents of +the next commit. When you run `git commit` without any other arguments, it will only commit staged changes. For example, if you've edited `file.c` and want From 2f4bf83ffc4ffd0a9ed13121d712beb4c1b9f276 Mon Sep 17 00:00:00 2001 From: "Kyle E. Mitchell" Date: Mon, 1 Sep 2025 11:04:18 -0700 Subject: [PATCH 15/25] doc: fix formatting of function-wrap shell alias MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a missed backtick to the end of a code segment so that it will be rendered like preceding examples. I deeply appreciate the thoroughness of this documentation. I noticed the formatting discrepancy reading https://git-scm.com/docs/git-config. Signed-off-by: Kyle E. Mitchell Acked-by: Jean-Noël AVILA Signed-off-by: Junio C Hamano --- Documentation/config/alias.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/config/alias.adoc b/Documentation/config/alias.adoc index 2c5db0ad8428e8..95825354bf1cdd 100644 --- a/Documentation/config/alias.adoc +++ b/Documentation/config/alias.adoc @@ -38,6 +38,6 @@ it will be treated as a shell command. For example, defining ** A convenient way to deal with this is to write your script operations in an inline function that is then called with any arguments from the command-line. For example `alias.cmd = "!c() { - echo $1 | grep $2 ; }; c" will correctly execute the prior example. + echo $1 | grep $2 ; }; c"` will correctly execute the prior example. ** Setting `GIT_TRACE=1` can help you debug the command being run for your alias. From 608cf5b7935e1d03d1a8854a928e9bf8b30835be Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 2 Sep 2025 12:47:41 +0200 Subject: [PATCH 16/25] gitlab-ci: disable realtime monitoring to unbreak Windows jobs The GitLab CI runners using Windows machines have realtime monitoring via Windows Defender enabled by default. This has just now started to cause issues in our CI jobs using Microsoft Visual Studio: Program 'meson.exe' failed to run: Operation did not complete successfully because the file contains a virus or potentially unwanted softwareAt line:356 char:1 + meson setup build --vsenv -Dperl=disabled -Dbackend_max_links=1 -Dcre ... + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~. At line:356 char:1 + meson setup build --vsenv -Dperl=disabled -Dbackend_max_links=1 -Dcre ... + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + CategoryInfo : ResourceUnavailable: (:) [], ApplicationFailedException + FullyQualifiedErrorId : NativeCommandFailed The detected issue is more likely than not completely bogus, but it breaks the jobs. Fix the issue by disabling realtime monitoring. Besides unbreaking CI, it also improves our build times a bit: - Building Git goes from 26 to 22 minutes. - Executing tests goes from ~1h for one slice of tests to ~30 minutes. This is still painfully slow, but the issue here is that the Windows runners on GitLab CI are quite underwhelming overall. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb6d5b976cdc36..ef57798e2b0afb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,6 +119,7 @@ build:mingw64: variables: NO_PERL: 1 before_script: + - Set-MpPreference -DisableRealtimeMonitoring $true - ./ci/install-sdk.ps1 -directory "git-sdk" script: - git-sdk/usr/bin/bash.exe -l -c 'ci/make-test-artifacts.sh artifacts' @@ -135,6 +136,7 @@ test:mingw64: - job: "build:mingw64" artifacts: true before_script: + - Set-MpPreference -DisableRealtimeMonitoring $true - git-sdk/usr/bin/bash.exe -l -c 'tar xf artifacts/artifacts.tar.gz' - New-Item -Path .git/info -ItemType Directory - New-Item .git/info/exclude -ItemType File -Value "/git-sdk" @@ -148,6 +150,7 @@ test:mingw64: tags: - saas-windows-medium-amd64 before_script: + - Set-MpPreference -DisableRealtimeMonitoring $true - choco install -y git meson ninja openssl - Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 - refreshenv From 0f0a8a11c00295ed30b02cc721b0994900c6a3d3 Mon Sep 17 00:00:00 2001 From: Meet Soni Date: Tue, 26 Aug 2025 12:11:07 +0530 Subject: [PATCH 17/25] builtin/refs: add 'exists' subcommand As part of the ongoing effort to consolidate reference handling, introduce a new `exists` subcommand. This command provides the same functionality and exit-code behavior as `git show-ref --exists`, serving as its modern replacement. The logic for `show-ref --exists` is minimal. Rather than creating a shared helper function which would be overkill for ~20 lines of code, its implementation is intentionally duplicated here. This contrasts with `git refs list`, where sharing the larger implementation of `for-each-ref` was necessary. Documentation for the new subcommand is also added to the `git-refs(1)` man page. Mentored-by: Patrick Steinhardt Mentored-by: shejialuo Signed-off-by: Meet Soni Acked-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- Documentation/git-refs.adoc | 7 ++++++ builtin/refs.c | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/Documentation/git-refs.adoc b/Documentation/git-refs.adoc index e608980711814a..5d2032b318a224 100644 --- a/Documentation/git-refs.adoc +++ b/Documentation/git-refs.adoc @@ -18,6 +18,7 @@ git refs list [--count=] [--shell|--perl|--python|--tcl] [--contains[=]] [--no-contains[=]] [(--exclude=)...] [--start-after=] [ --stdin | ... ] +git refs exists DESCRIPTION ----------- @@ -38,6 +39,12 @@ list:: formatting, and sorting. This subcommand is an alias for linkgit:git-for-each-ref[1] and offers identical functionality. +exists:: + Check whether the given reference exists. Returns an exit code of 0 if + it does, 2 if it is missing, and 1 in case looking up the reference + failed with an error other than the reference being missing. This does + not verify whether the reference resolves to an actual object. + OPTIONS ------- diff --git a/builtin/refs.c b/builtin/refs.c index 76224feba4d55a..91548783b75b58 100644 --- a/builtin/refs.c +++ b/builtin/refs.c @@ -7,6 +7,7 @@ #include "strbuf.h" #include "worktree.h" #include "for-each-ref.h" +#include "refs/refs-internal.h" #define REFS_MIGRATE_USAGE \ N_("git refs migrate --ref-format= [--no-reflog] [--dry-run]") @@ -14,6 +15,9 @@ #define REFS_VERIFY_USAGE \ N_("git refs verify [--strict] [--verbose]") +#define REFS_EXISTS_USAGE \ + N_("git refs exists ") + static int cmd_refs_migrate(int argc, const char **argv, const char *prefix, struct repository *repo UNUSED) { @@ -113,6 +117,48 @@ static int cmd_refs_list(int argc, const char **argv, const char *prefix, return for_each_ref_core(argc, argv, prefix, repo, refs_list_usage); } +static int cmd_refs_exists(int argc, const char **argv, const char *prefix, + struct repository *repo UNUSED) +{ + struct strbuf unused_referent = STRBUF_INIT; + struct object_id unused_oid; + unsigned int unused_type; + int failure_errno = 0; + const char *ref; + int ret = 0; + const char * const exists_usage[] = { + REFS_EXISTS_USAGE, + NULL, + }; + struct option options[] = { + OPT_END(), + }; + + argc = parse_options(argc, argv, prefix, options, exists_usage, 0); + if (argc != 1) + die(_("'git refs exists' requires a reference")); + + ref = *argv++; + if (refs_read_raw_ref(get_main_ref_store(the_repository), ref, + &unused_oid, &unused_referent, &unused_type, + &failure_errno)) { + if (failure_errno == ENOENT || failure_errno == EISDIR) { + error(_("reference does not exist")); + ret = 2; + } else { + errno = failure_errno; + error_errno(_("failed to look up reference")); + ret = 1; + } + + goto out; + } + +out: + strbuf_release(&unused_referent); + return ret; +} + int cmd_refs(int argc, const char **argv, const char *prefix, @@ -122,6 +168,7 @@ int cmd_refs(int argc, REFS_MIGRATE_USAGE, REFS_VERIFY_USAGE, "git refs list " COMMON_USAGE_FOR_EACH_REF, + REFS_EXISTS_USAGE, NULL, }; parse_opt_subcommand_fn *fn = NULL; @@ -129,6 +176,7 @@ int cmd_refs(int argc, OPT_SUBCOMMAND("migrate", &fn, cmd_refs_migrate), OPT_SUBCOMMAND("verify", &fn, cmd_refs_verify), OPT_SUBCOMMAND("list", &fn, cmd_refs_list), + OPT_SUBCOMMAND("exists", &fn, cmd_refs_exists), OPT_END(), }; From 0749b93ab369cda5e316da1c2a87d325ce02fe1d Mon Sep 17 00:00:00 2001 From: Meet Soni Date: Tue, 26 Aug 2025 12:11:08 +0530 Subject: [PATCH 18/25] t1403: split 'show-ref --exists' tests into a separate file The test file for git-show-ref(1), `t1403-show-ref.sh`, contains a group of tests for the '--exists' flag. To improve organization and to prepare for refactoring these tests to be shareable, move the '--exists' tests and their corresponding setup logic into a self-contained test suite, `t1422-show-ref-exists.sh`. This is a pure code-movement refactoring with no change in test coverage or behavior. Mentored-by: Patrick Steinhardt Mentored-by: shejialuo Signed-off-by: Meet Soni Acked-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- t/meson.build | 3 +- t/t1403-show-ref.sh | 65 ----------------------------- t/t1422-show-ref-exists.sh | 83 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 66 deletions(-) create mode 100644 t/t1422-show-ref-exists.sh diff --git a/t/meson.build b/t/meson.build index daf01fb5d0a290..4d6bc3d38e28a1 100644 --- a/t/meson.build +++ b/t/meson.build @@ -205,6 +205,7 @@ integration_tests = [ 't1419-exclude-refs.sh', 't1420-lost-found.sh', 't1421-reflog-write.sh', + 't1422-show-ref-exists.sh', 't1430-bad-ref-name.sh', 't1450-fsck.sh', 't1451-fsck-buffer.sh', @@ -1216,4 +1217,4 @@ if perl.found() and time.found() timeout: 0, ) endforeach -endif \ No newline at end of file +endif diff --git a/t/t1403-show-ref.sh b/t/t1403-show-ref.sh index 9da3650e91bac5..36c903ca1901bf 100755 --- a/t/t1403-show-ref.sh +++ b/t/t1403-show-ref.sh @@ -228,69 +228,4 @@ test_expect_success 'show-ref sub-modes are mutually exclusive' ' grep "cannot be used together" err ' -test_expect_success '--exists with existing reference' ' - git show-ref --exists refs/heads/$GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME -' - -test_expect_success '--exists with missing reference' ' - test_expect_code 2 git show-ref --exists refs/heads/does-not-exist -' - -test_expect_success '--exists does not use DWIM' ' - test_expect_code 2 git show-ref --exists $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME 2>err && - grep "reference does not exist" err -' - -test_expect_success '--exists with HEAD' ' - git show-ref --exists HEAD -' - -test_expect_success '--exists with bad reference name' ' - test_when_finished "git update-ref -d refs/heads/bad...name" && - new_oid=$(git rev-parse HEAD) && - test-tool ref-store main update-ref msg refs/heads/bad...name $new_oid $ZERO_OID REF_SKIP_REFNAME_VERIFICATION && - git show-ref --exists refs/heads/bad...name -' - -test_expect_success '--exists with arbitrary symref' ' - test_when_finished "git symbolic-ref -d refs/symref" && - git symbolic-ref refs/symref refs/heads/$GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME && - git show-ref --exists refs/symref -' - -test_expect_success '--exists with dangling symref' ' - test_when_finished "git symbolic-ref -d refs/heads/dangling" && - git symbolic-ref refs/heads/dangling refs/heads/does-not-exist && - git show-ref --exists refs/heads/dangling -' - -test_expect_success '--exists with nonexistent object ID' ' - test-tool ref-store main update-ref msg refs/heads/missing-oid $(test_oid 001) $ZERO_OID REF_SKIP_OID_VERIFICATION && - git show-ref --exists refs/heads/missing-oid -' - -test_expect_success '--exists with non-commit object' ' - tree_oid=$(git rev-parse HEAD^{tree}) && - test-tool ref-store main update-ref msg refs/heads/tree ${tree_oid} $ZERO_OID REF_SKIP_OID_VERIFICATION && - git show-ref --exists refs/heads/tree -' - -test_expect_success '--exists with directory fails with generic error' ' - cat >expect <<-EOF && - error: reference does not exist - EOF - test_expect_code 2 git show-ref --exists refs/heads 2>err && - test_cmp expect err -' - -test_expect_success '--exists with non-existent special ref' ' - test_expect_code 2 git show-ref --exists FETCH_HEAD -' - -test_expect_success '--exists with existing special ref' ' - test_when_finished "rm .git/FETCH_HEAD" && - git rev-parse HEAD >.git/FETCH_HEAD && - git show-ref --exists FETCH_HEAD -' - test_done diff --git a/t/t1422-show-ref-exists.sh b/t/t1422-show-ref-exists.sh new file mode 100644 index 00000000000000..0eccb2dce114db --- /dev/null +++ b/t/t1422-show-ref-exists.sh @@ -0,0 +1,83 @@ +#!/bin/sh + +test_description='show-ref --exists' +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +test_expect_success setup ' + test_commit --annotate A && + git checkout -b side && + test_commit --annotate B && + git checkout main && + test_commit C && + git branch B A^0 +' + +test_expect_success '--exists with existing reference' ' + git show-ref --exists refs/heads/$GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +' + +test_expect_success '--exists with missing reference' ' + test_expect_code 2 git show-ref --exists refs/heads/does-not-exist +' + +test_expect_success '--exists does not use DWIM' ' + test_expect_code 2 git show-ref --exists $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME 2>err && + grep "reference does not exist" err +' + +test_expect_success '--exists with HEAD' ' + git show-ref --exists HEAD +' + +test_expect_success '--exists with bad reference name' ' + test_when_finished "git update-ref -d refs/heads/bad...name" && + new_oid=$(git rev-parse HEAD) && + test-tool ref-store main update-ref msg refs/heads/bad...name $new_oid $ZERO_OID REF_SKIP_REFNAME_VERIFICATION && + git show-ref --exists refs/heads/bad...name +' + +test_expect_success '--exists with arbitrary symref' ' + test_when_finished "git symbolic-ref -d refs/symref" && + git symbolic-ref refs/symref refs/heads/$GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME && + git show-ref --exists refs/symref +' + +test_expect_success '--exists with dangling symref' ' + test_when_finished "git symbolic-ref -d refs/heads/dangling" && + git symbolic-ref refs/heads/dangling refs/heads/does-not-exist && + git show-ref --exists refs/heads/dangling +' + +test_expect_success '--exists with nonexistent object ID' ' + test-tool ref-store main update-ref msg refs/heads/missing-oid $(test_oid 001) $ZERO_OID REF_SKIP_OID_VERIFICATION && + git show-ref --exists refs/heads/missing-oid +' + +test_expect_success '--exists with non-commit object' ' + tree_oid=$(git rev-parse HEAD^{tree}) && + test-tool ref-store main update-ref msg refs/heads/tree ${tree_oid} $ZERO_OID REF_SKIP_OID_VERIFICATION && + git show-ref --exists refs/heads/tree +' + +test_expect_success '--exists with directory fails with generic error' ' + cat >expect <<-EOF && + error: reference does not exist + EOF + test_expect_code 2 git show-ref --exists refs/heads 2>err && + test_cmp expect err +' + +test_expect_success '--exists with non-existent special ref' ' + test_expect_code 2 git show-ref --exists FETCH_HEAD +' + +test_expect_success '--exists with existing special ref' ' + test_when_finished "rm .git/FETCH_HEAD" && + git rev-parse HEAD >.git/FETCH_HEAD && + git show-ref --exists FETCH_HEAD +' + +test_done From 01d429c7bfe6bc494ca43476dfc08cec0ad90a4a Mon Sep 17 00:00:00 2001 From: Meet Soni Date: Tue, 26 Aug 2025 12:11:09 +0530 Subject: [PATCH 19/25] t1422: refactor tests to be shareable In preparation for adding tests for the `git refs exists` command, refactor the existing t1422 test suite to make its logic shareable. Move the core test logic from `t1422-show-ref-exists.sh` to `show-ref-exists-tests.sh` file. Inside this script, replace hardcoded calls to "git show-ref --exists" with the `$git_show_ref_exists` variable. The original `t1422-show-ref-exists.sh` script now becomes a simple "driver". It is responsible for setting the default value of the variable and then sourcing the test library. This structure follows an established pattern for sharing tests and prepares the test suite for the `refs exists` tests to be added in a subsequent commit. Mentored-by: Patrick Steinhardt Mentored-by: shejialuo Signed-off-by: Meet Soni Acked-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- t/show-ref-exists-tests.sh | 77 ++++++++++++++++++++++++++++++++++++++ t/t1422-show-ref-exists.sh | 76 +------------------------------------ 2 files changed, 78 insertions(+), 75 deletions(-) create mode 100644 t/show-ref-exists-tests.sh mode change 100644 => 100755 t/t1422-show-ref-exists.sh diff --git a/t/show-ref-exists-tests.sh b/t/show-ref-exists-tests.sh new file mode 100644 index 00000000000000..36e8e9df33ac37 --- /dev/null +++ b/t/show-ref-exists-tests.sh @@ -0,0 +1,77 @@ +git_show_ref_exists=${git_show_ref_exists:-git show-ref --exists} + +test_expect_success setup ' + test_commit --annotate A && + git checkout -b side && + test_commit --annotate B && + git checkout main && + test_commit C && + git branch B A^0 +' + +test_expect_success '--exists with existing reference' ' + ${git_show_ref_exists} refs/heads/$GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +' + +test_expect_success '--exists with missing reference' ' + test_expect_code 2 ${git_show_ref_exists} refs/heads/does-not-exist +' + +test_expect_success '--exists does not use DWIM' ' + test_expect_code 2 ${git_show_ref_exists} $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME 2>err && + grep "reference does not exist" err +' + +test_expect_success '--exists with HEAD' ' + ${git_show_ref_exists} HEAD +' + +test_expect_success '--exists with bad reference name' ' + test_when_finished "git update-ref -d refs/heads/bad...name" && + new_oid=$(git rev-parse HEAD) && + test-tool ref-store main update-ref msg refs/heads/bad...name $new_oid $ZERO_OID REF_SKIP_REFNAME_VERIFICATION && + ${git_show_ref_exists} refs/heads/bad...name +' + +test_expect_success '--exists with arbitrary symref' ' + test_when_finished "git symbolic-ref -d refs/symref" && + git symbolic-ref refs/symref refs/heads/$GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME && + ${git_show_ref_exists} refs/symref +' + +test_expect_success '--exists with dangling symref' ' + test_when_finished "git symbolic-ref -d refs/heads/dangling" && + git symbolic-ref refs/heads/dangling refs/heads/does-not-exist && + ${git_show_ref_exists} refs/heads/dangling +' + +test_expect_success '--exists with nonexistent object ID' ' + test-tool ref-store main update-ref msg refs/heads/missing-oid $(test_oid 001) $ZERO_OID REF_SKIP_OID_VERIFICATION && + ${git_show_ref_exists} refs/heads/missing-oid +' + +test_expect_success '--exists with non-commit object' ' + tree_oid=$(git rev-parse HEAD^{tree}) && + test-tool ref-store main update-ref msg refs/heads/tree ${tree_oid} $ZERO_OID REF_SKIP_OID_VERIFICATION && + ${git_show_ref_exists} refs/heads/tree +' + +test_expect_success '--exists with directory fails with generic error' ' + cat >expect <<-EOF && + error: reference does not exist + EOF + test_expect_code 2 ${git_show_ref_exists} refs/heads 2>err && + test_cmp expect err +' + +test_expect_success '--exists with non-existent special ref' ' + test_expect_code 2 ${git_show_ref_exists} FETCH_HEAD +' + +test_expect_success '--exists with existing special ref' ' + test_when_finished "rm .git/FETCH_HEAD" && + git rev-parse HEAD >.git/FETCH_HEAD && + ${git_show_ref_exists} FETCH_HEAD +' + +test_done diff --git a/t/t1422-show-ref-exists.sh b/t/t1422-show-ref-exists.sh old mode 100644 new mode 100755 index 0eccb2dce114db..fdca3f16c8b9fb --- a/t/t1422-show-ref-exists.sh +++ b/t/t1422-show-ref-exists.sh @@ -6,78 +6,4 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh -test_expect_success setup ' - test_commit --annotate A && - git checkout -b side && - test_commit --annotate B && - git checkout main && - test_commit C && - git branch B A^0 -' - -test_expect_success '--exists with existing reference' ' - git show-ref --exists refs/heads/$GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME -' - -test_expect_success '--exists with missing reference' ' - test_expect_code 2 git show-ref --exists refs/heads/does-not-exist -' - -test_expect_success '--exists does not use DWIM' ' - test_expect_code 2 git show-ref --exists $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME 2>err && - grep "reference does not exist" err -' - -test_expect_success '--exists with HEAD' ' - git show-ref --exists HEAD -' - -test_expect_success '--exists with bad reference name' ' - test_when_finished "git update-ref -d refs/heads/bad...name" && - new_oid=$(git rev-parse HEAD) && - test-tool ref-store main update-ref msg refs/heads/bad...name $new_oid $ZERO_OID REF_SKIP_REFNAME_VERIFICATION && - git show-ref --exists refs/heads/bad...name -' - -test_expect_success '--exists with arbitrary symref' ' - test_when_finished "git symbolic-ref -d refs/symref" && - git symbolic-ref refs/symref refs/heads/$GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME && - git show-ref --exists refs/symref -' - -test_expect_success '--exists with dangling symref' ' - test_when_finished "git symbolic-ref -d refs/heads/dangling" && - git symbolic-ref refs/heads/dangling refs/heads/does-not-exist && - git show-ref --exists refs/heads/dangling -' - -test_expect_success '--exists with nonexistent object ID' ' - test-tool ref-store main update-ref msg refs/heads/missing-oid $(test_oid 001) $ZERO_OID REF_SKIP_OID_VERIFICATION && - git show-ref --exists refs/heads/missing-oid -' - -test_expect_success '--exists with non-commit object' ' - tree_oid=$(git rev-parse HEAD^{tree}) && - test-tool ref-store main update-ref msg refs/heads/tree ${tree_oid} $ZERO_OID REF_SKIP_OID_VERIFICATION && - git show-ref --exists refs/heads/tree -' - -test_expect_success '--exists with directory fails with generic error' ' - cat >expect <<-EOF && - error: reference does not exist - EOF - test_expect_code 2 git show-ref --exists refs/heads 2>err && - test_cmp expect err -' - -test_expect_success '--exists with non-existent special ref' ' - test_expect_code 2 git show-ref --exists FETCH_HEAD -' - -test_expect_success '--exists with existing special ref' ' - test_when_finished "rm .git/FETCH_HEAD" && - git rev-parse HEAD >.git/FETCH_HEAD && - git show-ref --exists FETCH_HEAD -' - -test_done +. "$TEST_DIRECTORY"/show-ref-exists-tests.sh From ef94b3e5c609ea2bd87c7ed9e9fcf8750430e4ac Mon Sep 17 00:00:00 2001 From: Meet Soni Date: Tue, 26 Aug 2025 12:11:10 +0530 Subject: [PATCH 20/25] t: add test for git refs exists subcommand Add a test script, `t/t1462-refs-exists.sh`, for the `git refs exists` command. This script acts as a simple driver, leveraging the shared test library created in the preceding commit. It works by overriding the `$git_show_ref_exists` variable to "git refs exists" and then sourcing the shared library (`t/show-ref-exists-tests.sh`). This approach ensures that `git refs exists` is tested against the entire comprehensive test suite of `git show-ref --exists`, verifying that it acts as a compatible drop-in replacement. Mentored-by: Patrick Steinhardt Mentored-by: shejialuo Signed-off-by: Meet Soni Acked-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- t/meson.build | 1 + t/t1462-refs-exists.sh | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100755 t/t1462-refs-exists.sh diff --git a/t/meson.build b/t/meson.build index 4d6bc3d38e28a1..93e9773ec8c6bb 100644 --- a/t/meson.build +++ b/t/meson.build @@ -211,6 +211,7 @@ integration_tests = [ 't1451-fsck-buffer.sh', 't1460-refs-migrate.sh', 't1461-refs-list.sh', + 't1462-refs-exists.sh', 't1500-rev-parse.sh', 't1501-work-tree.sh', 't1502-rev-parse-parseopt.sh', diff --git a/t/t1462-refs-exists.sh b/t/t1462-refs-exists.sh new file mode 100755 index 00000000000000..349453c4caed11 --- /dev/null +++ b/t/t1462-refs-exists.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +test_description='refs exists' +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +git_show_ref_exists='git refs exists' +. "$TEST_DIRECTORY"/show-ref-exists-tests.sh From bb4a83bb94e9307d9ec9bb87731effa37e58c86a Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Tue, 2 Sep 2025 22:35:46 +0200 Subject: [PATCH 21/25] doc: add missing backtick for inline-verbatim Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/fetch-options.adoc | 2 +- Documentation/git-multi-pack-index.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/fetch-options.adoc b/Documentation/fetch-options.adoc index b01372e4b3c659..65f7d1e7c7b435 100644 --- a/Documentation/fetch-options.adoc +++ b/Documentation/fetch-options.adoc @@ -1,7 +1,7 @@ --[no-]all:: Fetch all remotes, except for the ones that has the `remote..skipFetchAll` configuration variable set. - This overrides the configuration variable fetch.all`. + This overrides the configuration variable `fetch.all`. -a:: --append:: diff --git a/Documentation/git-multi-pack-index.adoc b/Documentation/git-multi-pack-index.adoc index 631d5c7d15c0de..15b333041764be 100644 --- a/Documentation/git-multi-pack-index.adoc +++ b/Documentation/git-multi-pack-index.adoc @@ -28,7 +28,7 @@ OPTIONS --[no-]progress:: Turn progress on/off explicitly. If neither is specified, progress is shown if standard error is connected to a terminal. Supported by - sub-commands `write`, `verify`, `expire`, and `repack. + sub-commands `write`, `verify`, `expire`, and `repack`. The following subcommands are available: From b0d97aac19b91886e4ae8745d56746e1df4d1457 Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Tue, 2 Sep 2025 22:35:47 +0200 Subject: [PATCH 22/25] doc: remove extra backtick for inline-verbatim Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-fast-import.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/git-fast-import.adoc b/Documentation/git-fast-import.adoc index 58a2eaa51a8034..a54fcd7f21dd48 100644 --- a/Documentation/git-fast-import.adoc +++ b/Documentation/git-fast-import.adoc @@ -597,7 +597,7 @@ External data format:: + Here usually `` must be either a mark reference (`:`) set by a prior `blob` command, or a full 40-byte SHA-1 of an -existing Git blob object. If `` is `040000`` then +existing Git blob object. If `` is `040000` then `` must be the full 40-byte SHA-1 of an existing Git tree object or a mark reference set with `--import-marks`. From 30598ccc4d93bb235e0807c04835cd00ba63577b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Tue, 2 Sep 2025 20:24:52 +0200 Subject: [PATCH 23/25] describe: use oidset in finish_depth_computation() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Depth computation can end early if all remaining commits are flagged. The current code determines if that's the case by checking all queue items each time it dequeues a flagged commit. This can cause quadratic complexity. We could simply count the flagged items in the queue and then update that number as we add and remove items. That would provide a general speedup, but leave one case where we have to scan the whole queue: When we flag a previously seen, but unflagged commit. It could be on the queue and then we'd have to decrease our count. We could dedicate an object flag to track queue membership, but that would leave less for candidate tags, affecting the results. So use a hash table, specifically an oidset of commit hashes, to track that. This avoids quadratic behaviour in all cases and provides a nice performance boost over the previous commit, 08bb69d70f (describe: use prio_queue_replace(), 2025-08-03): Benchmark 1: ./git_08bb69d70f describe $(git rev-list v2.41.0..v2.47.0) Time (mean ± σ): 855.3 ms ± 1.3 ms [User: 790.8 ms, System: 49.9 ms] Range (min … max): 853.7 ms … 857.8 ms 10 runs Benchmark 2: ./git describe $(git rev-list v2.41.0..v2.47.0) Time (mean ± σ): 610.8 ms ± 1.7 ms [User: 546.9 ms, System: 49.3 ms] Range (min … max): 608.9 ms … 613.3 ms 10 runs Summary ./git describe $(git rev-list v2.41.0..v2.47.0) ran 1.40 ± 0.00 times faster than ./git_08bb69d70f describe $(git rev-list v2.41.0..v2.47.0) Helped-by: Jeff King Signed-off-by: René Scharfe Signed-off-by: Junio C Hamano --- builtin/describe.c | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/builtin/describe.c b/builtin/describe.c index c18e4b3e4b714f..deb84a84297bc3 100644 --- a/builtin/describe.c +++ b/builtin/describe.c @@ -24,6 +24,7 @@ #include "commit-slab.h" #include "wildmatch.h" #include "prio-queue.h" +#include "oidset.h" #define MAX_TAGS (FLAG_BITS - 1) #define DEFAULT_CANDIDATES 10 @@ -286,38 +287,47 @@ static void lazy_queue_clear(struct lazy_queue *queue) queue->get_pending = false; } -static bool all_have_flag(const struct lazy_queue *queue, unsigned flag) +static unsigned long finish_depth_computation(struct lazy_queue *queue, + struct possible_tag *best) { + unsigned long seen_commits = 0; + struct oidset unflagged = OIDSET_INIT; + for (size_t i = queue->get_pending ? 1 : 0; i < queue->queue.nr; i++) { struct commit *commit = queue->queue.array[i].data; - if (!(commit->object.flags & flag)) - return false; + if (!(commit->object.flags & best->flag_within)) + oidset_insert(&unflagged, &commit->object.oid); } - return true; -} -static unsigned long finish_depth_computation(struct lazy_queue *queue, - struct possible_tag *best) -{ - unsigned long seen_commits = 0; while (!lazy_queue_empty(queue)) { struct commit *c = lazy_queue_get(queue); struct commit_list *parents = c->parents; seen_commits++; if (c->object.flags & best->flag_within) { - if (all_have_flag(queue, best->flag_within)) + if (!oidset_size(&unflagged)) break; - } else + } else { + oidset_remove(&unflagged, &c->object.oid); best->depth++; + } while (parents) { + unsigned seen, flag_before, flag_after; struct commit *p = parents->item; repo_parse_commit(the_repository, p); - if (!(p->object.flags & SEEN)) + seen = p->object.flags & SEEN; + if (!seen) lazy_queue_put(queue, p); + flag_before = p->object.flags & best->flag_within; p->object.flags |= c->object.flags; + flag_after = p->object.flags & best->flag_within; + if (!seen && !flag_after) + oidset_insert(&unflagged, &p->object.oid); + if (seen && !flag_before && flag_after) + oidset_remove(&unflagged, &p->object.oid); parents = parents->next; } } + oidset_clear(&unflagged); return seen_commits; } From f9a6705d9a1012bc5349a9f010fee3abbf8c53e6 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 8 Aug 2025 11:59:43 +0200 Subject: [PATCH 24/25] t0450: add allowlist for builtins with missing .adoc Before we were silently skipping all builtins that don't have a matching .adoc file. This is overly loose and might skip documentation files when it shouldn't, for example when there was a typo in the filename. To ensure no new builtins are added without documentation, add an allowlist: t0450/adoc-missing. In this file only builtin commands that do *not* have a corresponding .adoc file shall be listed. If there is a mismatch, fail the test. This should force future contributions to either add an .adoc, or add the builtin name to the allowlist file. Signed-off-by: Toon Claes [jc: squashed Patrick's "missing file fix" in] Signed-off-by: Junio C Hamano --- t/t0450-txt-doc-vs-help.sh | 13 +++++++++++-- t/t0450/adoc-missing | 9 +++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 t/t0450/adoc-missing diff --git a/t/t0450-txt-doc-vs-help.sh b/t/t0450-txt-doc-vs-help.sh index da2d0af5b07810..e12e18f97f02eb 100755 --- a/t/t0450-txt-doc-vs-help.sh +++ b/t/t0450-txt-doc-vs-help.sh @@ -112,10 +112,19 @@ do adoc="$(builtin_to_adoc "$builtin")" && preq="$(echo BUILTIN_ADOC_$builtin | tr '[:lower:]-' '[:upper:]_')" && - if test -f "$adoc" + # If and only if *.adoc is missing, builtin shall be listed in t0450/adoc-missing. + if grep -q "^$builtin$" "$TEST_DIRECTORY"/t0450/adoc-missing then + test_expect_success "$builtin appropriately marked as not having .adoc" ' + ! test -f "$adoc" + ' + else test_set_prereq "$preq" - fi && + + test_expect_success "$builtin appropriately marked as having .adoc" ' + test -f "$adoc" + ' + fi # *.adoc output assertions test_expect_success "$preq" "$builtin *.adoc SYNOPSIS has dashed labels" ' diff --git a/t/t0450/adoc-missing b/t/t0450/adoc-missing new file mode 100644 index 00000000000000..1ec9f8dcf32358 --- /dev/null +++ b/t/t0450/adoc-missing @@ -0,0 +1,9 @@ +checkout--worker +merge-ours +merge-recursive +merge-recursive-ours +merge-recursive-theirs +merge-subtree +pickaxe +submodule--helper +upload-archive--writer From 92c87bdc406e5bcf1b516a861d609b03d99c23b7 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 12 Sep 2025 10:41:02 -0700 Subject: [PATCH 25/25] The eighth batch Signed-off-by: Junio C Hamano --- Documentation/RelNotes/2.52.0.adoc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Documentation/RelNotes/2.52.0.adoc b/Documentation/RelNotes/2.52.0.adoc index 60660314911cc1..6f8579bdeed781 100644 --- a/Documentation/RelNotes/2.52.0.adoc +++ b/Documentation/RelNotes/2.52.0.adoc @@ -17,6 +17,9 @@ UI, Workflows & Features * A new command "git last-modified" has been added to show the closest ancestor commit that touched each path. + * "git refs exists" that works like "git show-ref --exists" has been + added. + Performance, Internal Implementation, Development Support etc. -------------------------------------------------------------- @@ -47,6 +50,11 @@ Performance, Internal Implementation, Development Support etc. hash function implementation used to hash lines have been updated to the one used for ELF symbol lookup by Glibc. + * Instead of scanning for the remaining items to see if there are + still commits to be explored in the queue, use khash to remember + which items are still on the queue (an unacceptable alternative is + to reserve one object flag bits). + Fixes since v2.51 ----------------- @@ -151,6 +159,11 @@ including security updates, are included in this release. this. (merge 681f26bccc ds/ls-files-lazy-unsparse later to maint). + * Windows "real-time monitoring" interferes with the execution of + tests and affects negatively in both correctness and performance, + which has been disabled in Gitlab CI. + (merge 608cf5b793 ps/gitlab-ci-disable-windows-monitoring later to maint). + * Other code cleanup, docfix, build fix, etc. (merge 823d537fa7 kh/doc-git-log-markup-fix later to maint). (merge cf7efa4f33 rj/t6137-cygwin-fix later to maint). @@ -164,3 +177,8 @@ including security updates, are included in this release. (merge 374579c6d4 kh/doc-interpret-trailers-markup-fix later to maint). (merge 44dce6541c kh/doc-config-typofix later to maint). (merge 785628b173 js/doc-sending-patch-via-thunderbird later to maint). + (merge e5c27bd3d8 je/doc-add later to maint). + (merge 13296ac909 ps/object-store-midx-dedup-info later to maint). + (merge 2f4bf83ffc km/alias-doc-markup-fix later to maint). + (merge b0d97aac19 kh/doc-markup-fixes later to maint). + (merge f9a6705d9a tc/t0450-harden later to maint).