From 55bb7689f7bf454fcf5bf98da74572c2f3a1374f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Sat, 11 Oct 2025 22:10:45 +0200 Subject: [PATCH 1/5] Add simple-odb test to meson integration list --- Documentation/technical/native-odb-api.txt | 65 +++++++ Makefile | 2 + simple-odb.c | 195 +++++++++++++++++++++ simple-odb.h | 24 +++ t/helper/test-simple-odb.c | 114 ++++++++++++ t/helper/test-tool.c | 9 +- t/helper/test-tool.h | 1 + t/meson.build | 1 + t/t0039-simple-odb.sh | 46 +++++ 9 files changed, 453 insertions(+), 4 deletions(-) create mode 100644 Documentation/technical/native-odb-api.txt create mode 100644 simple-odb.c create mode 100644 simple-odb.h create mode 100644 t/helper/test-simple-odb.c create mode 100755 t/t0039-simple-odb.sh diff --git a/Documentation/technical/native-odb-api.txt b/Documentation/technical/native-odb-api.txt new file mode 100644 index 00000000000000..1e92c492ff7825 --- /dev/null +++ b/Documentation/technical/native-odb-api.txt @@ -0,0 +1,65 @@ +Native ODB API overview +========================= + +Git's native object database (ODB) exposes `struct object_database` and +`struct odb_source` as the central data structures for working with local and +alternate object stores.【F:odb.h†L102-L160】 The API provides helpers to create a +database (`odb_new()`), attach paths as object sources, and read or write +objects through functions such as `odb_write_object_ext()` that operate on the +local repository's primary object directory.【F:odb.h†L169-L477】 + +A consumer that wants to experiment with custom storage can allocate its own ODB +using `odb_new()`, populate `struct odb_source` entries, and reuse Git's object +hashing helpers (for example `hash_object_file()`) to stay compatible with Git's +loose-object format.【F:object-file.h†L1-L126】【F:odb.c†L983-L1007】 + +Simple ODB example helper +------------------------- + +The `test-tool simple-odb` helper demonstrates a minimal object database that +stores entries using Git's loose-object layout. It hashes payloads using the +repository's hash algorithm, compresses the `" \0"` payload, +and writes the result underneath an `objects` directory it maintains on disk. +The helper exposes commands to initialise the store, append new objects, and +record the resulting directory as an alternate for the current repository so +that Git can discover the objects without further patches.【F:simple-odb.c†L1-L158】【F:t/helper/test-simple-odb.c†L1-L89】 + +The accompanying regression test (`t/t0039-simple-odb.sh`) uses the helper to +write and read blobs, verifying that repositories can read from the alternate +store via `git cat-file` once the helper has created the loose object and added +its path to `objects/info/alternates`.【F:t/t0039-simple-odb.sh†L1-L43】 + +Activating the helper via alternates +------------------------------------ + +The helper relies entirely on Git's existing alternates mechanism: adding the +simple store's `objects` directory to `objects/info/alternates` (or setting +`GIT_ALTERNATE_OBJECT_DIRECTORIES`) is enough for Git to consult it during +object lookups. No changes to `odb.c` are required because the helper writes +objects in the same layout as a regular loose object directory. The test suite +demonstrates this by initialising a repository, adding the helper-managed +directory as an alternate, and reading objects through `git cat-file` without +any additional plumbing.【F:t/helper/test-simple-odb.c†L64-L89】【F:t/t0039-simple-odb.sh†L8-L43】 + +Comparison with other ODB APIs +------------------------------ + +* **libgit2** exposes an `git_odb` type with pluggable backends and callbacks + that are registered globally. Implementers provide `read`, `write`, and + iteration function pointers, but the integration is centered around a single + multi-backend registry instead of Git's notion of one primary source plus a + linked list of alternates. +* **gitoxide** (gix) models its ODB as a layered `gix_odb::Store`, combining + a cache and multiple stores selected via configuration. Custom stores + implement the `Store` trait and are typically used by wiring them into a + `Repository` configuration object. + +Key compatibility considerations +-------------------------------- + +Git's native API must preserve backwards compatibility with repositories that +may be accessed by older clients, so helpers should reuse Git's hashing +functions and object formats rather than inventing new on-disk layouts. The +example therefore hashes payloads with the repository's configured algorithm and +reuses the conventional loose-object directory structure, letting existing +clients take advantage of the alternate without special knowledge.【F:simple-odb.c†L63-L135】 diff --git a/Makefile b/Makefile index 7ea149598d8ed8..fd578dd03c4d59 100644 --- a/Makefile +++ b/Makefile @@ -854,6 +854,7 @@ TEST_BUILTINS_OBJS += test-sha1.o TEST_BUILTINS_OBJS += test-sha256.o TEST_BUILTINS_OBJS += test-sigchain.o TEST_BUILTINS_OBJS += test-simple-ipc.o +TEST_BUILTINS_OBJS += test-simple-odb.o TEST_BUILTINS_OBJS += test-string-list.o TEST_BUILTINS_OBJS += test-submodule-config.o TEST_BUILTINS_OBJS += test-submodule-nested-repo-config.o @@ -1265,6 +1266,7 @@ LIB_OBJS += setup.o LIB_OBJS += shallow.o LIB_OBJS += sideband.o LIB_OBJS += sigchain.o +LIB_OBJS += simple-odb.o LIB_OBJS += sparse-index.o LIB_OBJS += split-index.o LIB_OBJS += stable-qsort.o diff --git a/simple-odb.c b/simple-odb.c new file mode 100644 index 00000000000000..4409ec81125334 --- /dev/null +++ b/simple-odb.c @@ -0,0 +1,195 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "git-compat-util.h" +#include "abspath.h" +#include "environment.h" +#include "dir.h" +#include "hash.h" +#include "hex.h" +#include "path.h" +#include "repository.h" +#include "simple-odb.h" +#include "wrapper.h" +#include "git-zlib.h" + +static int make_dir(const char *path) +{ + char *dup; + + if (!path || !*path) + return error("simple-odb: empty path"); + + dup = xstrdup(path); + if (safe_create_leading_directories_no_share(dup) < 0) { + int save_errno = errno; + free(dup); + errno = save_errno; + return error_errno("unable to create directories for '%s'", path); + } + free(dup); + + if (mkdir(path, 0777) && errno != EEXIST) + return error_errno("unable to create '%s'", path); + + return 0; +} + +void simple_odb_init(struct simple_odb *odb) +{ + strbuf_init(&odb->root, 0); + strbuf_init(&odb->objects_dir, 0); +} + +void simple_odb_release(struct simple_odb *odb) +{ + strbuf_release(&odb->root); + strbuf_release(&odb->objects_dir); +} + +int simple_odb_prepare(struct simple_odb *odb, const char *path) +{ + struct strbuf real = STRBUF_INIT; + struct strbuf tmp = STRBUF_INIT; + int ret = -1; + + if (!path || !*path) + return error("simple-odb: missing object directory path"); + + strbuf_addstr(&tmp, path); + if (make_dir(tmp.buf)) + goto out; + + if (!strbuf_realpath(&real, tmp.buf, 1)) { + error_errno("simple-odb: unable to canonicalize '%s'", tmp.buf); + goto out; + } + + strbuf_addf(&odb->objects_dir, "%s/objects", real.buf); + if (make_dir(odb->objects_dir.buf)) + goto out; + if (make_dir(mkpath("%s/info", odb->objects_dir.buf))) + goto out; + if (make_dir(mkpath("%s/pack", odb->objects_dir.buf))) + goto out; + + strbuf_swap(&odb->root, &real); + ret = 0; +out: + strbuf_release(&real); + strbuf_release(&tmp); + if (ret) + simple_odb_release(odb); + return ret; +} + +int simple_odb_store_buffer(struct simple_odb *odb, + enum object_type type, + const void *data, + size_t len, + struct object_id *oid) +{ + struct git_hash_ctx ctx; + struct strbuf dir = STRBUF_INIT; + struct strbuf path = STRBUF_INIT; + struct strbuf tmp = STRBUF_INIT; + struct strbuf header = STRBUF_INIT; + struct git_zstream stream; + unsigned long maxsize; + size_t header_len; + size_t total_len; + size_t compressed_len; + int fd = -1; + int ret = -1; + unsigned char *payload = NULL; + unsigned char *compressed = NULL; + const char *type_name_str = type_name(type); + const struct git_hash_algo *algo; + + if (!odb->objects_dir.len) + return error("simple-odb: object directory not initialized"); + if (!type_name_str) + return error("simple-odb: invalid object type"); + + strbuf_addf(&header, "%s %"PRIuMAX, type_name_str, (uintmax_t)len); + header_len = header.len + 1; + + total_len = header_len + len; + payload = xmalloc(total_len); + memcpy(payload, header.buf, header_len); + if (len) + memcpy(payload + header_len, data, len); + + algo = the_repository ? the_repository->hash_algo + : &hash_algos[GIT_HASH_SHA1_LEGACY]; + + oid_set_algo(oid, algo); + + algo->init_fn(&ctx); + algo->update_fn(&ctx, payload, total_len); + algo->final_oid_fn(oid, &ctx); + + git_deflate_init(&stream, zlib_compression_level); + maxsize = git_deflate_bound(&stream, total_len); + compressed = xmalloc(maxsize); + stream.next_in = payload; + stream.avail_in = total_len; + stream.next_out = compressed; + stream.avail_out = maxsize; + if (git_deflate(&stream, Z_FINISH) != Z_STREAM_END) { + error("simple-odb: unable to compress object"); + git_deflate_abort(&stream); + goto out; + } + compressed_len = maxsize - stream.avail_out; + git_deflate_end_gently(&stream); + + const char *hex = oid_to_hex(oid); + + strbuf_addf(&dir, "%s/%2.2s", odb->objects_dir.buf, hex); + if (make_dir(dir.buf)) + goto out; + + strbuf_addf(&path, "%s/%s", dir.buf, hex + 2); + if (!access(path.buf, F_OK)) { + ret = 0; + goto out; + } + + strbuf_addf(&tmp, "%s/.tmp_simple_XXXXXX", odb->objects_dir.buf); + fd = xmkstemp_mode(tmp.buf, 0444); + if (fd < 0) { + error_errno("simple-odb: unable to create temporary file"); + goto out; + } + if (write_in_full(fd, compressed, compressed_len) < 0) { + error_errno("simple-odb: unable to write object data"); + goto out; + } + if (close(fd) < 0) { + error_errno("simple-odb: unable to close object file"); + fd = -1; + goto out; + } + fd = -1; + + if (rename(tmp.buf, path.buf)) { + error_errno("simple-odb: unable to move object into place"); + goto out; + } + strbuf_setlen(&tmp, 0); + if (the_repository) + adjust_shared_perm(the_repository, path.buf); + ret = 0; +out: + if (fd >= 0) + close(fd); + if (tmp.len) + unlink_or_warn(tmp.buf); + strbuf_release(&dir); + strbuf_release(&path); + strbuf_release(&tmp); + strbuf_release(&header); + free(payload); + free(compressed); + return ret; +} diff --git a/simple-odb.h b/simple-odb.h new file mode 100644 index 00000000000000..475abd2d8350bf --- /dev/null +++ b/simple-odb.h @@ -0,0 +1,24 @@ +#ifndef SIMPLE_ODB_H +#define SIMPLE_ODB_H + +#include "git-compat-util.h" +#include "hash.h" +#include "object.h" +#include "strbuf.h" + +struct simple_odb { + struct strbuf root; + struct strbuf objects_dir; +}; + +void simple_odb_init(struct simple_odb *odb); +void simple_odb_release(struct simple_odb *odb); + +int simple_odb_prepare(struct simple_odb *odb, const char *path); +int simple_odb_store_buffer(struct simple_odb *odb, + enum object_type type, + const void *data, + size_t len, + struct object_id *oid); + +#endif /* SIMPLE_ODB_H */ diff --git a/t/helper/test-simple-odb.c b/t/helper/test-simple-odb.c new file mode 100644 index 00000000000000..1e6fe7690b4633 --- /dev/null +++ b/t/helper/test-simple-odb.c @@ -0,0 +1,114 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "test-tool.h" +#include "odb.h" +#include "repository.h" +#include "setup.h" +#include "simple-odb.h" +#include "strbuf.h" +#include "hex.h" + +static void simple_odb_usage(const char *arg0) +{ + die("usage: %s [...]\n" + "\n" + "Commands:\n" + " init \n" + " write \n" + " add-alternate \n", arg0); +} + +static int cmd_simple_init(const char *path) +{ + struct simple_odb odb; + + simple_odb_init(&odb); + if (simple_odb_prepare(&odb, path)) { + simple_odb_release(&odb); + return 1; + } + simple_odb_release(&odb); + return 0; +} + +static int cmd_simple_write(const char *path, const char *type_name, const char *file) +{ + struct simple_odb odb; + struct strbuf data = STRBUF_INIT; + struct object_id oid; + enum object_type type; + int ret = 0; + + type = type_from_string_gently(type_name, strlen(type_name), 1); + if (type < 0) + return error("unknown type '%s'", type_name); + + if (!strcmp(file, "-")) { + if (strbuf_read(&data, 0, 0) < 0) + return error_errno("unable to read from stdin"); + } else if (strbuf_read_file(&data, file, 0) < 0) { + return error_errno("unable to read '%s'", file); + } + + simple_odb_init(&odb); + if (simple_odb_prepare(&odb, path)) { + ret = 1; + goto out_release; + } + + if (simple_odb_store_buffer(&odb, type, data.buf, data.len, &oid)) { + ret = 1; + goto out_release; + } + + printf("%s\n", oid_to_hex(&oid)); + +out_release: + simple_odb_release(&odb); + strbuf_release(&data); + return ret; +} + +static int cmd_simple_attach(const char *path) +{ + struct simple_odb odb; + + if (!the_repository) + return error("simple-odb: repository not set"); + + simple_odb_init(&odb); + if (simple_odb_prepare(&odb, path)) { + simple_odb_release(&odb); + return 1; + } + + odb_add_to_alternates_file(the_repository->objects, odb.objects_dir.buf); + simple_odb_release(&odb); + return 0; +} + +int cmd__simple_odb(int argc, const char **argv) +{ + setup_git_directory(); + + if (argc < 2) + simple_odb_usage(argv[0]); + + if (!strcmp(argv[1], "init")) { + if (argc != 3) + simple_odb_usage(argv[0]); + return cmd_simple_init(argv[2]); + } + if (!strcmp(argv[1], "write")) { + if (argc != 5) + simple_odb_usage(argv[0]); + return cmd_simple_write(argv[2], argv[3], argv[4]); + } + if (!strcmp(argv[1], "add-alternate")) { + if (argc != 3) + simple_odb_usage(argv[0]); + return cmd_simple_attach(argv[2]); + } + + simple_odb_usage(argv[0]); + return 1; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index a7abc618b3887e..8772090e0ddc39 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -74,10 +74,11 @@ static struct test_cmd cmds[] = { { "sha1", cmd__sha1 }, { "sha1-is-sha1dc", cmd__sha1_is_sha1dc }, { "sha1-unsafe", cmd__sha1_unsafe }, - { "sha256", cmd__sha256 }, - { "sigchain", cmd__sigchain }, - { "simple-ipc", cmd__simple_ipc }, - { "string-list", cmd__string_list }, + { "sha256", cmd__sha256 }, + { "sigchain", cmd__sigchain }, + { "simple-ipc", cmd__simple_ipc }, + { "simple-odb", cmd__simple_odb }, + { "string-list", cmd__string_list }, { "submodule", cmd__submodule }, { "submodule-config", cmd__submodule_config }, { "submodule-nested-repo-config", cmd__submodule_nested_repo_config }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 7f150fa1eb9ad2..5b9e3fe0b0f5d1 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -70,6 +70,7 @@ int cmd__sha1_unsafe(int argc, const char **argv); int cmd__sha256(int argc, const char **argv); int cmd__sigchain(int argc, const char **argv); int cmd__simple_ipc(int argc, const char **argv); +int cmd__simple_odb(int argc, const char **argv); int cmd__string_list(int argc, const char **argv); int cmd__submodule(int argc, const char **argv); int cmd__submodule_config(int argc, const char **argv); diff --git a/t/meson.build b/t/meson.build index 11376b9e256dd6..15af0c96711c32 100644 --- a/t/meson.build +++ b/t/meson.build @@ -100,6 +100,7 @@ integration_tests = [ 't0033-safe-directory.sh', 't0034-root-safe-directory.sh', 't0035-safe-bare-repository.sh', + 't0039-simple-odb.sh', 't0040-parse-options.sh', 't0041-usage.sh', 't0050-filesystem.sh', diff --git a/t/t0039-simple-odb.sh b/t/t0039-simple-odb.sh new file mode 100755 index 00000000000000..faf06367ea8719 --- /dev/null +++ b/t/t0039-simple-odb.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +test_description='simple ODB experiment via alternates' + +. ./test-lib.sh + +TEST_PASSES_SANITIZE_LEAK=true + +simple_path=$PWD/simple-odb-store + +clean_simple () { + rm -rf "$simple_path" +} + +test_when_finished 'clean_simple' + +test_expect_success 'blob written to simple ODB is accessible via alternates' ' + test_create_repo main && + ( + cd main && + test-tool simple-odb init "$simple_path" && + echo "hello from simple" >payload && + blob=$(test-tool simple-odb write "$simple_path" blob payload) && + test-tool simple-odb add-alternate "$simple_path" && + git cat-file -p "$blob" >actual && + test_cmp payload actual + ) +' + +test_expect_success 'stdin writes work for alternate ODB' ' + test_create_repo stdin-repo && + ( + cd stdin-repo && + other_store=$PWD/../simple-stdin && + test_when_finished "rm -rf $other_store" && + test-tool simple-odb init "$other_store" && + printf "data via stdin" | test-tool simple-odb write "$other_store" blob - >oid && + test-tool simple-odb add-alternate "$other_store" && + oid=$(cat oid) && + git cat-file -p "$oid" >out && + printf "data via stdin" >expect && + test_cmp expect out + ) +' + +test_done From 8b8eb94acaa08c174742ef4f7b6c5e28d70559ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Sun, 12 Oct 2025 21:22:20 +0200 Subject: [PATCH 2/5] t0039: fix cleanup when running in subshell --- t/t0039-simple-odb.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/t0039-simple-odb.sh b/t/t0039-simple-odb.sh index faf06367ea8719..4ff06588bd375d 100755 --- a/t/t0039-simple-odb.sh +++ b/t/t0039-simple-odb.sh @@ -29,10 +29,10 @@ test_expect_success 'blob written to simple ODB is accessible via alternates' ' test_expect_success 'stdin writes work for alternate ODB' ' test_create_repo stdin-repo && + other_store=$PWD/simple-stdin && + test_when_finished "rm -rf stdin-repo \"$other_store\"" && ( cd stdin-repo && - other_store=$PWD/../simple-stdin && - test_when_finished "rm -rf $other_store" && test-tool simple-odb init "$other_store" && printf "data via stdin" | test-tool simple-odb write "$other_store" blob - >oid && test-tool simple-odb add-alternate "$other_store" && From 4feaf62510b0f67e366e0e92cb3615d2597fe117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Mon, 13 Oct 2025 19:05:31 +0200 Subject: [PATCH 3/5] Teach simple-odb helper to route large blobs to LOP store --- Documentation/technical/native-odb-api.txt | 20 ++++++- t/helper/test-simple-odb.c | 67 ++++++++++++++++++++++ t/t0039-simple-odb.sh | 25 ++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/Documentation/technical/native-odb-api.txt b/Documentation/technical/native-odb-api.txt index 1e92c492ff7825..545f650138a379 100644 --- a/Documentation/technical/native-odb-api.txt +++ b/Documentation/technical/native-odb-api.txt @@ -22,7 +22,11 @@ repository's hash algorithm, compresses the `" \0"` payload, and writes the result underneath an `objects` directory it maintains on disk. The helper exposes commands to initialise the store, append new objects, and record the resulting directory as an alternate for the current repository so -that Git can discover the objects without further patches.【F:simple-odb.c†L1-L158】【F:t/helper/test-simple-odb.c†L1-L89】 +that Git can discover the objects without further patches. It also provides a +`lop-write` command that dispatches blobs according to a size threshold: small +payloads are written to the repository's primary object store while larger +payloads are diverted into the simple store and automatically added as an +alternate.【F:simple-odb.c†L1-L158】【F:t/helper/test-simple-odb.c†L1-L157】 The accompanying regression test (`t/t0039-simple-odb.sh`) uses the helper to write and read blobs, verifying that repositories can read from the alternate @@ -41,6 +45,20 @@ demonstrates this by initialising a repository, adding the helper-managed directory as an alternate, and reading objects through `git cat-file` without any additional plumbing.【F:t/helper/test-simple-odb.c†L64-L89】【F:t/t0039-simple-odb.sh†L8-L43】 +Large Object Promisor experiments +--------------------------------- + +The Large Object Promisor (LOP) design aims to keep very large blobs on +dedicated promisor remotes while the primary remote serves the remainder of the +repository.【F:Documentation/technical/large-object-promisors.adoc†L15-L114】 The +`lop-write` helper command mirrors that split locally: a caller passes the +simple store path and a `blob:limit`-style threshold, and the helper stores +objects larger than the limit in the alternate while leaving smaller objects in +the main store. Because the alternate is recorded automatically, subsequent +commands like `git cat-file` can resolve those large blobs transparently, which +makes it easy to prototype LOP-aware workflows without modifying Git's core +ODB routines.【F:t/helper/test-simple-odb.c†L90-L157】【F:t/t0039-simple-odb.sh†L45-L80】 + Comparison with other ODB APIs ------------------------------ diff --git a/t/helper/test-simple-odb.c b/t/helper/test-simple-odb.c index 1e6fe7690b4633..ac7630e17baa6c 100644 --- a/t/helper/test-simple-odb.c +++ b/t/helper/test-simple-odb.c @@ -1,6 +1,8 @@ #define USE_THE_REPOSITORY_VARIABLE #include "test-tool.h" #include "odb.h" +#include "object-file.h" +#include "parse.h" #include "repository.h" #include "setup.h" #include "simple-odb.h" @@ -14,6 +16,7 @@ static void simple_odb_usage(const char *arg0) "Commands:\n" " init \n" " write \n" + " lop-write \n" " add-alternate \n", arg0); } @@ -68,6 +71,65 @@ static int cmd_simple_write(const char *path, const char *type_name, const char return ret; } +static int cmd_simple_lop_write(const char *path, const char *limit_str, + const char *type_name, const char *file) +{ + struct simple_odb odb; + struct strbuf data = STRBUF_INIT; + struct object_id oid; + enum object_type type; + unsigned long limit; + size_t size_limit; + int ret = 0; + + if (!the_repository) + return error("simple-odb: repository not set"); + + if (!git_parse_ulong(limit_str, &limit)) + return error("simple-odb: invalid size limit '%s'", limit_str); + if (limit > SIZE_MAX) + return error("simple-odb: size limit '%s' exceeds platform support", limit_str); + size_limit = limit; + + type = type_from_string_gently(type_name, strlen(type_name), 1); + if (type < 0) + return error("unknown type '%s'", type_name); + + if (!strcmp(file, "-")) { + if (strbuf_read(&data, 0, 0) < 0) + return error_errno("unable to read from stdin"); + } else if (strbuf_read_file(&data, file, 0) < 0) { + return error_errno("unable to read '%s'", file); + } + + simple_odb_init(&odb); + if (simple_odb_prepare(&odb, path)) { + ret = 1; + goto out_release; + } + + if (data.len > size_limit) { + if (simple_odb_store_buffer(&odb, type, data.buf, data.len, &oid)) { + ret = 1; + goto out_release; + } + odb_add_to_alternates_file(the_repository->objects, odb.objects_dir.buf); + } else { + if (write_object_file(the_repository->objects->sources, data.buf, + data.len, type, &oid, NULL, 0)) { + ret = 1; + goto out_release; + } + } + + printf("%s\n", oid_to_hex(&oid)); + +out_release: + simple_odb_release(&odb); + strbuf_release(&data); + return ret; +} + static int cmd_simple_attach(const char *path) { struct simple_odb odb; @@ -103,6 +165,11 @@ int cmd__simple_odb(int argc, const char **argv) simple_odb_usage(argv[0]); return cmd_simple_write(argv[2], argv[3], argv[4]); } + if (!strcmp(argv[1], "lop-write")) { + if (argc != 6) + simple_odb_usage(argv[0]); + return cmd_simple_lop_write(argv[2], argv[3], argv[4], argv[5]); + } if (!strcmp(argv[1], "add-alternate")) { if (argc != 3) simple_odb_usage(argv[0]); diff --git a/t/t0039-simple-odb.sh b/t/t0039-simple-odb.sh index 4ff06588bd375d..13afeed386020f 100755 --- a/t/t0039-simple-odb.sh +++ b/t/t0039-simple-odb.sh @@ -43,4 +43,29 @@ test_expect_success 'stdin writes work for alternate ODB' ' ) ' +test_expect_success 'lop-write dispatches by blob size and keeps large blobs in LOP' ' + test_create_repo lop-main && + lop_store=$PWD/lop-store && + test_when_finished "rm -rf lop-main \"$lop_store\"" && + ( + cd lop-main && + test-tool simple-odb init "$lop_store" && + echo "tiny" >small && + small=$(test-tool simple-odb lop-write "$lop_store" 8 blob small) && + test_path_is_file ".git/objects/$(test_oid_to_path $small)" && + test_path_is_missing "$lop_store/objects/$(test_oid_to_path $small)" && + cat >large <<-EOF && + contents stored in lop +EOF + large=$(test-tool simple-odb lop-write "$lop_store" 8 blob large) && + test_path_is_missing ".git/objects/$(test_oid_to_path $large)" && + test_path_is_file "$lop_store/objects/$(test_oid_to_path $large)" && + git cat-file -p "$large" >out && + cat >expect <<-EOF && + contents stored in lop +EOF + test_cmp expect out + ) +' + test_done From c9f60eca9157622b2b06cb56e2a981eb806c5bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Mon, 13 Oct 2025 20:37:07 +0200 Subject: [PATCH 4/5] t0039: exercise simple ODB via porcelain commit and push --- t/t0039-simple-odb.sh | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/t/t0039-simple-odb.sh b/t/t0039-simple-odb.sh index 13afeed386020f..4713cb0a199094 100755 --- a/t/t0039-simple-odb.sh +++ b/t/t0039-simple-odb.sh @@ -68,4 +68,55 @@ EOF ) ' +test_expect_success 'porcelain commit stores large blob in simple ODB alternate' ' + test_create_repo lop-porcelain && + lop_store=$PWD/lop-porcelain-store && + test_when_finished "rm -rf lop-porcelain \"$lop_store\"" && + ( + cd lop-porcelain && + test-tool simple-odb init "$lop_store" && + perl -e "binmode STDOUT; print pack(q(C*), map { \$_ % 256 } 0 .. 2047)" >binary.bin && + blob=$(test-tool simple-odb lop-write "$lop_store" 512 blob binary.bin) && + git add binary.bin && + git commit -m "commit binary via lop" && + git show HEAD:binary.bin >out && + test_cmp binary.bin out && + test_path_is_missing ".git/objects/$(test_oid_to_path $blob)" && + test_path_is_file "$lop_store/objects/$(test_oid_to_path $blob)" + ) +' + +test_expect_success 'porcelain push keeps large blob in shared simple ODB store' ' + test_create_repo lop-sender && + lop_store=$PWD/lop-shared-store && + test_when_finished "rm -rf lop-sender lop-clone lop-remote.git \"$lop_store\"" && + ( + cd lop-sender && + test-tool simple-odb init "$lop_store" && + perl -e "binmode STDOUT; print pack(q(C*), map { \$_ % 256 } 0 .. 4095)" >huge.bin && + blob=$(test-tool simple-odb lop-write "$lop_store" 1024 blob huge.bin) && + git add huge.bin && + git commit -m "stage binary for push" && + git init --bare ../lop-remote.git && + git -C ../lop-remote.git config receive.unpackLimit 1 && + mkdir -p ../lop-remote.git/objects/info && + echo "$lop_store/objects" >../lop-remote.git/objects/info/alternates && + git remote add origin ../lop-remote.git && + git push origin HEAD:main && + grep -F "$lop_store/objects" ../lop-remote.git/objects/info/alternates && + git --git-dir=../lop-remote.git symbolic-ref HEAD refs/heads/main && + git --git-dir=../lop-remote.git show main:huge.bin >../received.bin && + test_cmp huge.bin ../received.bin && + test_path_is_file "$lop_store/objects/$(test_oid_to_path $blob)" && + git clone ../lop-remote.git ../lop-clone && + ( + cd ../lop-clone && + mkdir -p .git/objects/info && + echo "$lop_store/objects" >.git/objects/info/alternates && + git show HEAD:huge.bin >../clone.bin + ) && + test_cmp huge.bin ../clone.bin + ) +' + test_done From 4768b68725ffa9632d00cf5af8fc42e1d77d6bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Mon, 13 Oct 2025 20:37:13 +0200 Subject: [PATCH 5/5] t0039: add LOP partial clone coverage --- t/t0039-simple-odb.sh | 83 ++++++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/t/t0039-simple-odb.sh b/t/t0039-simple-odb.sh index 4713cb0a199094..a5ef0e3713d752 100755 --- a/t/t0039-simple-odb.sh +++ b/t/t0039-simple-odb.sh @@ -86,37 +86,62 @@ test_expect_success 'porcelain commit stores large blob in simple ODB alternate' ) ' -test_expect_success 'porcelain push keeps large blob in shared simple ODB store' ' - test_create_repo lop-sender && - lop_store=$PWD/lop-shared-store && - test_when_finished "rm -rf lop-sender lop-clone lop-remote.git \"$lop_store\"" && +test_expect_success 'partial clone fetches large blob from LOP remote via alternates' ' + test_create_repo lop-producer && + lop_store=$PWD/lop-promisor-store && + server=$PWD/lop-server.git && + lop_remote=$PWD/lop-promisor.git && + client=$PWD/lop-client && + blob_oid_file=$PWD/lop-blob.oid && + test_when_finished "rm -rf lop-producer" && + test_when_finished "rm -rf \"$client\"" && + test_when_finished "rm -rf \"$lop_store\" \"$server\" \"$lop_remote\"" && + test_when_finished "rm -f client.bin \"$blob_oid_file\"" && + test-tool simple-odb init "$lop_store" && ( - cd lop-sender && - test-tool simple-odb init "$lop_store" && - perl -e "binmode STDOUT; print pack(q(C*), map { \$_ % 256 } 0 .. 4095)" >huge.bin && - blob=$(test-tool simple-odb lop-write "$lop_store" 1024 blob huge.bin) && + cd lop-producer && + perl -e "binmode STDOUT; print pack(q(C*), map { \$_ % 251 } 0 .. 7000)" >huge.bin && + blob=$(test-tool simple-odb lop-write "$lop_store" 4096 blob huge.bin) && git add huge.bin && - git commit -m "stage binary for push" && - git init --bare ../lop-remote.git && - git -C ../lop-remote.git config receive.unpackLimit 1 && - mkdir -p ../lop-remote.git/objects/info && - echo "$lop_store/objects" >../lop-remote.git/objects/info/alternates && - git remote add origin ../lop-remote.git && - git push origin HEAD:main && - grep -F "$lop_store/objects" ../lop-remote.git/objects/info/alternates && - git --git-dir=../lop-remote.git symbolic-ref HEAD refs/heads/main && - git --git-dir=../lop-remote.git show main:huge.bin >../received.bin && - test_cmp huge.bin ../received.bin && - test_path_is_file "$lop_store/objects/$(test_oid_to_path $blob)" && - git clone ../lop-remote.git ../lop-clone && - ( - cd ../lop-clone && - mkdir -p .git/objects/info && - echo "$lop_store/objects" >.git/objects/info/alternates && - git show HEAD:huge.bin >../clone.bin - ) && - test_cmp huge.bin ../clone.bin - ) + git commit -m "commit huge blob via lop" && + echo "$blob" >"$blob_oid_file" + ) && + blob_oid=$(cat "$blob_oid_file") && + git init --bare "$server" && + mkdir -p "$server/objects/info" && + echo "$lop_store/objects" >"$server/objects/info/alternates" && + git -C "$server" config uploadpack.allowFilter true && + git -C "$server" config uploadpack.allowAnySHA1InWant true && + git -C "$server" config promisor.advertise true && + ( + cd lop-producer && + git remote add origin "$server" && + git push origin HEAD:main + ) && + git -C "$server" symbolic-ref HEAD refs/heads/main && + test_path_is_missing "$server/objects/$(test_oid_to_path $blob_oid)" && + git init --bare "$lop_remote" && + mkdir -p "$lop_remote/objects/info" && + echo "$lop_store/objects" >"$lop_remote/objects/info/alternates" && + git -C "$lop_remote" config uploadpack.allowFilter true && + git -C "$lop_remote" config uploadpack.allowAnySHA1InWant true && + git -C "$server" remote add lop "file://$lop_remote" && + git -C "$server" config remote.lop.promisor true && + git -C "$server" config remote.lop.fetch "+refs/heads/*:refs/remotes/lop/*" && + git -C "$server" config remote.lop.url "file://$lop_remote" && + GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ + -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ + -c remote.lop.url="file://$lop_remote" \ + -c promisor.acceptfromserver=All \ + --no-local --filter="blob:limit=5k" "$server" "$client" && + test_path_is_missing "$client/.git/objects/$(test_oid_to_path $blob_oid)" && + ( + cd "$client" && + git cat-file -p HEAD:huge.bin >../client.bin + ) && + test_cmp lop-producer/huge.bin client.bin && + test_path_is_file "$lop_store/objects/$(test_oid_to_path $blob_oid)" && + git -C "$client" cat-file -e "$blob_oid" ' test_done