From ff9487bce560606fbea5098e7c959bc54d68cbc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Wed, 15 Oct 2025 20:56:19 +0200 Subject: [PATCH 1/8] Add smart clone and repo size summary to LOP demo --- contrib/large-object-promisors/README.md | 24 +++ contrib/large-object-promisors/demo.sh | 206 +++++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 contrib/large-object-promisors/README.md create mode 100755 contrib/large-object-promisors/demo.sh diff --git a/contrib/large-object-promisors/README.md b/contrib/large-object-promisors/README.md new file mode 100644 index 00000000000000..6686bb9946f78d --- /dev/null +++ b/contrib/large-object-promisors/README.md @@ -0,0 +1,24 @@ +# Large Object Promisors demo + +This directory contains `demo.sh`, a self-contained script that builds a throwaway +setup for demonstrating the large object promisor plumbing. It wires a bare server +with two promisor remotes ("small" and "large" stores), installs a post-receive hook +that splits incoming blobs by size, and clones clients that can push and later show +partial-clone behaviour. + +The script expects to be run with a Git binary that already includes the Large Object +Promisors feature (git.git `master` or later). It removes any previous `$DEST` +directory before creating the demo repositories. + +```sh +PATH=/path/to/git/bin-wrappers:/path/to/git:$PATH \ + bash contrib/large-object-promisors/demo.sh +``` + +The `DEST`, `BRANCH`, `THRESHOLD`, and `SEED_INITIAL` environment variables can be +overridden to tweak where the repositories live, which branch is created, the size +threshold used to classify blobs (default 1 MiB), and whether to create the sample +commits. When `SEED_INITIAL=1` the script pushes two large blobs (8 MiB then 2 MiB) +from the first client, clones a fresh `client2` after those pushes to demonstrate a +"smart" clone that only downloads the final 2 MiB blob, and prints a size summary for +each repository. diff --git a/contrib/large-object-promisors/demo.sh b/contrib/large-object-promisors/demo.sh new file mode 100755 index 00000000000000..48b82408e93a7b --- /dev/null +++ b/contrib/large-object-promisors/demo.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +set -euo pipefail + +DEST=${DEST:-$PWD/lop-demo} +BRANCH=${BRANCH:-main} +THRESHOLD=${THRESHOLD:-1048576} +SEED_INITIAL=${SEED_INITIAL:-1} + +say() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; } +abspath() { (cd "$1" >/dev/null 2>&1 && pwd -P) || { echo "ERROR: '$1' not found" >&2; exit 1; }; } +file_url() { printf 'file://%s\n' "$(abspath "$1")"; } + +rm -rf "$DEST" +mkdir -p "$DEST" +ROOT=$(abspath "$DEST") + +LOP_SMALL="$ROOT/lop-small.git" +LOP_LARGE="$ROOT/lop-large.git" +SERVER="$ROOT/server.git" +CLIENT="$ROOT/client" +CLIENT2="$ROOT/client2" + +init_bare() { + git init --bare "$1" >/dev/null + git -C "$1" config uploadpack.allowFilter true + git -C "$1" config uploadpack.allowAnySHA1InWant true +} + +say "Create bare repositories" +init_bare "$LOP_SMALL" +init_bare "$LOP_LARGE" +init_bare "$SERVER" + +SERVER_URL=$(file_url "$SERVER") +LOP_SMALL_URL=$(file_url "$LOP_SMALL") +LOP_LARGE_URL=$(file_url "$LOP_LARGE") + +say "Configure promisor remotes on server" +git -C "$SERVER" config promisor.advertise true +git -C "$SERVER" config promisor.sendFields partialCloneFilter + +config_promisor_remote() { + local repo=$1 name=$2 url=$3 filter=$4 + git -C "$repo" config "remote.${name}.url" "$url" + git -C "$repo" config "remote.${name}.fetch" "+refs/heads/*:refs/remotes/${name}/*" + git -C "$repo" config "remote.${name}.promisor" true + git -C "$repo" config "remote.${name}.partialCloneFilter" "$filter" +} + +config_promisor_remote "$SERVER" lopSmall "$LOP_SMALL_URL" "blob:none" +config_promisor_remote "$SERVER" lopLarge "$LOP_LARGE_URL" "blob:none" + +say "Link server alternates to promisor stores" +mkdir -p "$SERVER/objects/info" +{ + printf '%s/objects\n' "$(abspath "$LOP_LARGE")" + printf '%s/objects\n' "$(abspath "$LOP_SMALL")" +} >"$SERVER/objects/info/alternates" + +git -C "$SERVER" config gc.writeBitmaps false +git -C "$SERVER" config repack.writeBitmaps false + +say "Install post-receive hook" +cat <<'HOOK' >"$SERVER/hooks/post-receive" +#!/usr/bin/env bash +set -euo pipefail + +zeros=0000000000000000000000000000000000000000 +root=$(pwd -P) +small=$(git config --get lop.smallPath) +large=$(git config --get lop.largePath) +threshold=$(git config --get lop.thresholdBytes) + +[ -n "${small:-}" ] && [ -n "${large:-}" ] && [ -n "${threshold:-}" ] || exit 0 + +git -C "$small" config remote.server.url "file://$root" +git -C "$small" config remote.server.fetch "+refs/heads/*:refs/heads/*" +git -C "$large" config remote.server.url "file://$root" +git -C "$large" config remote.server.fetch "+refs/heads/*:refs/heads/*" + +refs=() +tips=() +while read -r old new ref; do + [ "$new" = "$zeros" ] && continue + case "$ref" in + refs/heads/*) + refs+=("+$ref:$ref") + tips+=("$new") + ;; + esac +done + +[ "${#refs[@]}" -eq 0 ] && exit 0 + +git -C "$small" fetch --filter="blob:limit=$threshold" server "${refs[@]}" +git -C "$large" fetch --filter="blob:none" server "${refs[@]}" + +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT + +git rev-list --objects --filter="blob:limit=$threshold" --filter-print-omitted "${tips[@]}" \ + | sed -n 's/^~//p' >"$tmp" +if [ -s "$tmp" ]; then + xargs -r -n256 git -C "$large" fetch server <"$tmp" +fi + +repack() { + git -c repack.writeBitmaps=false -c repack.packKeptObjects=true -C "$root" \ + repack -Ad -d --no-write-bitmap-index "$@" +} + +repack --filter="blob:limit=$threshold" --filter-to="$large/objects" +repack --filter="blob:none" --filter-to="$small/objects" +repack --filter="blob:none" + +purge_blob_packs() { + for idx in "$root/objects/pack"/pack-*.idx; do + [ -e "$idx" ] || continue + base=${idx%.idx} + if [ -f "$base.promisor" ] || git -C "$root" verify-pack -v "$idx" 2>/dev/null | grep -q ' blob '; then + rm -f "$base".idx "$base".pack "$base".rev "$base".promisor 2>/dev/null || true + fi + done +} + +purge_blob_packs +HOOK +chmod +x "$SERVER/hooks/post-receive" + +git -C "$SERVER" config lop.smallPath "$LOP_SMALL" +git -C "$SERVER" config lop.largePath "$LOP_LARGE" +git -C "$SERVER" config lop.thresholdBytes "$THRESHOLD" + +clone_with_promisors() { + local dest=$1 + rm -rf "$dest" + git clone \ + -c promisor.acceptFromServer=All \ + -c remote.lopSmall.promisor=true \ + -c remote.lopSmall.url="$LOP_SMALL_URL" \ + -c remote.lopSmall.fetch="+refs/heads/*:refs/remotes/lopSmall/*" \ + -c remote.lopLarge.promisor=true \ + -c remote.lopLarge.url="$LOP_LARGE_URL" \ + -c remote.lopLarge.fetch="+refs/heads/*:refs/remotes/lopLarge/*" \ + "$SERVER_URL" "$dest" >/dev/null +} + +say "Clone client" +clone_with_promisors "$CLIENT" + +if [ "$SEED_INITIAL" = 1 ]; then + say "Create sample commits" + git -C "$CLIENT" switch -c "$BRANCH" >/dev/null || git -C "$CLIENT" checkout -b "$BRANCH" >/dev/null + git -C "$CLIENT" config user.name "LOP Demo" + git -C "$CLIENT" config user.email "lop-demo@example.invalid" + echo '# LOP demo' >"$CLIENT/README.md" + git -C "$CLIENT" add README.md + git -C "$CLIENT" commit -m "Initial commit" >/dev/null + git -C "$CLIENT" push -u origin "$BRANCH" >/dev/null + + ( + cd "$CLIENT" >/dev/null + add_blob_commit() { + local count=$1 message=$2 + dd if=/dev/urandom of=big-8MiB.bin bs=1M count="$count" status=none + git add big-8MiB.bin + git commit -m "$message" >/dev/null + git push >/dev/null + } + add_blob_commit 8 "Add 8MiB demo blob" + add_blob_commit 2 "Replace with 2MiB blob" + ) + + git -C "$SERVER" symbolic-ref HEAD "refs/heads/$BRANCH" >/dev/null || true + + say "Clone smart client (client2)" + clone_with_promisors "$CLIENT2" +fi + +say "Repository sizes" +du -hs "$SERVER" "$LOP_SMALL" "$LOP_LARGE" "$CLIENT" ${CLIENT2:+"$CLIENT2"} + +cat <} + +Threshold: $THRESHOLD bytes + +Inspect server: + ls -lh "$SERVER/objects/pack" + +Inspect promisor stores: + git -C "$LOP_SMALL" rev-list --objects --all | head + git -C "$LOP_LARGE" rev-list --objects --all | grep big-8MiB.bin || true + +Inspect smart clone: + git -C "$CLIENT2" rev-list --objects --all | grep big-8MiB.bin || true + git -C "$CLIENT2" count-objects -vH +============================================================ +EOF From d41dee8d88a3aef646285d9f9fd38d66263db957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Wed, 15 Oct 2025 21:13:42 +0200 Subject: [PATCH 2/8] Refine smart clone workflow in LOP demo --- contrib/large-object-promisors/README.md | 7 +++--- contrib/large-object-promisors/demo.sh | 30 ++++++++++++++++++------ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/contrib/large-object-promisors/README.md b/contrib/large-object-promisors/README.md index 6686bb9946f78d..5f8669ab91cf00 100644 --- a/contrib/large-object-promisors/README.md +++ b/contrib/large-object-promisors/README.md @@ -19,6 +19,7 @@ The `DEST`, `BRANCH`, `THRESHOLD`, and `SEED_INITIAL` environment variables can overridden to tweak where the repositories live, which branch is created, the size threshold used to classify blobs (default 1 MiB), and whether to create the sample commits. When `SEED_INITIAL=1` the script pushes two large blobs (8 MiB then 2 MiB) -from the first client, clones a fresh `client2` after those pushes to demonstrate a -"smart" clone that only downloads the final 2 MiB blob, and prints a size summary for -each repository. +from the first client, clones a fresh `client2` after those pushes with +`--filter=blob:none`, explicitly prefetches just the tip blob from the large +promisor store, and prints a size summary for each repository so you can compare how +much data landed in the smart clone. diff --git a/contrib/large-object-promisors/demo.sh b/contrib/large-object-promisors/demo.sh index 48b82408e93a7b..1035965e71da22 100755 --- a/contrib/large-object-promisors/demo.sh +++ b/contrib/large-object-promisors/demo.sh @@ -40,15 +40,16 @@ git -C "$SERVER" config promisor.advertise true git -C "$SERVER" config promisor.sendFields partialCloneFilter config_promisor_remote() { - local repo=$1 name=$2 url=$3 filter=$4 + local repo=$1 name=$2 url=$3 git -C "$repo" config "remote.${name}.url" "$url" git -C "$repo" config "remote.${name}.fetch" "+refs/heads/*:refs/remotes/${name}/*" git -C "$repo" config "remote.${name}.promisor" true - git -C "$repo" config "remote.${name}.partialCloneFilter" "$filter" } -config_promisor_remote "$SERVER" lopSmall "$LOP_SMALL_URL" "blob:none" -config_promisor_remote "$SERVER" lopLarge "$LOP_LARGE_URL" "blob:none" +config_promisor_remote "$SERVER" lopSmall "$LOP_SMALL_URL" +config_promisor_remote "$SERVER" lopLarge "$LOP_LARGE_URL" +git -C "$SERVER" config --add promisor.remote lopSmall +git -C "$SERVER" config --add promisor.remote lopLarge say "Link server alternates to promisor stores" mkdir -p "$SERVER/objects/info" @@ -132,8 +133,15 @@ git -C "$SERVER" config lop.largePath "$LOP_LARGE" git -C "$SERVER" config lop.thresholdBytes "$THRESHOLD" clone_with_promisors() { - local dest=$1 + local dest=$1 filter=${2:-} no_checkout=${3:-0} rm -rf "$dest" + local clone_args=("$SERVER_URL" "$dest") + if [ -n "$filter" ]; then + clone_args=("--filter=$filter" "${clone_args[@]}") + fi + if [ "$no_checkout" = 1 ]; then + clone_args=(--no-checkout "${clone_args[@]}") + fi git clone \ -c promisor.acceptFromServer=All \ -c remote.lopSmall.promisor=true \ @@ -142,7 +150,10 @@ clone_with_promisors() { -c remote.lopLarge.promisor=true \ -c remote.lopLarge.url="$LOP_LARGE_URL" \ -c remote.lopLarge.fetch="+refs/heads/*:refs/remotes/lopLarge/*" \ - "$SERVER_URL" "$dest" >/dev/null + "${clone_args[@]}" >/dev/null + if [ -n "$filter" ]; then + git -C "$dest" config extensions.partialclone origin + fi } say "Clone client" @@ -174,7 +185,12 @@ if [ "$SEED_INITIAL" = 1 ]; then git -C "$SERVER" symbolic-ref HEAD "refs/heads/$BRANCH" >/dev/null || true say "Clone smart client (client2)" - clone_with_promisors "$CLIENT2" + clone_with_promisors "$CLIENT2" "blob:none" 1 + latest_blob=$(git -C "$CLIENT2" ls-tree "origin/$BRANCH" -- big-8MiB.bin | awk '{print $3}') + if [ -n "$latest_blob" ]; then + git -C "$CLIENT2" fetch lopLarge "$latest_blob" >/dev/null + fi + git -C "$CLIENT2" checkout "$BRANCH" >/dev/null fi say "Repository sizes" From 865ad7596ad70b56fb91b528ecad7aba95809a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Thu, 16 Oct 2025 06:53:46 +0200 Subject: [PATCH 3/8] Streamline promisor setup in demo --- contrib/large-object-promisors/README.md | 8 +++--- contrib/large-object-promisors/demo.sh | 34 +++++++----------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/contrib/large-object-promisors/README.md b/contrib/large-object-promisors/README.md index 5f8669ab91cf00..497a7e562df657 100644 --- a/contrib/large-object-promisors/README.md +++ b/contrib/large-object-promisors/README.md @@ -20,6 +20,8 @@ overridden to tweak where the repositories live, which branch is created, the si threshold used to classify blobs (default 1 MiB), and whether to create the sample commits. When `SEED_INITIAL=1` the script pushes two large blobs (8 MiB then 2 MiB) from the first client, clones a fresh `client2` after those pushes with -`--filter=blob:none`, explicitly prefetches just the tip blob from the large -promisor store, and prints a size summary for each repository so you can compare how -much data landed in the smart clone. +`--filter=blob:none`, checks out the branch to trigger a lazy fetch of just the +tip blob from the large promisor store, and prints a size summary for each +repository so you can compare how much data landed in the smart clone. The +promisor configuration itself now flows from the server, so the clone commands +shown in the script mirror what an ordinary user would type. diff --git a/contrib/large-object-promisors/demo.sh b/contrib/large-object-promisors/demo.sh index 1035965e71da22..33180933286fb5 100755 --- a/contrib/large-object-promisors/demo.sh +++ b/contrib/large-object-promisors/demo.sh @@ -51,6 +51,12 @@ config_promisor_remote "$SERVER" lopLarge "$LOP_LARGE_URL" git -C "$SERVER" config --add promisor.remote lopSmall git -C "$SERVER" config --add promisor.remote lopLarge +say "Configure promisor stores" +git -C "$LOP_SMALL" config remote.server.url "$SERVER_URL" +git -C "$LOP_SMALL" config remote.server.fetch "+refs/heads/*:refs/heads/*" +git -C "$LOP_LARGE" config remote.server.url "$SERVER_URL" +git -C "$LOP_LARGE" config remote.server.fetch "+refs/heads/*:refs/heads/*" + say "Link server alternates to promisor stores" mkdir -p "$SERVER/objects/info" { @@ -74,11 +80,6 @@ threshold=$(git config --get lop.thresholdBytes) [ -n "${small:-}" ] && [ -n "${large:-}" ] && [ -n "${threshold:-}" ] || exit 0 -git -C "$small" config remote.server.url "file://$root" -git -C "$small" config remote.server.fetch "+refs/heads/*:refs/heads/*" -git -C "$large" config remote.server.url "file://$root" -git -C "$large" config remote.server.fetch "+refs/heads/*:refs/heads/*" - refs=() tips=() while read -r old new ref; do @@ -135,25 +136,14 @@ git -C "$SERVER" config lop.thresholdBytes "$THRESHOLD" clone_with_promisors() { local dest=$1 filter=${2:-} no_checkout=${3:-0} rm -rf "$dest" - local clone_args=("$SERVER_URL" "$dest") + local clone_args=() if [ -n "$filter" ]; then - clone_args=("--filter=$filter" "${clone_args[@]}") + clone_args+=("--filter=$filter") fi if [ "$no_checkout" = 1 ]; then - clone_args=(--no-checkout "${clone_args[@]}") - fi - git clone \ - -c promisor.acceptFromServer=All \ - -c remote.lopSmall.promisor=true \ - -c remote.lopSmall.url="$LOP_SMALL_URL" \ - -c remote.lopSmall.fetch="+refs/heads/*:refs/remotes/lopSmall/*" \ - -c remote.lopLarge.promisor=true \ - -c remote.lopLarge.url="$LOP_LARGE_URL" \ - -c remote.lopLarge.fetch="+refs/heads/*:refs/remotes/lopLarge/*" \ - "${clone_args[@]}" >/dev/null - if [ -n "$filter" ]; then - git -C "$dest" config extensions.partialclone origin + clone_args+=(--no-checkout) fi + git clone "${clone_args[@]}" "$SERVER_URL" "$dest" >/dev/null } say "Clone client" @@ -186,10 +176,6 @@ if [ "$SEED_INITIAL" = 1 ]; then say "Clone smart client (client2)" clone_with_promisors "$CLIENT2" "blob:none" 1 - latest_blob=$(git -C "$CLIENT2" ls-tree "origin/$BRANCH" -- big-8MiB.bin | awk '{print $3}') - if [ -n "$latest_blob" ]; then - git -C "$CLIENT2" fetch lopLarge "$latest_blob" >/dev/null - fi git -C "$CLIENT2" checkout "$BRANCH" >/dev/null fi From 3b5a0881f05d35aa79a53d90d61f0fb188675890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Thu, 16 Oct 2025 06:53:51 +0200 Subject: [PATCH 4/8] Populate small LOP store and simplify hook --- contrib/large-object-promisors/README.md | 15 ++--- contrib/large-object-promisors/demo.sh | 75 ++++++++++++++++-------- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/contrib/large-object-promisors/README.md b/contrib/large-object-promisors/README.md index 497a7e562df657..5268698f75c6fa 100644 --- a/contrib/large-object-promisors/README.md +++ b/contrib/large-object-promisors/README.md @@ -18,10 +18,11 @@ PATH=/path/to/git/bin-wrappers:/path/to/git:$PATH \ The `DEST`, `BRANCH`, `THRESHOLD`, and `SEED_INITIAL` environment variables can be overridden to tweak where the repositories live, which branch is created, the size threshold used to classify blobs (default 1 MiB), and whether to create the sample -commits. When `SEED_INITIAL=1` the script pushes two large blobs (8 MiB then 2 MiB) -from the first client, clones a fresh `client2` after those pushes with -`--filter=blob:none`, checks out the branch to trigger a lazy fetch of just the -tip blob from the large promisor store, and prints a size summary for each -repository so you can compare how much data landed in the smart clone. The -promisor configuration itself now flows from the server, so the clone commands -shown in the script mirror what an ordinary user would type. +commits. When `SEED_INITIAL=1` the script pushes two large blobs (first oversized, +then a replacement that still exceeds the threshold) and two small blobs (both +below the threshold) from the first client. After those pushes it clones a fresh +`client2` with `--filter=blob:none`, checks out the branch to trigger a lazy fetch +of only the tip's large blob from the large promisor store, and prints a size +summary for each repository so you can compare where the large and small histories +ended up. The promisor configuration itself now flows from the server, so the +clone commands shown in the script mirror what an ordinary user would type. diff --git a/contrib/large-object-promisors/demo.sh b/contrib/large-object-promisors/demo.sh index 33180933286fb5..ad7c065e400c93 100755 --- a/contrib/large-object-promisors/demo.sh +++ b/contrib/large-object-promisors/demo.sh @@ -106,26 +106,17 @@ if [ -s "$tmp" ]; then xargs -r -n256 git -C "$large" fetch server <"$tmp" fi -repack() { - git -c repack.writeBitmaps=false -c repack.packKeptObjects=true -C "$root" \ - repack -Ad -d --no-write-bitmap-index "$@" -} +git -c repack.writeBitmaps=false -c repack.packKeptObjects=true -C "$root" \ + repack -Ad -d --no-write-bitmap-index \ + --filter="blob:limit=$threshold" \ + --filter-to="$large/objects" -repack --filter="blob:limit=$threshold" --filter-to="$large/objects" -repack --filter="blob:none" --filter-to="$small/objects" -repack --filter="blob:none" - -purge_blob_packs() { - for idx in "$root/objects/pack"/pack-*.idx; do - [ -e "$idx" ] || continue - base=${idx%.idx} - if [ -f "$base.promisor" ] || git -C "$root" verify-pack -v "$idx" 2>/dev/null | grep -q ' blob '; then - rm -f "$base".idx "$base".pack "$base".rev "$base".promisor 2>/dev/null || true - fi - done -} +git -c repack.writeBitmaps=false -c repack.packKeptObjects=true -C "$root" \ + repack -Ad -d --no-write-bitmap-index \ + --filter="blob:none" \ + --filter-to="$small/objects" -purge_blob_packs +git -C "$root" prune-packed HOOK chmod +x "$SERVER/hooks/post-receive" @@ -161,15 +152,53 @@ if [ "$SEED_INITIAL" = 1 ]; then ( cd "$CLIENT" >/dev/null + + write_random_blob() { + local path=$1 bytes=$2 + dd if=/dev/urandom of="$path" bs="$bytes" count=1 status=none + } + + ensure_above_threshold() { + local size=$1 threshold=$2 + while [ "$size" -le "$threshold" ]; do + size=$((size + 1024 * 1024)) + done + printf '%s' "$size" + } + + ensure_below_threshold() { + local size=$1 threshold=$2 + if [ "$threshold" -le 1 ]; then + printf '1' + return + fi + if [ "$size" -ge "$threshold" ]; then + size=$((threshold - 1)) + [ "$size" -lt 1 ] && size=1 + fi + printf '%s' "$size" + } + add_blob_commit() { - local count=$1 message=$2 - dd if=/dev/urandom of=big-8MiB.bin bs=1M count="$count" status=none - git add big-8MiB.bin + local path=$1 bytes=$2 message=$3 + write_random_blob "$path" "$bytes" + git add "$path" git commit -m "$message" >/dev/null git push >/dev/null } - add_blob_commit 8 "Add 8MiB demo blob" - add_blob_commit 2 "Replace with 2MiB blob" + + big_path=big-demo.bin + big_bytes_1=$(ensure_above_threshold $((8 * 1024 * 1024)) "$THRESHOLD") + big_bytes_2=$(ensure_above_threshold $((2 * 1024 * 1024)) "$THRESHOLD") + add_blob_commit "$big_path" "$big_bytes_1" "Add large demo blob" + add_blob_commit "$big_path" "$big_bytes_2" "Replace with newer large blob" + + small_path=small-demo.bin + small_bytes_1=$(ensure_below_threshold $((512 * 1024)) "$THRESHOLD") + small_bytes_2=$(ensure_below_threshold $((256 * 1024)) "$THRESHOLD") + [ "$small_bytes_2" -ge "$small_bytes_1" ] && small_bytes_2=$((small_bytes_1 > 1 ? small_bytes_1 - 1 : 1)) + add_blob_commit "$small_path" "$small_bytes_1" "Add small demo blob" + add_blob_commit "$small_path" "$small_bytes_2" "Replace with newer small blob" ) git -C "$SERVER" symbolic-ref HEAD "refs/heads/$BRANCH" >/dev/null || true From 2966982e44e95bfea0c0bb3c2055b1c134298e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Thu, 16 Oct 2025 07:43:46 +0200 Subject: [PATCH 5/8] Echo clone commands in LOP demo --- contrib/large-object-promisors/demo.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/large-object-promisors/demo.sh b/contrib/large-object-promisors/demo.sh index ad7c065e400c93..4a1537a1b07217 100755 --- a/contrib/large-object-promisors/demo.sh +++ b/contrib/large-object-promisors/demo.sh @@ -134,6 +134,11 @@ clone_with_promisors() { if [ "$no_checkout" = 1 ]; then clone_args+=(--no-checkout) fi + local display_opts="" + if [ "${#clone_args[@]}" -gt 0 ]; then + printf -v display_opts ' %q' "${clone_args[@]}" + fi + printf ' git clone%s %q %q\n' "$display_opts" "$SERVER_URL" "$dest" git clone "${clone_args[@]}" "$SERVER_URL" "$dest" >/dev/null } From c44e5e7a9f0593b6d8ddb98d137b7197ca0def80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Thu, 16 Oct 2025 14:13:48 +0200 Subject: [PATCH 6/8] Teach LOP demo clones to persist promisor remotes --- contrib/large-object-promisors/README.md | 6 ++++-- contrib/large-object-promisors/demo.sh | 21 +++++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/contrib/large-object-promisors/README.md b/contrib/large-object-promisors/README.md index 5268698f75c6fa..9f6da46db52366 100644 --- a/contrib/large-object-promisors/README.md +++ b/contrib/large-object-promisors/README.md @@ -24,5 +24,7 @@ below the threshold) from the first client. After those pushes it clones a fresh `client2` with `--filter=blob:none`, checks out the branch to trigger a lazy fetch of only the tip's large blob from the large promisor store, and prints a size summary for each repository so you can compare where the large and small histories -ended up. The promisor configuration itself now flows from the server, so the -clone commands shown in the script mirror what an ordinary user would type. +ended up. Each clone passes `-c promisor.acceptFromServer=All` and the +`remote.lop{Small,Large}.*` configuration so the promisor remotes are persisted +without any manual follow-up, and the script prints the full `git clone` command +for clarity during the demo. diff --git a/contrib/large-object-promisors/demo.sh b/contrib/large-object-promisors/demo.sh index 4a1537a1b07217..a743fed5393828 100755 --- a/contrib/large-object-promisors/demo.sh +++ b/contrib/large-object-promisors/demo.sh @@ -40,14 +40,17 @@ git -C "$SERVER" config promisor.advertise true git -C "$SERVER" config promisor.sendFields partialCloneFilter config_promisor_remote() { - local repo=$1 name=$2 url=$3 + local repo=$1 name=$2 url=$3 filter=${4:-} git -C "$repo" config "remote.${name}.url" "$url" git -C "$repo" config "remote.${name}.fetch" "+refs/heads/*:refs/remotes/${name}/*" git -C "$repo" config "remote.${name}.promisor" true + if [ -n "$filter" ]; then + git -C "$repo" config "remote.${name}.partialCloneFilter" "$filter" + fi } -config_promisor_remote "$SERVER" lopSmall "$LOP_SMALL_URL" -config_promisor_remote "$SERVER" lopLarge "$LOP_LARGE_URL" +config_promisor_remote "$SERVER" lopSmall "$LOP_SMALL_URL" "blob:limit=$THRESHOLD" +config_promisor_remote "$SERVER" lopLarge "$LOP_LARGE_URL" blob:none git -C "$SERVER" config --add promisor.remote lopSmall git -C "$SERVER" config --add promisor.remote lopLarge @@ -127,7 +130,17 @@ git -C "$SERVER" config lop.thresholdBytes "$THRESHOLD" clone_with_promisors() { local dest=$1 filter=${2:-} no_checkout=${3:-0} rm -rf "$dest" - local clone_args=() + local clone_args=(-c promisor.acceptFromServer=All) + clone_args+=( + -c remote.lopSmall.promisor=true + -c "remote.lopSmall.url=$LOP_SMALL_URL" + -c "remote.lopSmall.fetch=+refs/heads/*:refs/remotes/lopSmall/*" + -c "remote.lopSmall.partialCloneFilter=blob:limit=$THRESHOLD" + -c remote.lopLarge.promisor=true + -c "remote.lopLarge.url=$LOP_LARGE_URL" + -c "remote.lopLarge.fetch=+refs/heads/*:refs/remotes/lopLarge/*" + -c remote.lopLarge.partialCloneFilter=blob:none + ) if [ -n "$filter" ]; then clone_args+=("--filter=$filter") fi From 8f1cf19b14edd7aba8aaf01e500940b4d6e25304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Thu, 16 Oct 2025 15:05:05 +0200 Subject: [PATCH 7/8] Rely on advertised promisor remotes in demo clones --- contrib/large-object-promisors/README.md | 11 +++--- contrib/large-object-promisors/demo.sh | 43 ++++++++++++++++++------ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/contrib/large-object-promisors/README.md b/contrib/large-object-promisors/README.md index 9f6da46db52366..4ea86e67d81e0e 100644 --- a/contrib/large-object-promisors/README.md +++ b/contrib/large-object-promisors/README.md @@ -24,7 +24,10 @@ below the threshold) from the first client. After those pushes it clones a fresh `client2` with `--filter=blob:none`, checks out the branch to trigger a lazy fetch of only the tip's large blob from the large promisor store, and prints a size summary for each repository so you can compare where the large and small histories -ended up. Each clone passes `-c promisor.acceptFromServer=All` and the -`remote.lop{Small,Large}.*` configuration so the promisor remotes are persisted -without any manual follow-up, and the script prints the full `git clone` command -for clarity during the demo. +ended up. The script writes a throwaway global config that sets +`promisor.acceptFromServer=All`, allowing the clones it performs to accept the +promisor remotes advertised by the server without spelling out `-c remote.*` +arguments. When run with an older Git that lacks the advertisement support the +script emits a warning and copies the promisor-remote configuration directly +from the server so the rest of the demo still succeeds. The script also prints +the full `git clone` command for clarity during the demo. diff --git a/contrib/large-object-promisors/demo.sh b/contrib/large-object-promisors/demo.sh index a743fed5393828..216c4d30783ce2 100755 --- a/contrib/large-object-promisors/demo.sh +++ b/contrib/large-object-promisors/demo.sh @@ -20,6 +20,14 @@ SERVER="$ROOT/server.git" CLIENT="$ROOT/client" CLIENT2="$ROOT/client2" +say "Prepare demo-wide Git config" +DEMO_CONFIG="$ROOT/gitconfig" +cat >"$DEMO_CONFIG" <<'EOF' +[promisor] +acceptFromServer = All +EOF +export GIT_CONFIG_GLOBAL="$DEMO_CONFIG" + init_bare() { git init --bare "$1" >/dev/null git -C "$1" config uploadpack.allowFilter true @@ -130,17 +138,7 @@ git -C "$SERVER" config lop.thresholdBytes "$THRESHOLD" clone_with_promisors() { local dest=$1 filter=${2:-} no_checkout=${3:-0} rm -rf "$dest" - local clone_args=(-c promisor.acceptFromServer=All) - clone_args+=( - -c remote.lopSmall.promisor=true - -c "remote.lopSmall.url=$LOP_SMALL_URL" - -c "remote.lopSmall.fetch=+refs/heads/*:refs/remotes/lopSmall/*" - -c "remote.lopSmall.partialCloneFilter=blob:limit=$THRESHOLD" - -c remote.lopLarge.promisor=true - -c "remote.lopLarge.url=$LOP_LARGE_URL" - -c "remote.lopLarge.fetch=+refs/heads/*:refs/remotes/lopLarge/*" - -c remote.lopLarge.partialCloneFilter=blob:none - ) + local clone_args=() if [ -n "$filter" ]; then clone_args+=("--filter=$filter") fi @@ -155,8 +153,29 @@ clone_with_promisors() { git clone "${clone_args[@]}" "$SERVER_URL" "$dest" >/dev/null } +ensure_remote_advertised() { + local repo=$1 remote=$2 + if ! git -C "$repo" config --get "remote.${remote}.url" >/dev/null 2>&1; then + echo "WARNING: promisor remote '$remote' was not advertised to $repo; populating from server config" >&2 + local url filter fetch + url=$(git -C "$SERVER" config --get "remote.${remote}.url") + fetch=$(git -C "$SERVER" config --get "remote.${remote}.fetch") + filter=$(git -C "$SERVER" config --get "remote.${remote}.partialCloneFilter" || true) + git -C "$repo" config "remote.${remote}.url" "$url" + git -C "$repo" config "remote.${remote}.fetch" "$fetch" + git -C "$repo" config "remote.${remote}.promisor" true + if [ -n "$filter" ]; then + git -C "$repo" config "remote.${remote}.partialCloneFilter" "$filter" + fi + return + fi + git -C "$repo" config --get "remote.${remote}.promisor" >/dev/null +} + say "Clone client" clone_with_promisors "$CLIENT" +ensure_remote_advertised "$CLIENT" lopSmall +ensure_remote_advertised "$CLIENT" lopLarge if [ "$SEED_INITIAL" = 1 ]; then say "Create sample commits" @@ -224,6 +243,8 @@ if [ "$SEED_INITIAL" = 1 ]; then say "Clone smart client (client2)" clone_with_promisors "$CLIENT2" "blob:none" 1 git -C "$CLIENT2" checkout "$BRANCH" >/dev/null + ensure_remote_advertised "$CLIENT2" lopSmall + ensure_remote_advertised "$CLIENT2" lopLarge fi say "Repository sizes" From b2a7c7aad2067a202dcbb669ad830ab3648cb3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Thu, 16 Oct 2025 15:56:59 +0200 Subject: [PATCH 8/8] Require promisor remotes via protocol advertisement --- contrib/large-object-promisors/README.md | 10 +++++---- contrib/large-object-promisors/demo.sh | 27 +++++++++--------------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/contrib/large-object-promisors/README.md b/contrib/large-object-promisors/README.md index 4ea86e67d81e0e..9a8d0b9d543560 100644 --- a/contrib/large-object-promisors/README.md +++ b/contrib/large-object-promisors/README.md @@ -27,7 +27,9 @@ summary for each repository so you can compare where the large and small histori ended up. The script writes a throwaway global config that sets `promisor.acceptFromServer=All`, allowing the clones it performs to accept the promisor remotes advertised by the server without spelling out `-c remote.*` -arguments. When run with an older Git that lacks the advertisement support the -script emits a warning and copies the promisor-remote configuration directly -from the server so the rest of the demo still succeeds. The script also prints -the full `git clone` command for clarity during the demo. +arguments. Each clone forces protocol v2 (`-c protocol.version=2`) so the +`promisor-remote` capability is available. The script validates that both +promisor remotes arrive via the +advertisement protocol (see `t/t5710` for the low-level coverage) so you know +you are running with a recent enough Git. It also prints the full `git clone` +command for clarity during the demo. diff --git a/contrib/large-object-promisors/demo.sh b/contrib/large-object-promisors/demo.sh index 216c4d30783ce2..d3f2927edeb372 100755 --- a/contrib/large-object-promisors/demo.sh +++ b/contrib/large-object-promisors/demo.sh @@ -139,6 +139,7 @@ clone_with_promisors() { local dest=$1 filter=${2:-} no_checkout=${3:-0} rm -rf "$dest" local clone_args=() + clone_args+=("-c" "protocol.version=2") if [ -n "$filter" ]; then clone_args+=("--filter=$filter") fi @@ -153,29 +154,21 @@ clone_with_promisors() { git clone "${clone_args[@]}" "$SERVER_URL" "$dest" >/dev/null } -ensure_remote_advertised() { +require_advertised_remote() { local repo=$1 remote=$2 if ! git -C "$repo" config --get "remote.${remote}.url" >/dev/null 2>&1; then - echo "WARNING: promisor remote '$remote' was not advertised to $repo; populating from server config" >&2 - local url filter fetch - url=$(git -C "$SERVER" config --get "remote.${remote}.url") - fetch=$(git -C "$SERVER" config --get "remote.${remote}.fetch") - filter=$(git -C "$SERVER" config --get "remote.${remote}.partialCloneFilter" || true) - git -C "$repo" config "remote.${remote}.url" "$url" - git -C "$repo" config "remote.${remote}.fetch" "$fetch" - git -C "$repo" config "remote.${remote}.promisor" true - if [ -n "$filter" ]; then - git -C "$repo" config "remote.${remote}.partialCloneFilter" "$filter" - fi - return + cat <&2 +ERROR: promisor remote '$remote' was not advertised to $repo. + Ensure your Git is built from a revision that includes the + Large Object Promisors advertisement protocol (see t/t5710). +EOF + exit 1 fi git -C "$repo" config --get "remote.${remote}.promisor" >/dev/null } say "Clone client" clone_with_promisors "$CLIENT" -ensure_remote_advertised "$CLIENT" lopSmall -ensure_remote_advertised "$CLIENT" lopLarge if [ "$SEED_INITIAL" = 1 ]; then say "Create sample commits" @@ -243,8 +236,8 @@ if [ "$SEED_INITIAL" = 1 ]; then say "Clone smart client (client2)" clone_with_promisors "$CLIENT2" "blob:none" 1 git -C "$CLIENT2" checkout "$BRANCH" >/dev/null - ensure_remote_advertised "$CLIENT2" lopSmall - ensure_remote_advertised "$CLIENT2" lopLarge + require_advertised_remote "$CLIENT2" lopSmall + require_advertised_remote "$CLIENT2" lopLarge fi say "Repository sizes"