Add REAPI content-defined chunking (SplitBlob/SpliceBlob) support

BUILD.bazel

@@ -133,3 +133,12 @@ sh_test(
     timeout = "short",
     srcs = [":dummy_test_sh"],
 )
+
+# Large deterministic output used by integration_tests/chunking_cache_test.sh
+# to exercise --experimental_remote_cache_chunking (must exceed the chunking
+# threshold of 4x the average chunk size).
+genrule(
+    name = "chunking_test_artifact",
+    outs = ["chunking_test_artifact.txt"],
+    cmd = "seq 1 1000000 > \"$@\"",
+)

deployment-examples/docker-compose/local-storage-cas.json5

@@ -13,8 +13,8 @@
         },
         backend: {
           filesystem: {
-            content_path: "~/.cache/nativelink/content_path-cas",
-            temp_path: "~/.cache/nativelink/tmp_path-cas",
+            content_path: "/root/.cache/nativelink/content_path-cas",
+            temp_path: "/root/.cache/nativelink/tmp_path-cas",
             eviction_policy: {
               // 10gb.
               max_bytes: 10000000000,
@@ -23,11 +23,25 @@
         },
       },
     },
+    {
+      // Holds blob-to-chunks layouts for the SplitBlob/SpliceBlob RPCs used
+      // by Bazel's --experimental_remote_cache_chunking. Must not verify
+      // digests and must not be the same store as the CAS.
+      name: "CHUNK_INDEX_STORE",
+      filesystem: {
+        content_path: "/root/.cache/nativelink/content_path-chunk-index",
+        temp_path: "/root/.cache/nativelink/tmp_path-chunk-index",
+        eviction_policy: {
+          // 100mb.
+          max_bytes: 100000000,
+        },
+      },
+    },
     {
       name: "AC_MAIN_STORE",
       filesystem: {
-        content_path: "~/.cache/nativelink/content_path-ac",
-        temp_path: "~/.cache/nativelink/tmp_path-ac",
+        content_path: "/root/.cache/nativelink/content_path-ac",
+        temp_path: "/root/.cache/nativelink/tmp_path-ac",
         eviction_policy: {
           // 500mb.
           max_bytes: 500000000,
@@ -46,6 +60,13 @@
         cas: [
           {
             cas_store: "CAS_MAIN_STORE",
+
+            // Optional: enables content-defined chunking
+            // (SplitBlob/SpliceBlob) for Bazel clients running with
+            // --experimental_remote_cache_chunking.
+            experimental_chunking: {
+              index_store: "CHUNK_INDEX_STORE",
+            },
           },
         ],
         ac: [

integration_tests/chunking_cache_test.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Copyright 2026 The NativeLink Authors. All rights reserved.
+#
+# Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    See LICENSE file for details
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sanity check for REAPI content-defined chunking: uploads a large blob via
+# SpliceBlob (Bazel --experimental_remote_cache_chunking), verifies the
+# server registered a chunk layout, then re-fetches the blob from the remote
+# cache via the chunked download path and checks it is byte-identical.
+
+if [[ $UNDER_TEST_RUNNER -ne 1 ]]; then
+    echo "This script should be run under run_integration_tests.sh"
+    exit 1
+fi
+set -x
+
+# Bazel uploads cache entries in the background by default
+# (--remote_cache_async, on since Bazel 8), so a build can return before the
+# chunked upload and SpliceBlob complete. Force synchronous uploads so the
+# chunk-index assertions below cannot race the upload.
+CHUNKING_FLAGS=(--config self_test --experimental_remote_cache_chunking --remote_cache_async=false)
+EXPECTED_SHA=$(seq 1 1000000 | sha256sum | awk '{print $1}')
+# The test runner's working directory is not the workspace root, so resolve
+# the output location through bazel itself.
+ARTIFACT="$(bazel --output_base="$BAZEL_CACHE_DIR" info "${CHUNKING_FLAGS[@]}" bazel-bin)/chunking_test_artifact.txt"
+
+# First build executes the action locally and uploads the ~6.9MB output as
+# chunks (SpliceBlob).
+bazel --output_base="$BAZEL_CACHE_DIR" build "${CHUNKING_FLAGS[@]}" //:chunking_test_artifact
+FIRST_SHA=$(sha256sum "$ARTIFACT" | awk '{print $1}')
+if [[ $FIRST_SHA != "$EXPECTED_SHA" ]]; then
+    echo "Expected locally built artifact to have sha $EXPECTED_SHA, got $FIRST_SHA."
+    exit 1
+fi
+
+# The server must have registered a chunk layout for the spliced blob. The
+# index store is mounted from the host by docker-compose.
+CHUNK_INDEX_DIR="${NATIVELINK_DIR:-$HOME/.cache/nativelink}/content_path-chunk-index"
+if [[ -z $(find "$CHUNK_INDEX_DIR" -type f 2> /dev/null) ]]; then
+    echo "Expected a chunk layout in $CHUNK_INDEX_DIR after a chunked upload."
+    echo "SpliceBlob was likely not used; check that the server advertises"
+    echo "chunking support and that bazel supports the chunking flag."
+    echo "Diagnostics: contents of the mounted cache dir and root's default:"
+    sudo find "${NATIVELINK_DIR:-$HOME/.cache/nativelink}" -maxdepth 1 2> /dev/null || true
+    sudo find /root/.cache/nativelink -maxdepth 1 2> /dev/null || true
+    exit 1
+fi
+
+# Clean our local cache and re-fetch from the remote cache through the
+# chunked download path.
+bazel --output_base="$BAZEL_CACHE_DIR" clean
+OUTPUT=$(bazel --output_base="$BAZEL_CACHE_DIR" build "${CHUNKING_FLAGS[@]}" //:chunking_test_artifact 2>&1)
+if [[ ! $OUTPUT =~ 'remote cache hit' ]]; then
+    echo "Expected second bazel run to be a remote cache hit."
+    echo "STDOUT:"
+    echo "$OUTPUT"
+    exit 1
+fi
+SECOND_SHA=$(sha256sum "$ARTIFACT" | awk '{print $1}')
+if [[ $SECOND_SHA != "$EXPECTED_SHA" ]]; then
+    echo "Artifact fetched through the chunked download path is corrupt:"
+    echo "expected sha $EXPECTED_SHA, got $SECOND_SHA."
+    exit 1
+fi

nativelink-config/examples/chunking_cas.json5

@@ -0,0 +1,105 @@
+// Demonstrates REAPI content-defined chunking: the SplitBlob/SpliceBlob
+// RPCs used by Bazel's --experimental_remote_cache_chunking flag
+// (available in Bazel 8.7.0+ / 9.1.0+).
+//
+// Chunking is entirely optional and disabled by default: without the
+// `experimental_chunking` block below, NativeLink behaves exactly as
+// before and does not advertise chunking support. When enabled, clients
+// upload and download large blobs as content-defined chunks, so small
+// changes to large outputs only transfer the chunks that changed.
+{
+  stores: [
+    {
+      name: "CAS_MAIN_STORE",
+      filesystem: {
+        content_path: "/tmp/nativelink/data/content_path-cas",
+        temp_path: "/tmp/nativelink/data/tmp_path-cas",
+        eviction_policy: {
+          // 10gb.
+          max_bytes: 10000000000,
+        },
+      },
+    },
+    {
+      // Holds the blob-to-chunks layouts registered via SpliceBlob or
+      // created by on-demand chunking in SplitBlob. Layout entries are
+      // small (roughly 80-140 bytes per chunk). This store must not verify
+      // content digests and must not be the same store as the CAS itself.
+      name: "CHUNK_INDEX_STORE",
+      filesystem: {
+        content_path: "/tmp/nativelink/data/content_path-chunk-index",
+        temp_path: "/tmp/nativelink/data/tmp_path-chunk-index",
+        eviction_policy: {
+          // 100mb.
+          max_bytes: 100000000,
+        },
+      },
+    },
+    {
+      name: "AC_MAIN_STORE",
+      filesystem: {
+        content_path: "/tmp/nativelink/data/content_path-ac",
+        temp_path: "/tmp/nativelink/data/tmp_path-ac",
+        eviction_policy: {
+          // 500mb.
+          max_bytes: 500000000,
+        },
+      },
+    },
+  ],
+  servers: [
+    {
+      listener: {
+        http: {
+          socket_address: "0.0.0.0:50051",
+        },
+      },
+      services: {
+        cas: [
+          {
+            instance_name: "main",
+            cas_store: "CAS_MAIN_STORE",
+
+            // Optional: omit this block to disable chunking entirely.
+            experimental_chunking: {
+              // Required, unless `cas_store` is a grpc store — in that
+              // case the chunking RPCs are forwarded to the backend and
+              // `index_store` must be omitted.
+              index_store: "CHUNK_INDEX_STORE",
+
+              // Optional: the average chunk size in bytes advertised to
+              // clients and used for server-side chunking. Must be between
+              // 1 KiB and 1 MiB.
+              // Default: 524288 (512 KiB).
+              avg_chunk_size_bytes: 524288,
+
+              // Optional: blobs that would produce more chunks than this
+              // are served without chunking.
+              // Default: 50000.
+              max_chunk_count: 50000,
+            },
+          },
+        ],
+        ac: [
+          {
+            instance_name: "main",
+            ac_store: "AC_MAIN_STORE",
+          },
+        ],
+
+        // The capabilities service advertises chunking support; Bazel only
+        // issues SplitBlob/SpliceBlob when it is advertised.
+        capabilities: [
+          {
+            instance_name: "main",
+          },
+        ],
+        bytestream: {
+          cas_stores: {
+            main: "CAS_MAIN_STORE",
+          },
+        },
+      },
+    },
+  ],
+}