Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/main/java/com/google/devtools/build/lib/remote/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ package(
filegroup(
name = "srcs",
srcs = glob(["*"]) + [
"//src/main/java/com/google/devtools/build/lib/remote/chunking:srcs",
"//src/main/java/com/google/devtools/build/lib/remote/circuitbreaker:srcs",
"//src/main/java/com/google/devtools/build/lib/remote/common:srcs",
"//src/main/java/com/google/devtools/build/lib/remote/disk:srcs",
Expand All @@ -28,7 +29,9 @@ java_library(
srcs = glob(
["*.java"],
exclude = [
"ChunkingConfig.java",
"ExecutionStatusException.java",
"FastCDCChunker.java",
"ReferenceCountedChannel.java",
"ChannelConnectionWithServerCapabilitiesFactory.java",
"RemoteRetrier.java",
Expand Down Expand Up @@ -95,6 +98,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/exec/local",
"//src/main/java/com/google/devtools/build/lib/packages/semantics",
"//src/main/java/com/google/devtools/build/lib/profiler",
"//src/main/java/com/google/devtools/build/lib/remote/chunking",
"//src/main/java/com/google/devtools/build/lib/remote/circuitbreaker",
"//src/main/java/com/google/devtools/build/lib/remote/common",
"//src/main/java/com/google/devtools/build/lib/remote/common:bulk_transfer_exception",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// Copyright 2026 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.remote;

import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;

import build.bazel.remote.execution.v2.Digest;
import build.bazel.remote.execution.v2.SplitBlobResponse;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;

/** Downloads blobs by sequentially fetching chunks via the SplitBlob API. */
public class ChunkedBlobDownloader {
private final GrpcCacheClient grpcCacheClient;
private final CombinedCache combinedCache;

public ChunkedBlobDownloader(GrpcCacheClient grpcCacheClient, CombinedCache combinedCache) {
this.grpcCacheClient = grpcCacheClient;
this.combinedCache = combinedCache;
}

/**
* Downloads a blob using chunked download via the SplitBlob API. This should be called with
* virtual threads, as it blocks on futures via {@link
* com.google.devtools.build.lib.remote.util.Utils#getFromFuture}.
*/
public void downloadChunked(
RemoteActionExecutionContext context, Digest blobDigest, OutputStream out)
throws IOException, InterruptedException {
List<Digest> chunkDigests = getChunkDigests(context, blobDigest);
downloadAndReassembleChunks(context, chunkDigests, out);
}

private List<Digest> getChunkDigests(RemoteActionExecutionContext context, Digest blobDigest)
throws IOException, InterruptedException {
if (blobDigest.getSizeBytes() == 0) {
return List.of();
}
ListenableFuture<SplitBlobResponse> splitResponseFuture =
grpcCacheClient.splitBlob(context, blobDigest);
if (splitResponseFuture == null) {
throw new CacheNotFoundException(blobDigest);
}
List<Digest> chunkDigests = getFromFuture(splitResponseFuture).getChunkDigestsList();
if (chunkDigests.isEmpty()) {
throw new CacheNotFoundException(blobDigest);
}
return chunkDigests;
}

private void downloadAndReassembleChunks(
RemoteActionExecutionContext context, List<Digest> chunkDigests, OutputStream out)
throws IOException, InterruptedException {
for (Digest chunkDigest : chunkDigests) {
getFromFuture(combinedCache.downloadBlob(context, chunkDigest, out));
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// Copyright 2026 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.remote;

import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;

import build.bazel.remote.execution.v2.Digest;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.remote.chunking.ChunkingConfig;
import com.google.devtools.build.lib.remote.chunking.FastCdcChunker;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.vfs.Path;
import com.google.protobuf.ByteString;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
* Uploads blobs in chunks using Content-Defined Chunking with FastCDC 2020.
*
* <p>Upload flow for blobs above threshold:
*
* <ol>
* <li>Chunk file with FastCDC
* <li>Call findMissingDigests on chunk digests
* <li>Upload only missing chunks
* <li>Call SpliceBlob to register the blob as the concatenation of chunks
* </ol>
*/
public class ChunkedBlobUploader {

private final GrpcCacheClient grpcCacheClient;
private final CombinedCache combinedCache;
private final FastCdcChunker chunker;
private final long chunkingThreshold;

/**
* Creates a new uploader with the given chunking configuration.
*
* @param grpcCacheClient client used for {@code FindMissingDigests} and {@code SpliceBlob} RPCs
* @param combinedCache cache used to upload individual chunks
* @param config chunking parameters negotiated from server capabilities
* @param digestUtil utility for computing chunk digests
*/
public ChunkedBlobUploader(
GrpcCacheClient grpcCacheClient,
CombinedCache combinedCache,
ChunkingConfig config,
DigestUtil digestUtil) {
this.grpcCacheClient = grpcCacheClient;
this.combinedCache = combinedCache;
this.chunker = new FastCdcChunker(config, digestUtil);
this.chunkingThreshold = config.chunkingThreshold();
}

/** Returns the minimum blob size for chunked upload. */
public long getChunkingThreshold() {
return chunkingThreshold;
}

/**
* Uploads a blob in content-defined chunks. The file is chunked with FastCDC, missing chunks are
* uploaded, and {@code SpliceBlob} is called to register the blob as the concatenation of its
* chunks.
*/
public void uploadChunked(RemoteActionExecutionContext context, Digest blobDigest, Path file)
throws IOException, InterruptedException {
List<Digest> chunkDigests;
try (InputStream input = file.getInputStream()) {
chunkDigests = chunker.chunkToDigests(input);
}
if (chunkDigests.isEmpty()) {
return;
}

ImmutableSet<Digest> missingDigests =
getFromFuture(grpcCacheClient.findMissingDigests(context, chunkDigests));
uploadMissingChunks(context, missingDigests, chunkDigests, file);
getFromFuture(grpcCacheClient.spliceBlob(context, blobDigest, chunkDigests));
}

private void uploadMissingChunks(
RemoteActionExecutionContext context,
ImmutableSet<Digest> missingDigests,
List<Digest> chunkDigests,
Path file)
throws IOException, InterruptedException {
if (missingDigests.isEmpty()) {
return;
}

Set<Digest> uploaded = new HashSet<>();
try (InputStream input = file.getInputStream()) {
for (Digest chunkDigest : chunkDigests) {
if (missingDigests.contains(chunkDigest) && uploaded.add(chunkDigest)) {
ByteString.Output out = ByteString.newOutput((int) chunkDigest.getSizeBytes());
ByteStreams.limit(input, chunkDigest.getSizeBytes()).transferTo(out);
getFromFuture(combinedCache.uploadBlob(context, chunkDigest, out.toByteString()));
} else {
input.skipNBytes(chunkDigest.getSizeBytes());
}
}
}
}
}
Loading
Loading