Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .plzconfig
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[please]
version = >=17.0.0
version = >=17.21.0

[build]
path = /usr/local/go/bin:/usr/local/bin:/usr/bin:/bin
Expand Down Expand Up @@ -27,8 +27,8 @@ moduledir = third_party/python

[plugin "cc"]
target = //plugins:cc
defaultoptcflags = --std=c99 -O3 -pipe -DNDEBUG -Wall -Werror -Wno-error=stringop-overflow -Wno-error=misleading-indentation -Wno-error=unknown-warning-option
defaultdbgcflags = --std=c99 -g3 -pipe -DDEBUG -Wall -Werror -Wno-error=stringop-overflow -Wno-error=misleading-indentation -Wno-error=unknown-warning-option
defaultoptcflags = --std=c99 -O3 -pipe -DNDEBUG -Wall -Werror -Wno-error=stringop-overflow -Wno-error=misleading-indentation
defaultdbgcflags = --std=c99 -g3 -pipe -DDEBUG -Wall -Werror -Wno-error=stringop-overflow -Wno-error=misleading-indentation

[plugin "shell"]
target = //plugins:shell
Expand Down
4 changes: 4 additions & 0 deletions ChangeLog
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
Version 11.17.0
---------------
* Upgrade Go toolchain, packages and Please plugin

Version 11.16.9
---------------
* Apply defaultstatic = true to fix Mettle broken deployment
Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
11.16.9
11.17.0
2 changes: 1 addition & 1 deletion elan/rpc/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ go_test(
genrule(
name = "test_data",
cmd = [
'for i in `seq 1 10000`; do echo "$i ----------------------------------------- $i" >> "tmp.txt"; done',
'for i in `seq 1 100000`; do echo "$i ----------------------------------------- $i" >> "tmp.txt"; done',

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some new package caused the test server to return more bytes in a single Recv call, so I had to make the test file bigger

'HASH="`(command -v sha256sum >/dev/null 2>&1 && sha256sum tmp.txt || shasum -a 256 tmp.txt) | cut -c -64`"',
'PREFIX="${HASH:0:2}"',
'mkdir -p "data/cas/${PREFIX}" "data/zstd_cas/${PREFIX}"',
Expand Down
11 changes: 5 additions & 6 deletions elan/rpc/compression_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"context"
"io"
"io/ioutil"
"os"
"path"
"testing"
Expand All @@ -20,9 +19,9 @@ var bsClient bs.ByteStreamClient

const (
// These are determined empirically from the generated test files.
hash = "5201263d9a7365629360c09a9ab780a1c15f94aaf3b38874d41500df3e0be87f"
size = 517788
ssize = "517788"
hash = "2e2595aaffe0987a183d3ba9a3c5d81c79d29f3f6e2b2dd47e628d6bf9b12725"
size = 5377790
ssize = "5377790"
name = "blobs/" + hash + "/" + ssize
cname = "compressed-blobs/zstd/" + hash + "/" + ssize
)
Expand Down Expand Up @@ -152,11 +151,11 @@ func testMain(m *testing.M) int {
bsClient = bs.NewByteStreamClient(conn)
defer conn.Close()

expectedData, err = ioutil.ReadFile(path.Join("elan/rpc/cas", hash[:2], hash))
expectedData, err = os.ReadFile(path.Join("elan/rpc/cas", hash[:2], hash))
if err != nil {
log.Fatalf("%s", err)
}
compressedData, err = ioutil.ReadFile(path.Join("elan/rpc/zstd_cas", hash[:2], hash))
compressedData, err = os.ReadFile(path.Join("elan/rpc/zstd_cas", hash[:2], hash))
if err != nil {
log.Fatalf("%s", err)
}
Expand Down
16 changes: 12 additions & 4 deletions elan/rpc/rpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"os"
"path"
"regexp"
"slices"
"strconv"
"strings"
"sync"
Expand All @@ -39,7 +40,6 @@ import (
"github.com/sirupsen/logrus"
"gocloud.dev/blob"
"gocloud.dev/gcerrors"
"golang.org/x/exp/slices"
"google.golang.org/api/googleapi"
bs "google.golang.org/genproto/googleapis/bytestream"
rpcstatus "google.golang.org/genproto/googleapis/rpc/status"
Expand Down Expand Up @@ -557,7 +557,7 @@ func (s *server) readCompressed(ctx context.Context, prefix string, digest *pb.D
return r, false, err
}
if s.isEmpty(digest) {
return ioutil.NopCloser(bytes.NewReader(nil)), compressed, nil
return io.NopCloser(bytes.NewReader(nil)), compressed, nil
}
if s.readRedis != nil && prefix == CASPrefix && digest.SizeBytes < s.largeBlobSize {
// NOTE: we could use GETRANGE here, but given it's a bit more expensive on the redis
Expand All @@ -566,7 +566,7 @@ func (s *server) readCompressed(ctx context.Context, prefix string, digest *pb.D
if err != nil && err != redis.Nil {
log.Warningf("Failed to get blob in Redis: %v", err)
} else if err != redis.Nil && blob != nil && limit == 0 {
return ioutil.NopCloser(bytes.NewReader(nil)), false, nil
return io.NopCloser(bytes.NewReader(nil)), false, nil
} else if err != redis.Nil && blob != nil && limit > 0 {
return io.NopCloser(bytes.NewReader(blob[offset : offset+limit])), false, nil
} else if err != redis.Nil && blob != nil && limit < 0 {
Expand Down Expand Up @@ -630,10 +630,18 @@ func (s *server) QueryWriteStatus(ctx context.Context, req *bs.QueryWriteStatusR
return nil, status.Errorf(codes.NotFound, "write %s not found", req.ResourceName)
}

func (s *server) SpliceBlob(ctx context.Context, req *pb.SpliceBlobRequest) (*pb.SpliceBlobResponse, error) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New methods in the proto

return nil, status.Errorf(codes.Unimplemented, "not implemented")
}

func (s *server) SplitBlob(ctx context.Context, req *pb.SplitBlobRequest) (*pb.SplitBlobResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "not implemented")
}

func (s *server) readBlob(ctx context.Context, key string, offset, length int64) (io.ReadCloser, error) {
if length == 0 || strings.Contains(key, digest.Empty.Hash) {
// Special case any empty read request
return ioutil.NopCloser(bytes.NewReader(nil)), nil
return io.NopCloser(bytes.NewReader(nil)), nil
}
start := time.Now()
defer func() { readLatencies.Observe(time.Since(start).Seconds()) }()
Expand Down
8 changes: 8 additions & 0 deletions flair/rpc/rpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,14 @@ func (s *server) GetTree(req *pb.GetTreeRequest, srv pb.ContentAddressableStorag
return srv.Send(r)
}

func (s *server) SpliceBlob(ctx context.Context, req *pb.SpliceBlobRequest) (*pb.SpliceBlobResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "not implemented")
}

func (s *server) SplitBlob(ctx context.Context, req *pb.SplitBlobRequest) (*pb.SplitBlobResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "not implemented")
}

func (s *server) Read(req *bs.ReadRequest, srv bs.ByteStream_ReadServer) error {
hash, err := s.bytestreamBlobName(req.ResourceName)
if err != nil {
Expand Down
139 changes: 78 additions & 61 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,96 +1,113 @@
module github.com/thought-machine/please-servers

go 1.24.0
go 1.26

ignore plz-out

require (
cloud.google.com/go/profiler v0.4.0
cloud.google.com/go/pubsub v1.33.0
cloud.google.com/go/storage v1.36.0
github.com/bazelbuild/remote-apis v0.0.0-20230411132548-35aee1c4a425
github.com/bazelbuild/remote-apis-sdks v0.0.0-20230419185642-269815af5db1
github.com/dgraph-io/ristretto v0.1.1
cloud.google.com/go/profiler v0.6.0
cloud.google.com/go/pubsub v1.50.4
cloud.google.com/go/storage v1.63.0
github.com/bazelbuild/remote-apis v0.0.0-20260331222004-becdd8f9ff81
github.com/bazelbuild/remote-apis-sdks v0.0.0-20260610142741-7ffd493e6686
github.com/dgraph-io/ristretto v0.2.0
github.com/dustin/go-humanize v1.0.1
github.com/go-redis/redis/v8 v8.11.5
github.com/golang/protobuf v1.5.4
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/go-retryablehttp v0.7.7
github.com/klauspost/compress v1.17.4
github.com/mostynb/go-grpc-compression v1.2.2
github.com/hashicorp/go-retryablehttp v0.7.8
github.com/klauspost/compress v1.19.0
github.com/mostynb/go-grpc-compression v1.2.3
github.com/peterebden/go-cli-init/v4 v4.0.2
github.com/peterebden/go-copyfile v0.0.0-20200424115000-bc0baf74909c
github.com/peterebden/go-sri v1.1.1
github.com/prometheus/client_golang v1.18.0
github.com/prometheus/common v0.45.0
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/common v0.70.0
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/sirupsen/logrus v1.9.3
github.com/sirupsen/logrus v1.9.4
github.com/stretchr/testify v1.11.1
github.com/thought-machine/http-admin v1.1.1
go.uber.org/automaxprocs v1.5.3
gocloud.dev v0.36.0
golang.org/x/crypto v0.46.0
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc
golang.org/x/sync v0.19.0
golang.org/x/time v0.5.0
google.golang.org/api v0.155.0
google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917
google.golang.org/genproto/googleapis/bytestream v0.0.0-20240102182953-50ed04b92917
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217
google.golang.org/grpc v1.79.3
google.golang.org/protobuf v1.36.10
go.uber.org/automaxprocs v1.6.0
gocloud.dev v0.46.0
golang.org/x/crypto v0.54.0
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
golang.org/x/sync v0.22.0
golang.org/x/time v0.15.0
google.golang.org/api v0.288.0
google.golang.org/genproto v0.0.0-20260706201446-f0a921348800
google.golang.org/genproto/googleapis/bytestream v0.0.0-20260706201446-f0a921348800
google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800
google.golang.org/grpc v1.82.0
google.golang.org/protobuf v1.36.11
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
)

require (
cloud.google.com/go v0.111.0 // indirect
cel.dev/expr v0.25.2 // indirect
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/auth v0.21.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.1.5 // indirect
cloud.google.com/go/longrunning v0.5.4 // indirect
cloud.google.com/go/iam v1.11.0 // indirect
cloud.google.com/go/longrunning v1.2.0 // indirect
cloud.google.com/go/monitoring v1.29.0 // indirect
cloud.google.com/go/pubsub/v2 v2.6.1 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
github.com/felixge/httpsnoop v1.1.0 // indirect
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/golang/glog v1.2.5 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/pprof v0.0.0-20260709232956-b9395ee17fa0 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/google/wire v0.7.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect
github.com/googleapis/gax-go/v2 v2.23.0 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/mostynb/zstdpool-syncpool v0.0.13 // indirect
github.com/pborman/uuid v1.2.1 // indirect
github.com/mostynb/zstdpool-syncpool v0.0.7 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pborman/uuid v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/xattr v0.4.9 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/thought-machine/go-flags v1.6.3 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.opencensus.io v0.24.0 // indirect
github.com/pkg/xattr v0.4.12 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
github.com/thought-machine/go-flags v1.7.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
go.opentelemetry.io/otel v1.39.0 // indirect
go.opentelemetry.io/otel/metric v1.39.0 // indirect
go.opentelemetry.io/otel/trace v1.39.0 // indirect
golang.org/x/net v0.48.0 // indirect
golang.org/x/oauth2 v0.34.0 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/term v0.38.0 // indirect
golang.org/x/text v0.32.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
golang.org/x/net v0.57.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/term v0.45.0 // indirect
golang.org/x/text v0.40.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

replace github.com/bazelbuild/remote-apis-sdks v0.0.0-20230419185642-269815af5db1 => github.com/peterebden/remote-apis-sdks v0.0.0-20230519151942-2a9420921957
replace github.com/bazelbuild/remote-apis-sdks => github.com/peterebden/remote-apis-sdks v0.0.0-20230519151942-2a9420921957
Loading
Loading