Feature/rework push oid checking

.github/workflows/build.yaml

@@ -20,10 +20,10 @@ jobs:
       with:
         submodules: recursive
 
-    - name: Set up Go 1.x
+    - name: Set up Go 1.26.5
       uses: actions/setup-go@v5
       with:
-        go-version-file: go.mod
+        go-version: '1.26.5'
 
     - name: Build Git-DRS
       run: make build

.github/workflows/pr-checks.yaml

@@ -26,7 +26,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version-file: go.mod
+        go-version: '1.26.5'
 
     - name: Run go vet
       run: go vet ./...
@@ -71,7 +71,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version-file: go.mod
+        go-version: '1.26.5'
 
     - name: Run tests
       run: go test -v -race $(go list ./... | grep -v '/cmd/addurl$')

.github/workflows/release.yaml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: go.mod
+          go-version: '1.26.5'
 
       - name: Import Apple Developer certificate
         env:

.github/workflows/syfon-backend-e2e.yaml

@@ -31,7 +31,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: git-drs/go.mod
+          go-version: '1.26.5'
 
       - name: Resolve Syfon ref from pinned module version
         id: syfon-ref

.github/workflows/test.yaml

@@ -28,7 +28,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version-file: go.mod
+        go-version: '1.26.5'
 
     - name: Generate coverage
       run: make coverage-html-full

Makefile

@@ -43,7 +43,7 @@ lint-depends:
 	go install github.com/client9/misspell/cmd/misspell@latest
 
 # Run code style and other checks
-# Note: Using native Go tools instead of golangci-lint for Go 1.24 compatibility
+# Note: Using native Go tools instead of golangci-lint for Go 1.26.5 compatibility
 lint:
 	@echo "Running go vet..."
 	@go vet ./...

README.md

@@ -118,7 +118,7 @@ Push and pull depend on server-side bucket mapping for the requested scope. That
 
 - Git
 - access credentials for the target Gen3/Syfon deployment
-- Go 1.26.2+ for local builds
+- Go 1.26.5+ for local builds
 
 ## Support
 

client/client.go

@@ -0,0 +1,182 @@
+// Package client provides the reusable Git-DRS client API.
+//
+// It is independent of the git-drs CLI and does not require a Git checkout.
+package client
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/calypr/git-drs/internal/remoteruntime"
+	internaltransfer "github.com/calypr/git-drs/internal/transfer"
+	syclient "github.com/calypr/syfon/client"
+)
+
+// Options configures a DRS client. Either AccessToken or Username and
+// Password must be supplied for an authenticated remote.
+type Options struct {
+	Endpoint     string
+	AccessToken  string
+	Username     string
+	Password     string
+	Organization string
+	Project      string
+	HTTPClient   *http.Client
+	Logger       *slog.Logger
+}
+
+// Client is a reusable connection to one DRS/Syfon project scope.
+type Client struct {
+	runtime *remoteruntime.GitContext
+}
+
+// New creates a reusable Git-DRS client without reading Git configuration or
+// changing process-global state.
+func New(opts Options) (*Client, error) {
+	if strings.TrimSpace(opts.Endpoint) == "" {
+		return nil, fmt.Errorf("endpoint is required")
+	}
+	if strings.TrimSpace(opts.Project) == "" {
+		return nil, fmt.Errorf("project is required")
+	}
+	if strings.TrimSpace(opts.AccessToken) == "" && (strings.TrimSpace(opts.Username) == "" || strings.TrimSpace(opts.Password) == "") {
+		return nil, fmt.Errorf("access token or username and password are required")
+	}
+
+	clientOpts := make([]syclient.Option, 0, 2)
+	if opts.HTTPClient != nil {
+		clientOpts = append(clientOpts, syclient.WithHTTPClient(opts.HTTPClient))
+	}
+	if strings.TrimSpace(opts.AccessToken) != "" {
+		clientOpts = append(clientOpts, syclient.WithBearerToken(opts.AccessToken))
+	} else {
+		clientOpts = append(clientOpts, syclient.WithBasicAuth(opts.Username, opts.Password))
+	}
+
+	raw, err := syclient.New(opts.Endpoint, clientOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("create DRS client: %w", err)
+	}
+	syfonClient, ok := raw.(*syclient.Client)
+	if !ok {
+		return nil, fmt.Errorf("unexpected Syfon client type %T", raw)
+	}
+
+	logger := opts.Logger
+	if logger == nil {
+		logger = slog.New(slog.NewTextHandler(io.Discard, nil))
+	}
+	return &Client{runtime: &remoteruntime.GitContext{
+		Client:       syfonClient,
+		Organization: opts.Organization,
+		ProjectId:    opts.Project,
+		Logger:       logger,
+	}}, nil
+}
+
+// File identifies one payload to pull. Path is relative to PullOptions.Root.
+type File struct {
+	Path string
+	OID  string
+	Size int64
+}
+
+// PullOptions describes the destination and files for a pull operation.
+type PullOptions struct {
+	Root      string
+	Files     []File
+	Overwrite bool
+}
+
+// Pull downloads the requested DRS payloads directly into Root. It does not
+// inspect Git state, read git-drs configuration, write the Git-LFS cache, or
+// update the Git index.
+func (c *Client) Pull(ctx context.Context, opts PullOptions) error {
+	if c == nil || c.runtime == nil {
+		return fmt.Errorf("client is nil")
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if strings.TrimSpace(opts.Root) == "" {
+		return fmt.Errorf("pull root is required")
+	}
+
+	root, err := filepath.Abs(opts.Root)
+	if err != nil {
+		return fmt.Errorf("resolve pull root: %w", err)
+	}
+	for _, file := range opts.Files {
+		rel, err := safeRelativePath(file.Path)
+		if err != nil {
+			return err
+		}
+		if strings.TrimSpace(file.OID) == "" {
+			return fmt.Errorf("OID is required for %q", file.Path)
+		}
+		if file.Size < 0 {
+			return fmt.Errorf("size must not be negative for %q", file.Path)
+		}
+
+		dst := filepath.Join(root, rel)
+		if !opts.Overwrite {
+			if _, statErr := os.Stat(dst); statErr == nil {
+				return fmt.Errorf("destination already exists: %s", dst)
+			} else if !os.IsNotExist(statErr) {
+				return fmt.Errorf("stat destination %s: %w", dst, statErr)
+			}
+		}
+		if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+			return fmt.Errorf("create destination directory for %s: %w", file.Path, err)
+		}
+		if err := internaltransfer.DownloadToPath(ctx, c.runtime, file.OID, dst); err != nil {
+			return fmt.Errorf("pull %s: %w", file.Path, err)
+		}
+		if err := verifyFile(dst, file.OID, file.Size); err != nil {
+			_ = os.Remove(dst)
+			return fmt.Errorf("verify %s: %w", file.Path, err)
+		}
+	}
+	return nil
+}
+
+func safeRelativePath(path string) (string, error) {
+	path = filepath.Clean(filepath.FromSlash(strings.TrimSpace(path)))
+	if path == "." || filepath.IsAbs(path) || path == ".." || strings.HasPrefix(path, ".."+string(filepath.Separator)) {
+		return "", fmt.Errorf("path must be relative to pull root: %q", path)
+	}
+	return path, nil
+}
+
+func verifyFile(path, expectedOID string, expectedSize int64) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	if info.Size() != expectedSize {
+		return fmt.Errorf("size %d does not match expected size %d", info.Size(), expectedSize)
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	hash := sha256.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return err
+	}
+	actual := hex.EncodeToString(hash.Sum(nil))
+	expected := strings.TrimPrefix(strings.TrimSpace(expectedOID), "sha256:")
+	if !strings.EqualFold(actual, expected) {
+		return fmt.Errorf("sha256 %s does not match expected %s", actual, expected)
+	}
+	return nil
+}

client/client_test.go

@@ -0,0 +1,26 @@
+package client
+
+import "testing"
+
+func TestNewValidatesConnectionOptions(t *testing.T) {
+	if _, err := New(Options{}); err == nil {
+		t.Fatal("expected missing endpoint error")
+	}
+	if _, err := New(Options{Endpoint: "https://example.test"}); err == nil {
+		t.Fatal("expected missing project error")
+	}
+	if _, err := New(Options{Endpoint: "https://example.test", Project: "project"}); err == nil {
+		t.Fatal("expected missing credentials error")
+	}
+}
+
+func TestSafeRelativePath(t *testing.T) {
+	for _, path := range []string{"/absolute", "../outside", "..", "."} {
+		if _, err := safeRelativePath(path); err == nil {
+			t.Fatalf("expected unsafe path %q to fail", path)
+		}
+	}
+	if got, err := safeRelativePath("data/file.bin"); err != nil || got != "data/file.bin" {
+		t.Fatalf("safe path = %q, err = %v", got, err)
+	}
+}

client/etl.go

@@ -0,0 +1,53 @@
+// Package client contains small programmatic entry points for services that
+// embed git-drs rather than invoking the git-drs executable.
+package client
+
+import (
+	"fmt"
+	"log/slog"
+
+	"github.com/calypr/git-drs/cmd/initialize"
+	remoteadd "github.com/calypr/git-drs/cmd/remote/add"
+)
+
+// Gen3RemoteOptions describes the minimal Gen3 remote setup needed by the ETL
+// worker. Token is the bearer token issued by Fence.
+type Gen3RemoteOptions struct {
+	RemoteName string
+	Token      string
+	Bucket     string
+	Scope      string
+	Logger     *slog.Logger
+}
+
+// ConfigureGen3Remote configures a repository-local Gen3 DRS remote without
+// invoking Cobra or the git-drs executable.
+func ConfigureGen3Remote(opts Gen3RemoteOptions) error {
+	if opts.Logger == nil {
+		opts.Logger = slog.Default()
+	}
+	if opts.RemoteName == "" {
+		return fmt.Errorf("remote name is required")
+	}
+	if opts.Token == "" {
+		return fmt.Errorf("Gen3 token is required")
+	}
+	if opts.Scope == "" {
+		return fmt.Errorf("Gen3 scope is required")
+	}
+	return remoteadd.ConfigureGen3(remoteadd.Gen3Options{
+		RemoteName: opts.RemoteName,
+		Token:      opts.Token,
+		Bucket:     opts.Bucket,
+		Scope:      opts.Scope,
+		Logger:     opts.Logger,
+	})
+}
+
+// InitializeRepository applies idempotent repository-local git-drs setup.
+func InitializeRepository(logger *slog.Logger) error {
+	if logger == nil {
+		logger = slog.Default()
+	}
+	return initialize.InitializeRepo(logger)
+}

cmd/push/main.go

@@ -80,7 +80,7 @@ var Cmd = &cobra.Command{
 		uniqueOIDs := countUniqueOIDs(lfsFiles)
 		if state.AckOID == "" {
 			fmt.Fprintf(os.Stdout, "DRS: no synchronization baseline for %s; bootstrapping full Git history and checking %d unique object(s)\n", state.RemoteRef, uniqueOIDs)
-			fmt.Fprintln(os.Stdout, "DRS: this full metadata check is normally needed once per branch; it repeats only when the remote synchronization ref is missing or behind")
+			fmt.Fprintln(os.Stdout, "DRS: this full Git history scan and bulk existence check is normally needed once per branch; it repeats only when the remote synchronization ref is missing or behind")
 		} else {
 			fmt.Fprintf(os.Stdout, "DRS: checking %d new reachable pointer(s) (%d unique object(s)) since synchronization %s\n", len(lfsFiles), uniqueOIDs, state.AckOID[:12])
 		}

cmd/remote/add/api.go

@@ -0,0 +1,32 @@
+package add
+
+import "log/slog"
+
+// Gen3Options contains the repository-local values needed to configure a
+// Gen3 DRS remote programmatically.
+type Gen3Options struct {
+	RemoteName string
+	Token      string
+	Bucket     string
+	Scope      string
+	Logger     *slog.Logger
+}
+
+// ConfigureGen3 configures a Gen3 remote without requiring a Cobra command or
+// a git-drs subprocess. It is intentionally small so embedded services can
+// reuse the same setup path as the CLI.
+func ConfigureGen3(opts Gen3Options) error {
+	logger := opts.Logger
+	if logger == nil {
+		logger = slog.Default()
+	}
+	previousToken := fenceToken
+	previousBucket := selectedBucket
+	defer func() {
+		fenceToken = previousToken
+		selectedBucket = previousBucket
+	}()
+	fenceToken = opts.Token
+	selectedBucket = opts.Bucket
+	return gen3Init(opts.RemoteName, "", opts.Token, opts.Scope, logger)
+}