Fix harness config sync against local Hub storage

cmd/harness_config.go

@@ -492,7 +492,7 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT
 
 		needsFullUpload := false
 		if err != nil {
-			if strings.Contains(err.Error(), "harness config has no files") {
+			if isHarnessConfigNoFilesError(err) {
 				fmt.Printf("Harness-config '%s' exists but has no files. Uploading all files...\n", name)
 				needsFullUpload = true
 				filesToUpload = fileReqs
@@ -553,24 +553,8 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT
 
 	// Upload files
 	fmt.Printf("Uploading %d file(s)...\n", len(uploadResp.UploadURLs))
-	for _, urlInfo := range uploadResp.UploadURLs {
-		fileInfo := localFileMap[urlInfo.Path]
-		if fileInfo == nil {
-			fmt.Printf("  Warning: no matching file for %s\n", urlInfo.Path)
-			continue
-		}
-
-		f, err := os.Open(fileInfo.FullPath)
-		if err != nil {
-			return fmt.Errorf("failed to open %s: %w", fileInfo.Path, err)
-		}
-
-		err = hubCtx.Client.HarnessConfigs().UploadFile(ctx, urlInfo.URL, urlInfo.Method, urlInfo.Headers, f)
-		f.Close()
-		if err != nil {
-			return fmt.Errorf("failed to upload %s: %w", fileInfo.Path, err)
-		}
-		fmt.Printf("  Uploaded: %s\n", fileInfo.Path)
+	if err := uploadHarnessConfigFiles(ctx, hubCtx.Client.HarnessConfigs(), hcID, localFileMap, filesToUpload, uploadResp.UploadURLs); err != nil {
+		return err
 	}
 
 	// Build manifest
@@ -592,7 +576,7 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT
 	fmt.Println("Finalizing harness-config...")
 	hc, err := hubCtx.Client.HarnessConfigs().Finalize(ctx, hcID, manifest)
 	if err != nil {
-		if !strings.Contains(err.Error(), "file not found") {
+		if !isHarnessConfigMissingFileError(err) {
 			return fmt.Errorf("failed to finalize: %w", err)
 		}
 
@@ -607,14 +591,8 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT
 			if fileInfo == nil {
 				continue
 			}
-			f, openErr := os.Open(fileInfo.FullPath)
-			if openErr != nil {
-				return fmt.Errorf("failed to open %s: %w", fileInfo.Path, openErr)
-			}
-			uploadErr := hubCtx.Client.HarnessConfigs().UploadFile(ctx, urlInfo.URL, urlInfo.Method, urlInfo.Headers, f)
-			f.Close()
-			if uploadErr != nil {
-				return fmt.Errorf("failed to upload %s: %w", fileInfo.Path, uploadErr)
+			if err := uploadHarnessConfigFileBySignedURL(ctx, hubCtx.Client.HarnessConfigs(), fileInfo, urlInfo); err != nil {
+				return err
 			}
 			fmt.Printf("  Re-uploaded: %s\n", fileInfo.Path)
 		}
@@ -648,6 +626,20 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT
 	return nil
 }
 
+func isHarnessConfigNoFilesError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), "harness config has no files")
+}
+
+func isHarnessConfigMissingFileError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), "file not found")
+}
+
 // pullHarnessConfigFromHub downloads a harness config from the Hub to local disk.
 func pullHarnessConfigFromHub(hubCtx *HubContext, hc *hubclient.HarnessConfig, toPath string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
@@ -681,20 +673,21 @@ func pullHarnessConfigFromHub(hubCtx *HubContext, hc *hubclient.HarnessConfig, t
 	}
 
 	fmt.Printf("Downloading %d files to %s...\n", len(downloadResp.Files), destPath)
+	useHubFileRead := hasLocalDownloadURLs(downloadResp.Files)
 	for _, fileInfo := range downloadResp.Files {
 		filePath := filepath.Join(destPath, filepath.FromSlash(fileInfo.Path))
 
-		if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
-			return fmt.Errorf("failed to create directory for %s: %w", fileInfo.Path, err)
+		if err := ensureParentDir(filePath); err != nil {
+			return err
 		}
 
-		content, err := hubCtx.Client.HarnessConfigs().DownloadFile(ctx, fileInfo.URL)
+		content, err := downloadHarnessConfigContent(ctx, hubCtx.Client.HarnessConfigs(), hc.ID, fileInfo, useHubFileRead)
 		if err != nil {
-			return fmt.Errorf("failed to download %s: %w", fileInfo.Path, err)
+			return err
 		}
 
-		if err := os.WriteFile(filePath, content, 0644); err != nil {
-			return fmt.Errorf("failed to write %s: %w", fileInfo.Path, err)
+		if err := writeHarnessConfigFile(filePath, content); err != nil {
+			return err
 		}
 		fmt.Printf("  Downloaded: %s\n", fileInfo.Path)
 	}

cmd/harness_config_hub_test.go

@@ -0,0 +1,172 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/scion/pkg/hubclient"
+	"github.com/stretchr/testify/require"
+)
+
+func newMockHubServerForLocalStorageHarnessConfig(t *testing.T, uploadedPaths *[]string) *httptest.Server {
+	t.Helper()
+
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+
+		switch {
+		case r.URL.Path == "/api/v1/harness-configs" && r.Method == http.MethodPost:
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"harnessConfig": map[string]interface{}{
+					"id":      "local-storage-hc-id",
+					"name":    "codex",
+					"harness": "codex",
+				},
+			})
+
+		case r.URL.Path == "/api/v1/harness-configs" && r.Method == http.MethodGet:
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"harnessConfigs": []map[string]interface{}{},
+			})
+
+		case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/upload" && r.Method == http.MethodPost:
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"uploadUrls": []map[string]interface{}{
+					{
+						"path":   "config.yaml",
+						"url":    "file:///home/scion/.scion/storage/harness-configs/global/codex/config.yaml",
+						"method": "PUT",
+					},
+				},
+			})
+
+		case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/files" && r.Method == http.MethodPost:
+			require.NoError(t, r.ParseMultipartForm(10<<20))
+			require.NotNil(t, r.MultipartForm)
+			for field, headers := range r.MultipartForm.File {
+				*uploadedPaths = append(*uploadedPaths, field)
+				for _, fh := range headers {
+					file, err := fh.Open()
+					require.NoError(t, err)
+					_, err = io.ReadAll(file)
+					file.Close()
+					require.NoError(t, err)
+				}
+			}
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"files": []map[string]interface{}{
+					{
+						"path":    "config.yaml",
+						"size":    16,
+						"modTime": "2026-04-10T00:00:00Z",
+						"mode":    "0644",
+					},
+				},
+				"hash": "sha256:test-hash",
+			})
+
+		case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/finalize" && r.Method == http.MethodPost:
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"id":          "local-storage-hc-id",
+				"name":        "codex",
+				"harness":     "codex",
+				"status":      "active",
+				"contentHash": "sha256:abc123",
+			})
+
+		case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/download" && r.Method == http.MethodGet:
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"files": []map[string]interface{}{
+					{
+						"path": "config.yaml",
+						"hash": "sha256:download-hash",
+						"url":  "file:///home/scion/.scion/storage/harness-configs/global/codex/config.yaml",
+					},
+				},
+			})
+
+		case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/files/config.yaml" && r.Method == http.MethodGet:
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"path":     "config.yaml",
+				"content":  "harness: codex\n",
+				"size":     16,
+				"modTime":  "2026-04-10T00:00:00Z",
+				"encoding": "utf-8",
+				"hash":     "sha256:download-hash",
+			})
+
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+}
+
+func TestPullHarnessConfigFromHub_FallsBackToHubFileAPIForLocalStorageURLs(t *testing.T) {
+	tmpHome := t.TempDir()
+	var uploadedPaths []string
+
+	server := newMockHubServerForLocalStorageHarnessConfig(t, &uploadedPaths)
+	defer server.Close()
+
+	client, err := hubclient.New(server.URL)
+	require.NoError(t, err)
+
+	hubCtx := &HubContext{
+		Client:   client,
+		Endpoint: server.URL,
+	}
+
+	hc := &hubclient.HarnessConfig{
+		ID:      "local-storage-hc-id",
+		Name:    "codex",
+		Harness: "codex",
+	}
+
+	destPath := filepath.Join(tmpHome, "pulled-harness-config")
+	err = pullHarnessConfigFromHub(hubCtx, hc, destPath)
+	require.NoError(t, err)
+
+	content, err := os.ReadFile(filepath.Join(destPath, "config.yaml"))
+	require.NoError(t, err)
+	require.Equal(t, "harness: codex\n", string(content))
+}
+
+func TestSyncHarnessConfigToHub_FallsBackToHubFileAPIForLocalStorageURLs(t *testing.T) {
+	localPath := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(localPath, "config.yaml"), []byte("harness: codex\n"), 0644))
+
+	var uploadedPaths []string
+	server := newMockHubServerForLocalStorageHarnessConfig(t, &uploadedPaths)
+	defer server.Close()
+
+	client, err := hubclient.New(server.URL)
+	require.NoError(t, err)
+
+	hubCtx := &HubContext{
+		Client:   client,
+		Endpoint: server.URL,
+	}
+
+	err = syncHarnessConfigToHub(hubCtx, "codex", localPath, "global", "codex")
+	require.NoError(t, err)
+	require.Contains(t, uploadedPaths, "config.yaml")
+}