diff --git a/cmd/harness_config.go b/cmd/harness_config.go index 663c6e83..5eb87e76 100644 --- a/cmd/harness_config.go +++ b/cmd/harness_config.go @@ -492,7 +492,7 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT needsFullUpload := false if err != nil { - if strings.Contains(err.Error(), "harness config has no files") { + if isHarnessConfigNoFilesError(err) { fmt.Printf("Harness-config '%s' exists but has no files. Uploading all files...\n", name) needsFullUpload = true filesToUpload = fileReqs @@ -553,24 +553,8 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT // Upload files fmt.Printf("Uploading %d file(s)...\n", len(uploadResp.UploadURLs)) - for _, urlInfo := range uploadResp.UploadURLs { - fileInfo := localFileMap[urlInfo.Path] - if fileInfo == nil { - fmt.Printf(" Warning: no matching file for %s\n", urlInfo.Path) - continue - } - - f, err := os.Open(fileInfo.FullPath) - if err != nil { - return fmt.Errorf("failed to open %s: %w", fileInfo.Path, err) - } - - err = hubCtx.Client.HarnessConfigs().UploadFile(ctx, urlInfo.URL, urlInfo.Method, urlInfo.Headers, f) - f.Close() - if err != nil { - return fmt.Errorf("failed to upload %s: %w", fileInfo.Path, err) - } - fmt.Printf(" Uploaded: %s\n", fileInfo.Path) + if err := uploadHarnessConfigFiles(ctx, hubCtx.Client.HarnessConfigs(), hcID, localFileMap, filesToUpload, uploadResp.UploadURLs); err != nil { + return err } // Build manifest @@ -592,7 +576,7 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT fmt.Println("Finalizing harness-config...") hc, err := hubCtx.Client.HarnessConfigs().Finalize(ctx, hcID, manifest) if err != nil { - if !strings.Contains(err.Error(), "file not found") { + if !isHarnessConfigMissingFileError(err) { return fmt.Errorf("failed to finalize: %w", err) } @@ -607,14 +591,8 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT if fileInfo == nil { continue } - f, openErr := os.Open(fileInfo.FullPath) - if openErr != nil { - return fmt.Errorf("failed to open %s: %w", fileInfo.Path, openErr) - } - uploadErr := hubCtx.Client.HarnessConfigs().UploadFile(ctx, urlInfo.URL, urlInfo.Method, urlInfo.Headers, f) - f.Close() - if uploadErr != nil { - return fmt.Errorf("failed to upload %s: %w", fileInfo.Path, uploadErr) + if err := uploadHarnessConfigFileBySignedURL(ctx, hubCtx.Client.HarnessConfigs(), fileInfo, urlInfo); err != nil { + return err } fmt.Printf(" Re-uploaded: %s\n", fileInfo.Path) } @@ -648,6 +626,20 @@ func syncHarnessConfigToHub(hubCtx *HubContext, name, localPath, scope, harnessT return nil } +func isHarnessConfigNoFilesError(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "harness config has no files") +} + +func isHarnessConfigMissingFileError(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "file not found") +} + // pullHarnessConfigFromHub downloads a harness config from the Hub to local disk. func pullHarnessConfigFromHub(hubCtx *HubContext, hc *hubclient.HarnessConfig, toPath string) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) @@ -681,20 +673,21 @@ func pullHarnessConfigFromHub(hubCtx *HubContext, hc *hubclient.HarnessConfig, t } fmt.Printf("Downloading %d files to %s...\n", len(downloadResp.Files), destPath) + useHubFileRead := hasLocalDownloadURLs(downloadResp.Files) for _, fileInfo := range downloadResp.Files { filePath := filepath.Join(destPath, filepath.FromSlash(fileInfo.Path)) - if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil { - return fmt.Errorf("failed to create directory for %s: %w", fileInfo.Path, err) + if err := ensureParentDir(filePath); err != nil { + return err } - content, err := hubCtx.Client.HarnessConfigs().DownloadFile(ctx, fileInfo.URL) + content, err := downloadHarnessConfigContent(ctx, hubCtx.Client.HarnessConfigs(), hc.ID, fileInfo, useHubFileRead) if err != nil { - return fmt.Errorf("failed to download %s: %w", fileInfo.Path, err) + return err } - if err := os.WriteFile(filePath, content, 0644); err != nil { - return fmt.Errorf("failed to write %s: %w", fileInfo.Path, err) + if err := writeHarnessConfigFile(filePath, content); err != nil { + return err } fmt.Printf(" Downloaded: %s\n", fileInfo.Path) } diff --git a/cmd/harness_config_hub_test.go b/cmd/harness_config_hub_test.go new file mode 100644 index 00000000..218aa1f1 --- /dev/null +++ b/cmd/harness_config_hub_test.go @@ -0,0 +1,172 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/GoogleCloudPlatform/scion/pkg/hubclient" + "github.com/stretchr/testify/require" +) + +func newMockHubServerForLocalStorageHarnessConfig(t *testing.T, uploadedPaths *[]string) *httptest.Server { + t.Helper() + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.URL.Path == "/api/v1/harness-configs" && r.Method == http.MethodPost: + json.NewEncoder(w).Encode(map[string]interface{}{ + "harnessConfig": map[string]interface{}{ + "id": "local-storage-hc-id", + "name": "codex", + "harness": "codex", + }, + }) + + case r.URL.Path == "/api/v1/harness-configs" && r.Method == http.MethodGet: + json.NewEncoder(w).Encode(map[string]interface{}{ + "harnessConfigs": []map[string]interface{}{}, + }) + + case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/upload" && r.Method == http.MethodPost: + json.NewEncoder(w).Encode(map[string]interface{}{ + "uploadUrls": []map[string]interface{}{ + { + "path": "config.yaml", + "url": "file:///home/scion/.scion/storage/harness-configs/global/codex/config.yaml", + "method": "PUT", + }, + }, + }) + + case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/files" && r.Method == http.MethodPost: + require.NoError(t, r.ParseMultipartForm(10<<20)) + require.NotNil(t, r.MultipartForm) + for field, headers := range r.MultipartForm.File { + *uploadedPaths = append(*uploadedPaths, field) + for _, fh := range headers { + file, err := fh.Open() + require.NoError(t, err) + _, err = io.ReadAll(file) + file.Close() + require.NoError(t, err) + } + } + json.NewEncoder(w).Encode(map[string]interface{}{ + "files": []map[string]interface{}{ + { + "path": "config.yaml", + "size": 16, + "modTime": "2026-04-10T00:00:00Z", + "mode": "0644", + }, + }, + "hash": "sha256:test-hash", + }) + + case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/finalize" && r.Method == http.MethodPost: + json.NewEncoder(w).Encode(map[string]interface{}{ + "id": "local-storage-hc-id", + "name": "codex", + "harness": "codex", + "status": "active", + "contentHash": "sha256:abc123", + }) + + case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/download" && r.Method == http.MethodGet: + json.NewEncoder(w).Encode(map[string]interface{}{ + "files": []map[string]interface{}{ + { + "path": "config.yaml", + "hash": "sha256:download-hash", + "url": "file:///home/scion/.scion/storage/harness-configs/global/codex/config.yaml", + }, + }, + }) + + case r.URL.Path == "/api/v1/harness-configs/local-storage-hc-id/files/config.yaml" && r.Method == http.MethodGet: + json.NewEncoder(w).Encode(map[string]interface{}{ + "path": "config.yaml", + "content": "harness: codex\n", + "size": 16, + "modTime": "2026-04-10T00:00:00Z", + "encoding": "utf-8", + "hash": "sha256:download-hash", + }) + + default: + w.WriteHeader(http.StatusNotFound) + } + })) +} + +func TestPullHarnessConfigFromHub_FallsBackToHubFileAPIForLocalStorageURLs(t *testing.T) { + tmpHome := t.TempDir() + var uploadedPaths []string + + server := newMockHubServerForLocalStorageHarnessConfig(t, &uploadedPaths) + defer server.Close() + + client, err := hubclient.New(server.URL) + require.NoError(t, err) + + hubCtx := &HubContext{ + Client: client, + Endpoint: server.URL, + } + + hc := &hubclient.HarnessConfig{ + ID: "local-storage-hc-id", + Name: "codex", + Harness: "codex", + } + + destPath := filepath.Join(tmpHome, "pulled-harness-config") + err = pullHarnessConfigFromHub(hubCtx, hc, destPath) + require.NoError(t, err) + + content, err := os.ReadFile(filepath.Join(destPath, "config.yaml")) + require.NoError(t, err) + require.Equal(t, "harness: codex\n", string(content)) +} + +func TestSyncHarnessConfigToHub_FallsBackToHubFileAPIForLocalStorageURLs(t *testing.T) { + localPath := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(localPath, "config.yaml"), []byte("harness: codex\n"), 0644)) + + var uploadedPaths []string + server := newMockHubServerForLocalStorageHarnessConfig(t, &uploadedPaths) + defer server.Close() + + client, err := hubclient.New(server.URL) + require.NoError(t, err) + + hubCtx := &HubContext{ + Client: client, + Endpoint: server.URL, + } + + err = syncHarnessConfigToHub(hubCtx, "codex", localPath, "global", "codex") + require.NoError(t, err) + require.Contains(t, uploadedPaths, "config.yaml") +} diff --git a/cmd/harness_config_transfer.go b/cmd/harness_config_transfer.go new file mode 100644 index 00000000..a96b60b0 --- /dev/null +++ b/cmd/harness_config_transfer.go @@ -0,0 +1,141 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/GoogleCloudPlatform/scion/pkg/hubclient" +) + +func downloadHarnessConfigContent( + ctx context.Context, + service hubclient.HarnessConfigService, + harnessConfigID string, + file hubclient.DownloadURLInfo, + useHubFileRead bool, +) ([]byte, error) { + if useHubFileRead { + content, err := service.ReadFile(ctx, harnessConfigID, file.Path) + if err != nil { + return nil, fmt.Errorf("read harness config file through Hub API: %w", err) + } + return content, nil + } + + content, err := service.DownloadFile(ctx, file.URL) + if err != nil { + return nil, fmt.Errorf("download harness config file: %w", err) + } + return content, nil +} + +func uploadHarnessConfigFiles( + ctx context.Context, + service hubclient.HarnessConfigService, + harnessConfigID string, + localFileMap map[string]*hubclient.FileInfo, + filesToUpload []hubclient.FileUploadRequest, + uploadURLs []hubclient.UploadURLInfo, +) error { + if hasLocalSignedURLs(uploadURLs) { + return uploadHarnessConfigFilesThroughHubAPI(ctx, service, harnessConfigID, localFileMap, filesToUpload) + } + + return uploadHarnessConfigFilesBySignedURL(ctx, service, localFileMap, uploadURLs) +} + +func uploadHarnessConfigFilesThroughHubAPI( + ctx context.Context, + service hubclient.HarnessConfigService, + harnessConfigID string, + localFileMap map[string]*hubclient.FileInfo, + filesToUpload []hubclient.FileUploadRequest, +) error { + filesForFallback := make([]hubclient.FileInfo, 0, len(filesToUpload)) + for _, req := range filesToUpload { + fileInfo := localFileMap[req.Path] + if fileInfo == nil { + fmt.Printf(" Warning: no matching file for %s\n", req.Path) + continue + } + filesForFallback = append(filesForFallback, *fileInfo) + } + + if err := service.UploadFilesMultipart(ctx, harnessConfigID, filesForFallback); err != nil { + return fmt.Errorf("upload harness config files through Hub API: %w", err) + } + + for _, fileInfo := range filesForFallback { + fmt.Printf(" Uploaded: %s\n", fileInfo.Path) + } + return nil +} + +func uploadHarnessConfigFilesBySignedURL( + ctx context.Context, + service hubclient.HarnessConfigService, + localFileMap map[string]*hubclient.FileInfo, + uploadURLs []hubclient.UploadURLInfo, +) error { + for _, urlInfo := range uploadURLs { + fileInfo := localFileMap[urlInfo.Path] + if fileInfo == nil { + fmt.Printf(" Warning: no matching file for %s\n", urlInfo.Path) + continue + } + + if err := uploadHarnessConfigFileBySignedURL(ctx, service, fileInfo, urlInfo); err != nil { + return err + } + fmt.Printf(" Uploaded: %s\n", fileInfo.Path) + } + return nil +} + +func uploadHarnessConfigFileBySignedURL( + ctx context.Context, + service hubclient.HarnessConfigService, + fileInfo *hubclient.FileInfo, + urlInfo hubclient.UploadURLInfo, +) error { + f, err := os.Open(fileInfo.FullPath) + if err != nil { + return fmt.Errorf("open harness config file for upload: %w", err) + } + defer f.Close() + + if err := service.UploadFile(ctx, urlInfo.URL, urlInfo.Method, urlInfo.Headers, f); err != nil { + return fmt.Errorf("upload harness config file: %w", err) + } + return nil +} + +func ensureParentDir(filePath string) error { + if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil { + return fmt.Errorf("create harness config destination directory: %w", err) + } + return nil +} + +func writeHarnessConfigFile(filePath string, content []byte) error { + if err := os.WriteFile(filePath, content, 0644); err != nil { + return fmt.Errorf("write harness config file: %w", err) + } + return nil +} diff --git a/cmd/transfer_urls.go b/cmd/transfer_urls.go new file mode 100644 index 00000000..19edeca9 --- /dev/null +++ b/cmd/transfer_urls.go @@ -0,0 +1,39 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "strings" + + "github.com/GoogleCloudPlatform/scion/pkg/hubclient" +) + +func hasLocalSignedURLs(urls []hubclient.UploadURLInfo) bool { + for _, info := range urls { + if strings.HasPrefix(info.URL, "file://") { + return true + } + } + return false +} + +func hasLocalDownloadURLs(urls []hubclient.DownloadURLInfo) bool { + for _, info := range urls { + if strings.HasPrefix(info.URL, "file://") { + return true + } + } + return false +} diff --git a/pkg/hub/harness_config_file_handlers.go b/pkg/hub/harness_config_file_handlers.go new file mode 100644 index 00000000..ca41ea77 --- /dev/null +++ b/pkg/hub/harness_config_file_handlers.go @@ -0,0 +1,408 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hub + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "strings" + + "github.com/GoogleCloudPlatform/scion/pkg/storage" + "github.com/GoogleCloudPlatform/scion/pkg/store" +) + +const maxHarnessConfigFileSize = 1 << 20 // 1 MB + +type HarnessConfigFileListResponse = TemplateFileListResponse +type HarnessConfigFileEntry = TemplateFileEntry +type HarnessConfigFileContentResponse = TemplateFileContentResponse +type HarnessConfigFileUploadResponse = TemplateFileUploadResponse +type HarnessConfigFileWriteRequest = TemplateFileWriteRequest +type HarnessConfigFileWriteResponse = TemplateFileWriteResponse + +func (s *Server) handleHarnessConfigFiles(w http.ResponseWriter, r *http.Request, id, filePath string) { + if filePath == "" { + switch r.Method { + case http.MethodGet: + s.handleHarnessConfigFileList(w, r, id) + case http.MethodPost: + s.handleHarnessConfigFileUpload(w, r, id) + default: + MethodNotAllowed(w) + } + return + } + + switch r.Method { + case http.MethodGet: + s.handleHarnessConfigFileRead(w, r, id, filePath) + case http.MethodPut: + s.handleHarnessConfigFileWrite(w, r, id, filePath) + case http.MethodDelete: + s.handleHarnessConfigFileDelete(w, r, id, filePath) + default: + MethodNotAllowed(w) + } +} + +func (s *Server) handleHarnessConfigFileList(w http.ResponseWriter, r *http.Request, id string) { + ctx := r.Context() + + hc, err := s.store.GetHarnessConfig(ctx, id) + if err != nil { + writeErrorFromErr(w, err, "") + return + } + + var totalSize int64 + entries := make([]HarnessConfigFileEntry, len(hc.Files)) + for i, f := range hc.Files { + entries[i] = HarnessConfigFileEntry{ + Path: f.Path, + Size: f.Size, + ModTime: hc.Updated.UTC().Format("2006-01-02T15:04:05Z"), + Mode: f.Mode, + } + totalSize += f.Size + } + + writeJSON(w, http.StatusOK, HarnessConfigFileListResponse{ + Files: entries, + TotalSize: totalSize, + TotalCount: len(entries), + }) +} + +func (s *Server) handleHarnessConfigFileRead(w http.ResponseWriter, r *http.Request, id, filePath string) { + ctx := r.Context() + + hc, err := s.store.GetHarnessConfig(ctx, id) + if err != nil { + writeErrorFromErr(w, err, "") + return + } + + var found *store.TemplateFile + for i := range hc.Files { + if hc.Files[i].Path == filePath { + found = &hc.Files[i] + break + } + } + if found == nil { + NotFound(w, "Harness config file") + return + } + + if found.Size > maxHarnessConfigFileSize { + writeError(w, http.StatusRequestEntityTooLarge, "payload_too_large", + "File too large for inline viewing. Use the download endpoint instead.", nil) + return + } + + stor := s.GetStorage() + if stor == nil { + RuntimeError(w, "Storage not configured") + return + } + + objectPath := hc.StoragePath + "/" + filePath + reader, _, err := stor.Download(ctx, objectPath) + if err != nil { + if err == storage.ErrNotFound { + NotFound(w, "Harness config file") + return + } + RuntimeError(w, "Failed to read file from storage") + return + } + defer reader.Close() + + data, err := io.ReadAll(io.LimitReader(reader, maxHarnessConfigFileSize+1)) + if err != nil { + RuntimeError(w, "Failed to read file content") + return + } + + if int64(len(data)) > maxHarnessConfigFileSize { + writeError(w, http.StatusRequestEntityTooLarge, "payload_too_large", + "File too large for inline viewing. Use the download endpoint instead.", nil) + return + } + + writeJSON(w, http.StatusOK, HarnessConfigFileContentResponse{ + Path: filePath, + Content: string(data), + Size: int64(len(data)), + ModTime: hc.Updated.UTC().Format("2006-01-02T15:04:05Z"), + Encoding: "utf-8", + Hash: found.Hash, + }) +} + +func (s *Server) handleHarnessConfigFileWrite(w http.ResponseWriter, r *http.Request, id, filePath string) { + ctx := r.Context() + + hc, err := s.store.GetHarnessConfig(ctx, id) + if err != nil { + writeErrorFromErr(w, err, "") + return + } + if hc.Locked { + Forbidden(w) + return + } + + var req HarnessConfigFileWriteRequest + if strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { + if err := readJSON(r, &req); err != nil { + BadRequest(w, "Invalid request body: "+err.Error()) + return + } + } else { + data, err := io.ReadAll(io.LimitReader(r.Body, maxHarnessConfigFileSize+1)) + if err != nil { + BadRequest(w, "Failed to read request body") + return + } + if int64(len(data)) > maxHarnessConfigFileSize { + writeError(w, http.StatusRequestEntityTooLarge, "payload_too_large", + "File too large for inline upload.", nil) + return + } + req.Content = string(data) + } + + if req.ExpectedHash != "" { + for _, f := range hc.Files { + if f.Path == filePath && f.Hash != req.ExpectedHash { + writeError(w, http.StatusConflict, ErrCodeConflict, + "File has been modified since last read", nil) + return + } + } + } + + stor := s.GetStorage() + if stor == nil { + RuntimeError(w, "Storage not configured") + return + } + + content := []byte(req.Content) + objectPath := hc.StoragePath + "/" + filePath + _, err = stor.Upload(ctx, objectPath, strings.NewReader(req.Content), storage.UploadOptions{ + ContentType: "text/plain; charset=utf-8", + }) + if err != nil { + RuntimeError(w, "Failed to write file to storage") + return + } + + sum := sha256.Sum256(content) + fileHash := "sha256:" + hex.EncodeToString(sum[:]) + fileSize := int64(len(content)) + + updated := false + for i := range hc.Files { + if hc.Files[i].Path == filePath { + hc.Files[i].Hash = fileHash + hc.Files[i].Size = fileSize + hc.Files[i].Mode = "0644" + updated = true + break + } + } + if !updated { + hc.Files = append(hc.Files, store.TemplateFile{ + Path: filePath, + Size: fileSize, + Hash: fileHash, + Mode: "0644", + }) + } + + hc.ContentHash = computeContentHash(hc.Files) + if err := s.store.UpdateHarnessConfig(ctx, hc); err != nil { + RuntimeError(w, "Failed to update harness config manifest") + return + } + + writeJSON(w, http.StatusOK, HarnessConfigFileWriteResponse{ + Path: filePath, + Size: fileSize, + Hash: fileHash, + ModTime: hc.Updated.UTC().Format("2006-01-02T15:04:05Z"), + }) +} + +func (s *Server) handleHarnessConfigFileUpload(w http.ResponseWriter, r *http.Request, id string) { + ctx := r.Context() + + hc, err := s.store.GetHarnessConfig(ctx, id) + if err != nil { + writeErrorFromErr(w, err, "") + return + } + if hc.Locked { + Forbidden(w) + return + } + + // Apply total request body size limit + r.Body = http.MaxBytesReader(w, r.Body, maxUploadTotalSize) + + if err := r.ParseMultipartForm(maxUploadTotalSize); err != nil { + if err.Error() == "http: request body too large" { + BadRequest(w, "Request body exceeds 100MB limit") + return + } + BadRequest(w, "Invalid multipart form: "+err.Error()) + return + } + if r.MultipartForm == nil || len(r.MultipartForm.File) == 0 { + ValidationError(w, "at least one file is required", nil) + return + } + + stor := s.GetStorage() + if stor == nil { + RuntimeError(w, "Storage not configured") + return + } + + files := hc.Files + entries := make([]HarnessConfigFileEntry, 0, len(r.MultipartForm.File)) + for filePath, headers := range r.MultipartForm.File { + if len(headers) == 0 { + continue + } + + src, err := headers[0].Open() + if err != nil { + BadRequest(w, "Failed to open multipart file: "+err.Error()) + return + } + data, err := io.ReadAll(src) + src.Close() + if err != nil { + BadRequest(w, "Failed to read multipart file: "+err.Error()) + return + } + + objectPath := hc.StoragePath + "/" + filePath + _, err = stor.Upload(ctx, objectPath, strings.NewReader(string(data)), storage.UploadOptions{}) + if err != nil { + RuntimeError(w, "Failed to write file to storage") + return + } + + sum := sha256.Sum256(data) + fileHash := "sha256:" + hex.EncodeToString(sum[:]) + fileSize := int64(len(data)) + + updated := false + for i := range files { + if files[i].Path == filePath { + files[i].Hash = fileHash + files[i].Size = fileSize + files[i].Mode = "0644" + updated = true + break + } + } + if !updated { + files = append(files, store.TemplateFile{ + Path: filePath, + Size: fileSize, + Hash: fileHash, + Mode: "0644", + }) + } + + entries = append(entries, HarnessConfigFileEntry{ + Path: filePath, + Size: fileSize, + ModTime: hc.Updated.UTC().Format("2006-01-02T15:04:05Z"), + Mode: "0644", + }) + } + + hc.Files = files + hc.ContentHash = computeContentHash(hc.Files) + if err := s.store.UpdateHarnessConfig(ctx, hc); err != nil { + RuntimeError(w, "Failed to update harness config manifest") + return + } + + writeJSON(w, http.StatusOK, HarnessConfigFileUploadResponse{ + Files: entries, + Hash: hc.ContentHash, + }) +} + +func (s *Server) handleHarnessConfigFileDelete(w http.ResponseWriter, r *http.Request, id, filePath string) { + ctx := r.Context() + + hc, err := s.store.GetHarnessConfig(ctx, id) + if err != nil { + writeErrorFromErr(w, err, "") + return + } + if hc.Locked { + Forbidden(w) + return + } + + stor := s.GetStorage() + if stor == nil { + RuntimeError(w, "Storage not configured") + return + } + + remaining := hc.Files[:0] + found := false + for _, f := range hc.Files { + if f.Path == filePath { + found = true + continue + } + remaining = append(remaining, f) + } + if !found { + NotFound(w, "Harness config file") + return + } + + if err := stor.Delete(ctx, hc.StoragePath+"/"+filePath); err != nil && err != storage.ErrNotFound { + RuntimeError(w, "Failed to delete file from storage") + return + } + + hc.Files = remaining + hc.ContentHash = computeContentHash(hc.Files) + if err := s.store.UpdateHarnessConfig(ctx, hc); err != nil { + RuntimeError(w, "Failed to update harness config manifest") + return + } + + writeJSON(w, http.StatusOK, map[string]string{ + "path": filePath, + "message": fmt.Sprintf("Deleted %s", filePath), + }) +} diff --git a/pkg/hub/harness_config_handlers.go b/pkg/hub/harness_config_handlers.go index 38890ca4..af2d66ca 100644 --- a/pkg/hub/harness_config_handlers.go +++ b/pkg/hub/harness_config_handlers.go @@ -212,7 +212,14 @@ func (s *Server) handleHarnessConfigByID(w http.ResponseWriter, r *http.Request) s.handleHarnessConfigFinalize(w, r, hcID) case "download": s.handleHarnessConfigDownload(w, r, hcID) + case "files": + s.handleHarnessConfigFiles(w, r, hcID, "") default: + if strings.HasPrefix(action, "files/") { + filePath := strings.TrimPrefix(action, "files/") + s.handleHarnessConfigFiles(w, r, hcID, filePath) + return + } NotFound(w, "HarnessConfig action") } } diff --git a/pkg/hubclient/harness_configs.go b/pkg/hubclient/harness_configs.go index 8942a69e..c3ce7674 100644 --- a/pkg/hubclient/harness_configs.go +++ b/pkg/hubclient/harness_configs.go @@ -15,9 +15,16 @@ package hubclient import ( + "bytes" "context" + "fmt" "io" + "mime/multipart" + "net/http" "net/url" + "os" + "path" + "strings" "github.com/GoogleCloudPlatform/scion/pkg/apiclient" "github.com/GoogleCloudPlatform/scion/pkg/transfer" @@ -54,6 +61,12 @@ type HarnessConfigService interface { // DownloadFile downloads a file from the given signed URL. DownloadFile(ctx context.Context, url string) ([]byte, error) + + // UploadFilesMultipart uploads harness config files through the Hub API. + UploadFilesMultipart(ctx context.Context, id string, files []FileInfo) error + + // ReadFile reads a harness config file through the Hub API. + ReadFile(ctx context.Context, id, filePath string) ([]byte, error) } // harnessConfigService is the implementation of HarnessConfigService. @@ -120,6 +133,11 @@ type HarnessConfigFinalizeRequest struct { Manifest *HarnessConfigManifest `json:"manifest"` } +type harnessConfigFileContentResponse struct { + Path string `json:"path"` + Content string `json:"content"` +} + // List returns harness configs matching the filter criteria. func (s *harnessConfigService) List(ctx context.Context, opts *ListHarnessConfigsOptions) (*ListHarnessConfigsResponse, error) { query := url.Values{} @@ -253,9 +271,80 @@ func (s *harnessConfigService) DownloadFile(ctx context.Context, signedURL strin return client.DownloadFile(ctx, signedURL) } +// UploadFilesMultipart uploads harness config files through the Hub API. +func (s *harnessConfigService) UploadFilesMultipart(ctx context.Context, id string, files []FileInfo) error { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + for _, file := range files { + if err := appendMultipartFile(writer, file); err != nil { + return err + } + } + + if err := writer.Close(); err != nil { + return fmt.Errorf("finalize multipart body: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + s.c.transport.BaseURL+"/api/v1/harness-configs/"+id+"/files", &body) + if err != nil { + return fmt.Errorf("create multipart upload request: %w", err) + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + + resp, err := s.c.transport.Do(ctx, req) + if err != nil { + return err + } + return apiclient.CheckResponse(resp) +} + +// ReadFile reads a harness config file through the Hub API. +func (s *harnessConfigService) ReadFile(ctx context.Context, id, filePath string) ([]byte, error) { + endpoint := "/api/v1/harness-configs/" + id + "/files/" + escapePathSegments(filePath) + resp, err := s.c.transport.Get(ctx, endpoint, nil) + if err != nil { + return nil, err + } + + result, err := apiclient.DecodeResponse[harnessConfigFileContentResponse](resp) + if err != nil { + return nil, err + } + return []byte(result.Content), nil +} + func (s *harnessConfigService) getTransferClient() *transfer.Client { if s.transferClient == nil { s.transferClient = transfer.NewClient(s.c.transport.HTTPClient) } return s.transferClient } + +func appendMultipartFile(writer *multipart.Writer, file FileInfo) error { + src, err := os.Open(file.FullPath) + if err != nil { + return fmt.Errorf("open file for multipart upload: %w", err) + } + defer src.Close() + + part, err := writer.CreateFormFile(file.Path, path.Base(file.Path)) + if err != nil { + return fmt.Errorf("create multipart form file: %w", err) + } + + if _, err := io.Copy(part, src); err != nil { + return fmt.Errorf("copy file into multipart body: %w", err) + } + + return nil +} + +func escapePathSegments(filePath string) string { + parts := strings.Split(filePath, "/") + for i, part := range parts { + parts[i] = url.PathEscape(part) + } + return strings.Join(parts, "/") +}