From 42a6b9c958a08fdd0ce77bc1dbab003229267e8a Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 24 Jun 2026 16:23:34 -0700 Subject: [PATCH 01/24] add support for file level auditing / metrics management --- apigen/client/internalapi/internal.gen.go | 371 ++++++++++++- apigen/client/metricsapi/metrics.gen.go | 462 ++++++++++++++++ apigen/openapi/internal.openapi.yaml | 218 +++++++- apigen/openapi/metrics.openapi.yaml | 171 ++++++ apigen/server/internalapi/internal.gen.go | 281 +++++++++- apigen/server/metricsapi/metrics.gen.go | 406 ++++++++++++++ client/client.go | 3 + client/services/interface.go | 1 + client/services/metrics_service.go | 50 +- client/services/repair_service.go | 58 ++ client/services/services_test.go | 34 +- client/services/types.go | 38 ++ internal/api/internaldrs/index.go | 3 +- internal/api/internaldrs/repair_cleanup.go | 83 +++ internal/api/metrics/storage_summary.go | 140 +++++ internal/api/metrics/storage_summary_test.go | 131 +++++ internal/common/routes.go | 28 +- internal/core/storage_cleanup.go | 38 ++ internal/db/interface.go | 10 + internal/db/postgres/metrics.go | 114 ++++ internal/db/postgres/metrics_sqlmock_test.go | 42 +- internal/db/sqlite/metrics.go | 114 ++++ internal/db/sqlite/sqlite_test.go | 79 +++ internal/models/storage.go | 62 ++- internal/repair/format.go | 84 +++ internal/repair/service.go | 502 ++++++++++++++++++ internal/repair/service_test.go | 268 ++++++++++ internal/repair/storage_cleanup_service.go | 421 +++++++++++++++ .../repair/storage_cleanup_service_test.go | 207 ++++++++ internal/repair/storage_cleanup_types.go | 94 ++++ internal/repair/types.go | 137 +++++ internal/storagemetrics/storage_metrics.go | 248 +++++++++ .../storagemetrics/storage_metrics_test.go | 96 ++++ internal/testutils/mocks.go | 117 ++++ 34 files changed, 5069 insertions(+), 42 deletions(-) create mode 100644 client/services/repair_service.go create mode 100644 internal/api/internaldrs/repair_cleanup.go create mode 100644 internal/api/metrics/storage_summary.go create mode 100644 internal/api/metrics/storage_summary_test.go create mode 100644 internal/core/storage_cleanup.go create mode 100644 internal/repair/format.go create mode 100644 internal/repair/service.go create mode 100644 internal/repair/service_test.go create mode 100644 internal/repair/storage_cleanup_service.go create mode 100644 internal/repair/storage_cleanup_service_test.go create mode 100644 internal/repair/storage_cleanup_types.go create mode 100644 internal/repair/types.go create mode 100644 internal/storagemetrics/storage_metrics.go create mode 100644 internal/storagemetrics/storage_metrics_test.go diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index 11f0ae9c..de99bfa2 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "strings" + "time" externalRef0 "github.com/calypr/syfon/apigen/client/drs" "github.com/oapi-codegen/runtime" @@ -32,8 +33,7 @@ type BulkDocumentsRequest0 = []string // BulkDocumentsRequest1 defines model for . type BulkDocumentsRequest1 struct { - Dids *[]string `json:"dids,omitempty"` - Ids *[]string `json:"ids,omitempty"` + Ids *[]string `json:"ids,omitempty"` } // BulkHashesRequest defines model for BulkHashesRequest. @@ -204,6 +204,89 @@ type ListRecordsResponse struct { Records *[]InternalRecord `json:"records,omitempty"` } +// StorageCleanupApplyRequest defines model for StorageCleanupApplyRequest. +type StorageCleanupApplyRequest struct { + CheckStorage *bool `json:"check_storage,omitempty"` + DeleteRepoOrphans *bool `json:"delete_repo_orphans,omitempty"` + DeleteStaleDuplicates *bool `json:"delete_stale_duplicates,omitempty"` + DryRun *bool `json:"dry_run,omitempty"` + ExpectedPaths *[]string `json:"expected_paths,omitempty"` + Organization string `json:"organization"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project string `json:"project"` + SelectedFindingKinds *[]string `json:"selected_finding_kinds,omitempty"` + SelectedObjectIds *[]string `json:"selected_object_ids,omitempty"` + SelectedPaths *[]string `json:"selected_paths,omitempty"` +} + +// StorageCleanupApplyResult defines model for StorageCleanupApplyResult. +type StorageCleanupApplyResult struct { + DeletedRecordIds *[]string `json:"deleted_record_ids,omitempty"` + DryRun *bool `json:"dry_run,omitempty"` + RepoDeletePaths *[]string `json:"repo_delete_paths,omitempty"` + Report *StorageCleanupReport `json:"report,omitempty"` + Skipped *[]StorageCleanupSkipped `json:"skipped,omitempty"` + StoragePurgeResults *[]StorageCleanupStoragePurgeResult `json:"storage_purge_results,omitempty"` +} + +// StorageCleanupAuditRequest defines model for StorageCleanupAuditRequest. +type StorageCleanupAuditRequest struct { + CheckStorage *bool `json:"check_storage,omitempty"` + ExpectedPaths *[]string `json:"expected_paths,omitempty"` + Organization string `json:"organization"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project string `json:"project"` +} + +// StorageCleanupFinding defines model for StorageCleanupFinding. +type StorageCleanupFinding struct { + Kind *string `json:"kind,omitempty"` + Message *string `json:"message,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + RecommendedAction *string `json:"recommended_action,omitempty"` + Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` + RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` + Severity *string `json:"severity,omitempty"` +} + +// StorageCleanupRecordAudit defines model for StorageCleanupRecordAudit. +type StorageCleanupRecordAudit struct { + CurrentAccessUrls *[]string `json:"current_access_urls,omitempty"` + DownloadCount *int64 `json:"download_count,omitempty"` + LastDownloadTime *time.Time `json:"last_download_time,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + ObjectId *string `json:"object_id,omitempty"` + Size *int64 `json:"size,omitempty"` + StorageMessage *string `json:"storage_message,omitempty"` + StorageStatus *string `json:"storage_status,omitempty"` + UpdatedTime *time.Time `json:"updated_time,omitempty"` +} + +// StorageCleanupReport defines model for StorageCleanupReport. +type StorageCleanupReport struct { + Findings *[]StorageCleanupFinding `json:"findings,omitempty"` + Organization *string `json:"organization,omitempty"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project *string `json:"project,omitempty"` + Scanned *int `json:"scanned,omitempty"` + Summary *map[string]int `json:"summary,omitempty"` +} + +// StorageCleanupSkipped defines model for StorageCleanupSkipped. +type StorageCleanupSkipped struct { + Kind *string `json:"kind,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + ObjectId *string `json:"object_id,omitempty"` + Reason *string `json:"reason,omitempty"` +} + +// StorageCleanupStoragePurgeResult defines model for StorageCleanupStoragePurgeResult. +type StorageCleanupStoragePurgeResult struct { + Message *string `json:"message,omitempty"` + ObjectId *string `json:"object_id,omitempty"` + Purged *bool `json:"purged,omitempty"` +} + // InternalDownloadParams defines parameters for InternalDownload. type InternalDownloadParams struct { Redirect *bool `form:"redirect,omitempty" json:"redirect,omitempty"` @@ -255,6 +338,12 @@ type InternalMultipartInitJSONRequestBody = InternalMultipartInitRequest // InternalMultipartUploadJSONRequestBody defines body for InternalMultipartUpload for application/json ContentType. type InternalMultipartUploadJSONRequestBody = InternalMultipartUploadRequest +// InternalStorageCleanupApplyJSONRequestBody defines body for InternalStorageCleanupApply for application/json ContentType. +type InternalStorageCleanupApplyJSONRequestBody = StorageCleanupApplyRequest + +// InternalStorageCleanupAuditJSONRequestBody defines body for InternalStorageCleanupAudit for application/json ContentType. +type InternalStorageCleanupAuditJSONRequestBody = StorageCleanupAuditRequest + // InternalUploadBlankJSONRequestBody defines body for InternalUploadBlank for application/json ContentType. type InternalUploadBlankJSONRequestBody = InternalUploadBlankRequest @@ -441,6 +530,16 @@ type ClientInterface interface { InternalMultipartUpload(ctx context.Context, body InternalMultipartUploadJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalStorageCleanupApplyWithBody request with any body + InternalStorageCleanupApplyWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + InternalStorageCleanupApply(ctx context.Context, body InternalStorageCleanupApplyJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + + // InternalStorageCleanupAuditWithBody request with any body + InternalStorageCleanupAuditWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + InternalStorageCleanupAudit(ctx context.Context, body InternalStorageCleanupAuditJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalUploadBlankWithBody request with any body InternalUploadBlankWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -603,6 +702,54 @@ func (c *Client) InternalMultipartUpload(ctx context.Context, body InternalMulti return c.Client.Do(req) } +func (c *Client) InternalStorageCleanupApplyWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalStorageCleanupApplyRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalStorageCleanupApply(ctx context.Context, body InternalStorageCleanupApplyJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalStorageCleanupApplyRequest(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalStorageCleanupAuditWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalStorageCleanupAuditRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalStorageCleanupAudit(ctx context.Context, body InternalStorageCleanupAuditJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalStorageCleanupAuditRequest(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) InternalUploadBlankWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewInternalUploadBlankRequestWithBody(c.Server, contentType, body) if err != nil { @@ -1159,6 +1306,86 @@ func NewInternalMultipartUploadRequestWithBody(server string, contentType string return req, nil } +// NewInternalStorageCleanupApplyRequest calls the generic InternalStorageCleanupApply builder with application/json body +func NewInternalStorageCleanupApplyRequest(server string, body InternalStorageCleanupApplyJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewInternalStorageCleanupApplyRequestWithBody(server, "application/json", bodyReader) +} + +// NewInternalStorageCleanupApplyRequestWithBody generates requests for InternalStorageCleanupApply with any type of body +func NewInternalStorageCleanupApplyRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/data/repair/storage-cleanup/apply") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +// NewInternalStorageCleanupAuditRequest calls the generic InternalStorageCleanupAudit builder with application/json body +func NewInternalStorageCleanupAuditRequest(server string, body InternalStorageCleanupAuditJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewInternalStorageCleanupAuditRequestWithBody(server, "application/json", bodyReader) +} + +// NewInternalStorageCleanupAuditRequestWithBody generates requests for InternalStorageCleanupAudit with any type of body +func NewInternalStorageCleanupAuditRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/data/repair/storage-cleanup/audit") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewInternalUploadBlankRequest calls the generic InternalUploadBlank builder with application/json body func NewInternalUploadBlankRequest(server string, body InternalUploadBlankJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -2099,6 +2326,16 @@ type ClientWithResponsesInterface interface { InternalMultipartUploadWithResponse(ctx context.Context, body InternalMultipartUploadJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalMultipartUploadResponse, error) + // InternalStorageCleanupApplyWithBodyWithResponse request with any body + InternalStorageCleanupApplyWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalStorageCleanupApplyResponse, error) + + InternalStorageCleanupApplyWithResponse(ctx context.Context, body InternalStorageCleanupApplyJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalStorageCleanupApplyResponse, error) + + // InternalStorageCleanupAuditWithBodyWithResponse request with any body + InternalStorageCleanupAuditWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalStorageCleanupAuditResponse, error) + + InternalStorageCleanupAuditWithResponse(ctx context.Context, body InternalStorageCleanupAuditJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalStorageCleanupAuditResponse, error) + // InternalUploadBlankWithBodyWithResponse request with any body InternalUploadBlankWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalUploadBlankResponse, error) @@ -2274,6 +2511,50 @@ func (r InternalMultipartUploadResponse) StatusCode() int { return 0 } +type InternalStorageCleanupApplyResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *StorageCleanupApplyResult +} + +// Status returns HTTPResponse.Status +func (r InternalStorageCleanupApplyResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalStorageCleanupApplyResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type InternalStorageCleanupAuditResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *StorageCleanupReport +} + +// Status returns HTTPResponse.Status +func (r InternalStorageCleanupAuditResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalStorageCleanupAuditResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type InternalUploadBlankResponse struct { Body []byte HTTPResponse *http.Response @@ -2673,6 +2954,40 @@ func (c *ClientWithResponses) InternalMultipartUploadWithResponse(ctx context.Co return ParseInternalMultipartUploadResponse(rsp) } +// InternalStorageCleanupApplyWithBodyWithResponse request with arbitrary body returning *InternalStorageCleanupApplyResponse +func (c *ClientWithResponses) InternalStorageCleanupApplyWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalStorageCleanupApplyResponse, error) { + rsp, err := c.InternalStorageCleanupApplyWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalStorageCleanupApplyResponse(rsp) +} + +func (c *ClientWithResponses) InternalStorageCleanupApplyWithResponse(ctx context.Context, body InternalStorageCleanupApplyJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalStorageCleanupApplyResponse, error) { + rsp, err := c.InternalStorageCleanupApply(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalStorageCleanupApplyResponse(rsp) +} + +// InternalStorageCleanupAuditWithBodyWithResponse request with arbitrary body returning *InternalStorageCleanupAuditResponse +func (c *ClientWithResponses) InternalStorageCleanupAuditWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalStorageCleanupAuditResponse, error) { + rsp, err := c.InternalStorageCleanupAuditWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalStorageCleanupAuditResponse(rsp) +} + +func (c *ClientWithResponses) InternalStorageCleanupAuditWithResponse(ctx context.Context, body InternalStorageCleanupAuditJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalStorageCleanupAuditResponse, error) { + rsp, err := c.InternalStorageCleanupAudit(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalStorageCleanupAuditResponse(rsp) +} + // InternalUploadBlankWithBodyWithResponse request with arbitrary body returning *InternalUploadBlankResponse func (c *ClientWithResponses) InternalUploadBlankWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalUploadBlankResponse, error) { rsp, err := c.InternalUploadBlankWithBody(ctx, contentType, body, reqEditors...) @@ -3008,6 +3323,58 @@ func ParseInternalMultipartUploadResponse(rsp *http.Response) (*InternalMultipar return response, nil } +// ParseInternalStorageCleanupApplyResponse parses an HTTP response from a InternalStorageCleanupApplyWithResponse call +func ParseInternalStorageCleanupApplyResponse(rsp *http.Response) (*InternalStorageCleanupApplyResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalStorageCleanupApplyResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest StorageCleanupApplyResult + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseInternalStorageCleanupAuditResponse parses an HTTP response from a InternalStorageCleanupAuditWithResponse call +func ParseInternalStorageCleanupAuditResponse(rsp *http.Response) (*InternalStorageCleanupAuditResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalStorageCleanupAuditResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest StorageCleanupReport + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseInternalUploadBlankResponse parses an HTTP response from a InternalUploadBlankWithResponse call func ParseInternalUploadBlankResponse(rsp *http.Response) (*InternalUploadBlankResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/client/metricsapi/metrics.gen.go b/apigen/client/metricsapi/metrics.gen.go index 863b0183..e09245ef 100644 --- a/apigen/client/metricsapi/metrics.gen.go +++ b/apigen/client/metricsapi/metrics.gen.go @@ -31,6 +31,12 @@ const ( Unmatched ProviderTransferReconciliationStatus = "unmatched" ) +// Defines values for StoragePathChildType. +const ( + Directory StoragePathChildType = "directory" + File StoragePathChildType = "file" +) + // Defines values for TransferBreakdownResponseGroupBy. const ( TransferBreakdownResponseGroupByObject TransferBreakdownResponseGroupBy = "object" @@ -39,6 +45,20 @@ const ( TransferBreakdownResponseGroupByUser TransferBreakdownResponseGroupBy = "user" ) +// Defines values for ListStorageChildrenParamsSortBy. +const ( + Bytes ListStorageChildrenParamsSortBy = "bytes" + Name ListStorageChildrenParamsSortBy = "name" + Records ListStorageChildrenParamsSortBy = "records" + UpdatedTime ListStorageChildrenParamsSortBy = "updated_time" +) + +// Defines values for ListStorageChildrenParamsSortOrder. +const ( + Asc ListStorageChildrenParamsSortOrder = "asc" + Desc ListStorageChildrenParamsSortOrder = "desc" +) + // Defines values for GetTransferBreakdownParamsGroupBy. const ( GetTransferBreakdownParamsGroupByObject GetTransferBreakdownParamsGroupBy = "object" @@ -122,6 +142,45 @@ type ProviderTransferEventsRequest struct { // ProviderTransferReconciliationStatus defines model for ProviderTransferReconciliationStatus. type ProviderTransferReconciliationStatus string +// StoragePathChild defines model for StoragePathChild. +type StoragePathChild struct { + DownloadCount *int64 `json:"download_count,omitempty"` + FileCount *int64 `json:"file_count,omitempty"` + LastDownloadTime *time.Time `json:"last_download_time"` + LatestUpdateTime *time.Time `json:"latest_update_time"` + Name *string `json:"name,omitempty"` + Path *string `json:"path,omitempty"` + RecordCount *int64 `json:"record_count,omitempty"` + TotalBytes *int64 `json:"total_bytes,omitempty"` + Type *StoragePathChildType `json:"type,omitempty"` +} + +// StoragePathChildType defines model for StoragePathChild.Type. +type StoragePathChildType string + +// StoragePathChildrenResponse defines model for StoragePathChildrenResponse. +type StoragePathChildrenResponse struct { + Items *[]StoragePathChild `json:"items,omitempty"` + Organization *string `json:"organization,omitempty"` + Path *string `json:"path,omitempty"` + Project *string `json:"project,omitempty"` +} + +// StoragePathSummary defines model for StoragePathSummary. +type StoragePathSummary struct { + DirectChildCount *int64 `json:"direct_child_count,omitempty"` + DownloadCount *int64 `json:"download_count,omitempty"` + DuplicatePathCount *int64 `json:"duplicate_path_count,omitempty"` + FileCount *int64 `json:"file_count,omitempty"` + LastDownloadTime *time.Time `json:"last_download_time"` + LatestUpdateTime *time.Time `json:"latest_update_time"` + Organization *string `json:"organization,omitempty"` + Path *string `json:"path,omitempty"` + Project *string `json:"project,omitempty"` + RecordCount *int64 `json:"record_count,omitempty"` + TotalBytes *int64 `json:"total_bytes,omitempty"` +} + // TransferAttributionBreakdown defines model for TransferAttributionBreakdown. type TransferAttributionBreakdown struct { ActorEmail *string `json:"actor_email,omitempty"` @@ -190,6 +249,9 @@ type From = time.Time // Organization defines model for Organization. type Organization = string +// Path defines model for Path. +type Path = string + // Project defines model for Project. type Project = string @@ -202,6 +264,12 @@ type ReconciliationStatus = ProviderTransferReconciliationStatus // SHA256 defines model for SHA256. type SHA256 = string +// StorageOrganization defines model for StorageOrganization. +type StorageOrganization = string + +// StorageProject defines model for StorageProject. +type StorageProject = string + // To defines model for To. type To = time.Time @@ -239,6 +307,40 @@ type RecordProviderTransferEventsParams struct { Project *Project `form:"project,omitempty" json:"project,omitempty"` } +// ListStorageChildrenParams defines parameters for ListStorageChildren. +type ListStorageChildrenParams struct { + // Organization Organization scope for the storage subtree query. + Organization StorageOrganization `form:"organization" json:"organization"` + + // Project Project scope for the storage subtree query. + Project StorageProject `form:"project" json:"project"` + + // Path Optional storage subtree path. Omit for the project root. + Path *Path `form:"path,omitempty" json:"path,omitempty"` + Limit *int `form:"limit,omitempty" json:"limit,omitempty"` + Offset *int `form:"offset,omitempty" json:"offset,omitempty"` + SortBy *ListStorageChildrenParamsSortBy `form:"sort_by,omitempty" json:"sort_by,omitempty"` + SortOrder *ListStorageChildrenParamsSortOrder `form:"sort_order,omitempty" json:"sort_order,omitempty"` +} + +// ListStorageChildrenParamsSortBy defines parameters for ListStorageChildren. +type ListStorageChildrenParamsSortBy string + +// ListStorageChildrenParamsSortOrder defines parameters for ListStorageChildren. +type ListStorageChildrenParamsSortOrder string + +// GetStorageSummaryParams defines parameters for GetStorageSummary. +type GetStorageSummaryParams struct { + // Organization Organization scope for the storage subtree query. + Organization StorageOrganization `form:"organization" json:"organization"` + + // Project Project scope for the storage subtree query. + Project StorageProject `form:"project" json:"project"` + + // Path Optional storage subtree path. Omit for the project root. + Path *Path `form:"path,omitempty" json:"path,omitempty"` +} + // GetMetricsSummaryParams defines parameters for GetMetricsSummary. type GetMetricsSummaryParams struct { InactiveDays *int `form:"inactive_days,omitempty" json:"inactive_days,omitempty"` @@ -385,6 +487,12 @@ type ClientInterface interface { RecordProviderTransferEvents(ctx context.Context, params *RecordProviderTransferEventsParams, body RecordProviderTransferEventsJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // ListStorageChildren request + ListStorageChildren(ctx context.Context, params *ListStorageChildrenParams, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetStorageSummary request + GetStorageSummary(ctx context.Context, params *GetStorageSummaryParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetMetricsSummary request GetMetricsSummary(ctx context.Context, params *GetMetricsSummaryParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -443,6 +551,30 @@ func (c *Client) RecordProviderTransferEvents(ctx context.Context, params *Recor return c.Client.Do(req) } +func (c *Client) ListStorageChildren(ctx context.Context, params *ListStorageChildrenParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewListStorageChildrenRequest(c.Server, params) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) GetStorageSummary(ctx context.Context, params *GetStorageSummaryParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetStorageSummaryRequest(c.Server, params) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) GetMetricsSummary(ctx context.Context, params *GetMetricsSummaryParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetMetricsSummaryRequest(c.Server, params) if err != nil { @@ -742,6 +874,216 @@ func NewRecordProviderTransferEventsRequestWithBody(server string, params *Recor return req, nil } +// NewListStorageChildrenRequest generates requests for ListStorageChildren +func NewListStorageChildrenRequest(server string, params *ListStorageChildrenParams) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/index/v1/metrics/storage/children") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "organization", runtime.ParamLocationQuery, params.Organization); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "project", runtime.ParamLocationQuery, params.Project); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + if params.Path != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "path", runtime.ParamLocationQuery, *params.Path); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.Limit != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "limit", runtime.ParamLocationQuery, *params.Limit); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.Offset != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "offset", runtime.ParamLocationQuery, *params.Offset); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.SortBy != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "sort_by", runtime.ParamLocationQuery, *params.SortBy); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.SortOrder != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "sort_order", runtime.ParamLocationQuery, *params.SortOrder); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewGetStorageSummaryRequest generates requests for GetStorageSummary +func NewGetStorageSummaryRequest(server string, params *GetStorageSummaryParams) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/index/v1/metrics/storage/summary") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "organization", runtime.ParamLocationQuery, params.Organization); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "project", runtime.ParamLocationQuery, params.Project); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + if params.Path != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "path", runtime.ParamLocationQuery, *params.Path); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewGetMetricsSummaryRequest generates requests for GetMetricsSummary func NewGetMetricsSummaryRequest(server string, params *GetMetricsSummaryParams) (*http.Request, error) { var err error @@ -1311,6 +1653,12 @@ type ClientWithResponsesInterface interface { RecordProviderTransferEventsWithResponse(ctx context.Context, params *RecordProviderTransferEventsParams, body RecordProviderTransferEventsJSONRequestBody, reqEditors ...RequestEditorFn) (*RecordProviderTransferEventsResponse, error) + // ListStorageChildrenWithResponse request + ListStorageChildrenWithResponse(ctx context.Context, params *ListStorageChildrenParams, reqEditors ...RequestEditorFn) (*ListStorageChildrenResponse, error) + + // GetStorageSummaryWithResponse request + GetStorageSummaryWithResponse(ctx context.Context, params *GetStorageSummaryParams, reqEditors ...RequestEditorFn) (*GetStorageSummaryResponse, error) + // GetMetricsSummaryWithResponse request GetMetricsSummaryWithResponse(ctx context.Context, params *GetMetricsSummaryParams, reqEditors ...RequestEditorFn) (*GetMetricsSummaryResponse, error) @@ -1387,6 +1735,50 @@ func (r RecordProviderTransferEventsResponse) StatusCode() int { return 0 } +type ListStorageChildrenResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *StoragePathChildrenResponse +} + +// Status returns HTTPResponse.Status +func (r ListStorageChildrenResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r ListStorageChildrenResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type GetStorageSummaryResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *StoragePathSummary +} + +// Status returns HTTPResponse.Status +func (r GetStorageSummaryResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetStorageSummaryResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type GetMetricsSummaryResponse struct { Body []byte HTTPResponse *http.Response @@ -1488,6 +1880,24 @@ func (c *ClientWithResponses) RecordProviderTransferEventsWithResponse(ctx conte return ParseRecordProviderTransferEventsResponse(rsp) } +// ListStorageChildrenWithResponse request returning *ListStorageChildrenResponse +func (c *ClientWithResponses) ListStorageChildrenWithResponse(ctx context.Context, params *ListStorageChildrenParams, reqEditors ...RequestEditorFn) (*ListStorageChildrenResponse, error) { + rsp, err := c.ListStorageChildren(ctx, params, reqEditors...) + if err != nil { + return nil, err + } + return ParseListStorageChildrenResponse(rsp) +} + +// GetStorageSummaryWithResponse request returning *GetStorageSummaryResponse +func (c *ClientWithResponses) GetStorageSummaryWithResponse(ctx context.Context, params *GetStorageSummaryParams, reqEditors ...RequestEditorFn) (*GetStorageSummaryResponse, error) { + rsp, err := c.GetStorageSummary(ctx, params, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetStorageSummaryResponse(rsp) +} + // GetMetricsSummaryWithResponse request returning *GetMetricsSummaryResponse func (c *ClientWithResponses) GetMetricsSummaryWithResponse(ctx context.Context, params *GetMetricsSummaryParams, reqEditors ...RequestEditorFn) (*GetMetricsSummaryResponse, error) { rsp, err := c.GetMetricsSummary(ctx, params, reqEditors...) @@ -1593,6 +2003,58 @@ func ParseRecordProviderTransferEventsResponse(rsp *http.Response) (*RecordProvi return response, nil } +// ParseListStorageChildrenResponse parses an HTTP response from a ListStorageChildrenWithResponse call +func ParseListStorageChildrenResponse(rsp *http.Response) (*ListStorageChildrenResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &ListStorageChildrenResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest StoragePathChildrenResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseGetStorageSummaryResponse parses an HTTP response from a GetStorageSummaryWithResponse call +func ParseGetStorageSummaryResponse(rsp *http.Response) (*GetStorageSummaryResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetStorageSummaryResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest StoragePathSummary + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseGetMetricsSummaryResponse parses an HTTP response from a GetMetricsSummaryWithResponse call func ParseGetMetricsSummaryResponse(rsp *http.Response) (*GetMetricsSummaryResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/openapi/internal.openapi.yaml b/apigen/openapi/internal.openapi.yaml index f0e53d17..522a5a94 100644 --- a/apigen/openapi/internal.openapi.yaml +++ b/apigen/openapi/internal.openapi.yaml @@ -633,6 +633,58 @@ paths: description: Forbidden '500': description: Internal server error + /data/repair/storage-cleanup/audit: + post: + tags: + - internal-data + operationId: internalStorageCleanupAudit + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/StorageCleanupAuditRequest' + responses: + '200': + description: Storage cleanup audit report + content: + application/json: + schema: + $ref: '#/components/schemas/StorageCleanupReport' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error + /data/repair/storage-cleanup/apply: + post: + tags: + - internal-data + operationId: internalStorageCleanupApply + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/StorageCleanupApplyRequest' + responses: + '200': + description: Storage cleanup apply result + content: + application/json: + schema: + $ref: '#/components/schemas/StorageCleanupApplyResult' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error components: schemas: ControlledAccessRemoveRequest: @@ -760,10 +812,168 @@ components: type: array items: type: string - dids: - type: array - items: - type: string + StorageCleanupAuditRequest: + type: object + required: + - organization + - project + properties: + organization: + type: string + project: + type: string + path_prefix: + type: string + expected_paths: + type: array + items: + type: string + check_storage: + type: boolean + StorageCleanupApplyRequest: + type: object + required: + - organization + - project + properties: + organization: + type: string + project: + type: string + path_prefix: + type: string + expected_paths: + type: array + items: + type: string + delete_stale_duplicates: + type: boolean + delete_repo_orphans: + type: boolean + dry_run: + type: boolean + selected_paths: + type: array + items: + type: string + selected_object_ids: + type: array + items: + type: string + selected_finding_kinds: + type: array + items: + type: string + check_storage: + type: boolean + StorageCleanupReport: + type: object + properties: + organization: + type: string + project: + type: string + path_prefix: + type: string + scanned: + type: integer + summary: + type: object + additionalProperties: + type: integer + findings: + type: array + items: + $ref: '#/components/schemas/StorageCleanupFinding' + StorageCleanupFinding: + type: object + properties: + kind: + type: string + severity: + type: string + normalized_path: + type: string + message: + type: string + recommended_action: + type: string + repo_delete_candidate: + type: boolean + records: + type: array + items: + $ref: '#/components/schemas/StorageCleanupRecordAudit' + StorageCleanupRecordAudit: + type: object + properties: + object_id: + type: string + normalized_path: + type: string + size: + type: integer + format: int64 + updated_time: + type: string + format: date-time + download_count: + type: integer + format: int64 + last_download_time: + type: string + format: date-time + current_access_urls: + type: array + items: + type: string + storage_status: + type: string + storage_message: + type: string + StorageCleanupApplyResult: + type: object + properties: + report: + $ref: '#/components/schemas/StorageCleanupReport' + dry_run: + type: boolean + deleted_record_ids: + type: array + items: + type: string + storage_purge_results: + type: array + items: + $ref: '#/components/schemas/StorageCleanupStoragePurgeResult' + repo_delete_paths: + type: array + items: + type: string + skipped: + type: array + items: + $ref: '#/components/schemas/StorageCleanupSkipped' + StorageCleanupStoragePurgeResult: + type: object + properties: + object_id: + type: string + purged: + type: boolean + message: + type: string + StorageCleanupSkipped: + type: object + properties: + normalized_path: + type: string + object_id: + type: string + kind: + type: string + reason: + type: string DeleteByQueryResponse: type: object properties: diff --git a/apigen/openapi/metrics.openapi.yaml b/apigen/openapi/metrics.openapi.yaml index 5d1c502e..73297541 100644 --- a/apigen/openapi/metrics.openapi.yaml +++ b/apigen/openapi/metrics.openapi.yaml @@ -107,6 +107,79 @@ paths: description: Forbidden '500': description: Internal server error + /index/v1/metrics/storage/summary: + get: + tags: + - metrics + operationId: getStorageSummary + parameters: + - $ref: '#/components/parameters/StorageOrganization' + - $ref: '#/components/parameters/StorageProject' + - $ref: '#/components/parameters/Path' + responses: + '200': + description: Aggregate storage metrics for one path subtree + content: + application/json: + schema: + $ref: '#/components/schemas/StoragePathSummary' + '400': + description: Invalid query params + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error + /index/v1/metrics/storage/children: + get: + tags: + - metrics + operationId: listStorageChildren + parameters: + - $ref: '#/components/parameters/StorageOrganization' + - $ref: '#/components/parameters/StorageProject' + - $ref: '#/components/parameters/Path' + - in: query + name: limit + schema: + type: integer + minimum: 1 + maximum: 1000 + default: 200 + - in: query + name: offset + schema: + type: integer + minimum: 0 + default: 0 + - in: query + name: sort_by + schema: + type: string + enum: [bytes, name, updated_time, records] + default: name + - in: query + name: sort_order + schema: + type: string + enum: [asc, desc] + default: asc + responses: + '200': + description: Direct child storage metrics for one path subtree + content: + application/json: + schema: + $ref: '#/components/schemas/StoragePathChildrenResponse' + '400': + description: Invalid query params + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error /index/v1/metrics/provider-transfer-events: post: tags: @@ -226,6 +299,26 @@ components: schema: type: string description: Project scope filter. Requires organization when set. + StorageOrganization: + in: query + name: organization + required: true + schema: + type: string + description: Organization scope for the storage subtree query. + StorageProject: + in: query + name: project + required: true + schema: + type: string + description: Project scope for the storage subtree query. + Path: + in: query + name: path + schema: + type: string + description: Optional storage subtree path. Omit for the project root. Direction: in: query name: direction @@ -332,6 +425,84 @@ components: type: integer offset: type: integer + StoragePathSummary: + type: object + properties: + organization: + type: string + project: + type: string + path: + type: string + file_count: + type: integer + format: int64 + record_count: + type: integer + format: int64 + total_bytes: + type: integer + format: int64 + download_count: + type: integer + format: int64 + direct_child_count: + type: integer + format: int64 + duplicate_path_count: + type: integer + format: int64 + latest_update_time: + type: string + format: date-time + nullable: true + last_download_time: + type: string + format: date-time + nullable: true + StoragePathChild: + type: object + properties: + name: + type: string + path: + type: string + type: + type: string + enum: [file, directory] + file_count: + type: integer + format: int64 + record_count: + type: integer + format: int64 + total_bytes: + type: integer + format: int64 + download_count: + type: integer + format: int64 + latest_update_time: + type: string + format: date-time + nullable: true + last_download_time: + type: string + format: date-time + nullable: true + StoragePathChildrenResponse: + type: object + properties: + organization: + type: string + project: + type: string + path: + type: string + items: + type: array + items: + $ref: '#/components/schemas/StoragePathChild' ProviderTransferDirection: type: string enum: diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 5dac272a..79b0e569 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "time" externalRef0 "github.com/calypr/syfon/apigen/server/drs" "github.com/gofiber/fiber/v3" @@ -30,8 +31,7 @@ type BulkDocumentsRequest0 = []string // BulkDocumentsRequest1 defines model for . type BulkDocumentsRequest1 struct { - Dids *[]string `json:"dids,omitempty"` - Ids *[]string `json:"ids,omitempty"` + Ids *[]string `json:"ids,omitempty"` } // BulkHashesRequest defines model for BulkHashesRequest. @@ -202,6 +202,89 @@ type ListRecordsResponse struct { Records *[]InternalRecord `json:"records,omitempty"` } +// StorageCleanupApplyRequest defines model for StorageCleanupApplyRequest. +type StorageCleanupApplyRequest struct { + CheckStorage *bool `json:"check_storage,omitempty"` + DeleteRepoOrphans *bool `json:"delete_repo_orphans,omitempty"` + DeleteStaleDuplicates *bool `json:"delete_stale_duplicates,omitempty"` + DryRun *bool `json:"dry_run,omitempty"` + ExpectedPaths *[]string `json:"expected_paths,omitempty"` + Organization string `json:"organization"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project string `json:"project"` + SelectedFindingKinds *[]string `json:"selected_finding_kinds,omitempty"` + SelectedObjectIds *[]string `json:"selected_object_ids,omitempty"` + SelectedPaths *[]string `json:"selected_paths,omitempty"` +} + +// StorageCleanupApplyResult defines model for StorageCleanupApplyResult. +type StorageCleanupApplyResult struct { + DeletedRecordIds *[]string `json:"deleted_record_ids,omitempty"` + DryRun *bool `json:"dry_run,omitempty"` + RepoDeletePaths *[]string `json:"repo_delete_paths,omitempty"` + Report *StorageCleanupReport `json:"report,omitempty"` + Skipped *[]StorageCleanupSkipped `json:"skipped,omitempty"` + StoragePurgeResults *[]StorageCleanupStoragePurgeResult `json:"storage_purge_results,omitempty"` +} + +// StorageCleanupAuditRequest defines model for StorageCleanupAuditRequest. +type StorageCleanupAuditRequest struct { + CheckStorage *bool `json:"check_storage,omitempty"` + ExpectedPaths *[]string `json:"expected_paths,omitempty"` + Organization string `json:"organization"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project string `json:"project"` +} + +// StorageCleanupFinding defines model for StorageCleanupFinding. +type StorageCleanupFinding struct { + Kind *string `json:"kind,omitempty"` + Message *string `json:"message,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + RecommendedAction *string `json:"recommended_action,omitempty"` + Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` + RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` + Severity *string `json:"severity,omitempty"` +} + +// StorageCleanupRecordAudit defines model for StorageCleanupRecordAudit. +type StorageCleanupRecordAudit struct { + CurrentAccessUrls *[]string `json:"current_access_urls,omitempty"` + DownloadCount *int64 `json:"download_count,omitempty"` + LastDownloadTime *time.Time `json:"last_download_time,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + ObjectId *string `json:"object_id,omitempty"` + Size *int64 `json:"size,omitempty"` + StorageMessage *string `json:"storage_message,omitempty"` + StorageStatus *string `json:"storage_status,omitempty"` + UpdatedTime *time.Time `json:"updated_time,omitempty"` +} + +// StorageCleanupReport defines model for StorageCleanupReport. +type StorageCleanupReport struct { + Findings *[]StorageCleanupFinding `json:"findings,omitempty"` + Organization *string `json:"organization,omitempty"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project *string `json:"project,omitempty"` + Scanned *int `json:"scanned,omitempty"` + Summary *map[string]int `json:"summary,omitempty"` +} + +// StorageCleanupSkipped defines model for StorageCleanupSkipped. +type StorageCleanupSkipped struct { + Kind *string `json:"kind,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + ObjectId *string `json:"object_id,omitempty"` + Reason *string `json:"reason,omitempty"` +} + +// StorageCleanupStoragePurgeResult defines model for StorageCleanupStoragePurgeResult. +type StorageCleanupStoragePurgeResult struct { + Message *string `json:"message,omitempty"` + ObjectId *string `json:"object_id,omitempty"` + Purged *bool `json:"purged,omitempty"` +} + // InternalDownloadParams defines parameters for InternalDownload. type InternalDownloadParams struct { Redirect *bool `form:"redirect,omitempty" json:"redirect,omitempty"` @@ -253,6 +336,12 @@ type InternalMultipartInitJSONRequestBody = InternalMultipartInitRequest // InternalMultipartUploadJSONRequestBody defines body for InternalMultipartUpload for application/json ContentType. type InternalMultipartUploadJSONRequestBody = InternalMultipartUploadRequest +// InternalStorageCleanupApplyJSONRequestBody defines body for InternalStorageCleanupApply for application/json ContentType. +type InternalStorageCleanupApplyJSONRequestBody = StorageCleanupApplyRequest + +// InternalStorageCleanupAuditJSONRequestBody defines body for InternalStorageCleanupAudit for application/json ContentType. +type InternalStorageCleanupAuditJSONRequestBody = StorageCleanupAuditRequest + // InternalUploadBlankJSONRequestBody defines body for InternalUploadBlank for application/json ContentType. type InternalUploadBlankJSONRequestBody = InternalUploadBlankRequest @@ -363,6 +452,12 @@ type ServerInterface interface { // (POST /data/multipart/upload) InternalMultipartUpload(c fiber.Ctx) error + // (POST /data/repair/storage-cleanup/apply) + InternalStorageCleanupApply(c fiber.Ctx) error + + // (POST /data/repair/storage-cleanup/audit) + InternalStorageCleanupAudit(c fiber.Ctx) error + // (POST /data/upload) InternalUploadBlank(c fiber.Ctx) error @@ -516,6 +611,18 @@ func (siw *ServerInterfaceWrapper) InternalMultipartUpload(c fiber.Ctx) error { return siw.Handler.InternalMultipartUpload(c) } +// InternalStorageCleanupApply operation middleware +func (siw *ServerInterfaceWrapper) InternalStorageCleanupApply(c fiber.Ctx) error { + + return siw.Handler.InternalStorageCleanupApply(c) +} + +// InternalStorageCleanupAudit operation middleware +func (siw *ServerInterfaceWrapper) InternalStorageCleanupAudit(c fiber.Ctx) error { + + return siw.Handler.InternalStorageCleanupAudit(c) +} + // InternalUploadBlank operation middleware func (siw *ServerInterfaceWrapper) InternalUploadBlank(c fiber.Ctx) error { @@ -888,6 +995,10 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Post(options.BaseURL+"/data/multipart/upload", wrapper.InternalMultipartUpload) + router.Post(options.BaseURL+"/data/repair/storage-cleanup/apply", wrapper.InternalStorageCleanupApply) + + router.Post(options.BaseURL+"/data/repair/storage-cleanup/audit", wrapper.InternalStorageCleanupAudit) + router.Post(options.BaseURL+"/data/upload", wrapper.InternalUploadBlank) router.Post(options.BaseURL+"/data/upload/bulk", wrapper.InternalUploadBulk) @@ -1190,6 +1301,104 @@ func (response InternalMultipartUpload500Response) VisitInternalMultipartUploadR return nil } +type InternalStorageCleanupApplyRequestObject struct { + Body *InternalStorageCleanupApplyJSONRequestBody +} + +type InternalStorageCleanupApplyResponseObject interface { + VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error +} + +type InternalStorageCleanupApply200JSONResponse StorageCleanupApplyResult + +func (response InternalStorageCleanupApply200JSONResponse) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalStorageCleanupApply400Response struct { +} + +func (response InternalStorageCleanupApply400Response) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalStorageCleanupApply401Response struct { +} + +func (response InternalStorageCleanupApply401Response) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalStorageCleanupApply403Response struct { +} + +func (response InternalStorageCleanupApply403Response) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalStorageCleanupApply500Response struct { +} + +func (response InternalStorageCleanupApply500Response) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + +type InternalStorageCleanupAuditRequestObject struct { + Body *InternalStorageCleanupAuditJSONRequestBody +} + +type InternalStorageCleanupAuditResponseObject interface { + VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error +} + +type InternalStorageCleanupAudit200JSONResponse StorageCleanupReport + +func (response InternalStorageCleanupAudit200JSONResponse) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalStorageCleanupAudit400Response struct { +} + +func (response InternalStorageCleanupAudit400Response) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalStorageCleanupAudit401Response struct { +} + +func (response InternalStorageCleanupAudit401Response) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalStorageCleanupAudit403Response struct { +} + +func (response InternalStorageCleanupAudit403Response) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalStorageCleanupAudit500Response struct { +} + +func (response InternalStorageCleanupAudit500Response) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + type InternalUploadBlankRequestObject struct { Body *InternalUploadBlankJSONRequestBody } @@ -1914,6 +2123,12 @@ type StrictServerInterface interface { // (POST /data/multipart/upload) InternalMultipartUpload(ctx context.Context, request InternalMultipartUploadRequestObject) (InternalMultipartUploadResponseObject, error) + // (POST /data/repair/storage-cleanup/apply) + InternalStorageCleanupApply(ctx context.Context, request InternalStorageCleanupApplyRequestObject) (InternalStorageCleanupApplyResponseObject, error) + + // (POST /data/repair/storage-cleanup/audit) + InternalStorageCleanupAudit(ctx context.Context, request InternalStorageCleanupAuditRequestObject) (InternalStorageCleanupAuditResponseObject, error) + // (POST /data/upload) InternalUploadBlank(ctx context.Context, request InternalUploadBlankRequestObject) (InternalUploadBlankResponseObject, error) @@ -2122,6 +2337,68 @@ func (sh *strictHandler) InternalMultipartUpload(ctx fiber.Ctx) error { return nil } +// InternalStorageCleanupApply operation middleware +func (sh *strictHandler) InternalStorageCleanupApply(ctx fiber.Ctx) error { + var request InternalStorageCleanupApplyRequestObject + + var body InternalStorageCleanupApplyJSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalStorageCleanupApply(ctx.Context(), request.(InternalStorageCleanupApplyRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalStorageCleanupApply") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalStorageCleanupApplyResponseObject); ok { + if err := validResponse.VisitInternalStorageCleanupApplyResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + +// InternalStorageCleanupAudit operation middleware +func (sh *strictHandler) InternalStorageCleanupAudit(ctx fiber.Ctx) error { + var request InternalStorageCleanupAuditRequestObject + + var body InternalStorageCleanupAuditJSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalStorageCleanupAudit(ctx.Context(), request.(InternalStorageCleanupAuditRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalStorageCleanupAudit") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalStorageCleanupAuditResponseObject); ok { + if err := validResponse.VisitInternalStorageCleanupAuditResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + // InternalUploadBlank operation middleware func (sh *strictHandler) InternalUploadBlank(ctx fiber.Ctx) error { var request InternalUploadBlankRequestObject diff --git a/apigen/server/metricsapi/metrics.gen.go b/apigen/server/metricsapi/metrics.gen.go index ab20b076..87a4de98 100644 --- a/apigen/server/metricsapi/metrics.gen.go +++ b/apigen/server/metricsapi/metrics.gen.go @@ -26,6 +26,12 @@ const ( Unmatched ProviderTransferReconciliationStatus = "unmatched" ) +// Defines values for StoragePathChildType. +const ( + Directory StoragePathChildType = "directory" + File StoragePathChildType = "file" +) + // Defines values for TransferBreakdownResponseGroupBy. const ( TransferBreakdownResponseGroupByObject TransferBreakdownResponseGroupBy = "object" @@ -34,6 +40,20 @@ const ( TransferBreakdownResponseGroupByUser TransferBreakdownResponseGroupBy = "user" ) +// Defines values for ListStorageChildrenParamsSortBy. +const ( + Bytes ListStorageChildrenParamsSortBy = "bytes" + Name ListStorageChildrenParamsSortBy = "name" + Records ListStorageChildrenParamsSortBy = "records" + UpdatedTime ListStorageChildrenParamsSortBy = "updated_time" +) + +// Defines values for ListStorageChildrenParamsSortOrder. +const ( + Asc ListStorageChildrenParamsSortOrder = "asc" + Desc ListStorageChildrenParamsSortOrder = "desc" +) + // Defines values for GetTransferBreakdownParamsGroupBy. const ( GetTransferBreakdownParamsGroupByObject GetTransferBreakdownParamsGroupBy = "object" @@ -117,6 +137,45 @@ type ProviderTransferEventsRequest struct { // ProviderTransferReconciliationStatus defines model for ProviderTransferReconciliationStatus. type ProviderTransferReconciliationStatus string +// StoragePathChild defines model for StoragePathChild. +type StoragePathChild struct { + DownloadCount *int64 `json:"download_count,omitempty"` + FileCount *int64 `json:"file_count,omitempty"` + LastDownloadTime *time.Time `json:"last_download_time"` + LatestUpdateTime *time.Time `json:"latest_update_time"` + Name *string `json:"name,omitempty"` + Path *string `json:"path,omitempty"` + RecordCount *int64 `json:"record_count,omitempty"` + TotalBytes *int64 `json:"total_bytes,omitempty"` + Type *StoragePathChildType `json:"type,omitempty"` +} + +// StoragePathChildType defines model for StoragePathChild.Type. +type StoragePathChildType string + +// StoragePathChildrenResponse defines model for StoragePathChildrenResponse. +type StoragePathChildrenResponse struct { + Items *[]StoragePathChild `json:"items,omitempty"` + Organization *string `json:"organization,omitempty"` + Path *string `json:"path,omitempty"` + Project *string `json:"project,omitempty"` +} + +// StoragePathSummary defines model for StoragePathSummary. +type StoragePathSummary struct { + DirectChildCount *int64 `json:"direct_child_count,omitempty"` + DownloadCount *int64 `json:"download_count,omitempty"` + DuplicatePathCount *int64 `json:"duplicate_path_count,omitempty"` + FileCount *int64 `json:"file_count,omitempty"` + LastDownloadTime *time.Time `json:"last_download_time"` + LatestUpdateTime *time.Time `json:"latest_update_time"` + Organization *string `json:"organization,omitempty"` + Path *string `json:"path,omitempty"` + Project *string `json:"project,omitempty"` + RecordCount *int64 `json:"record_count,omitempty"` + TotalBytes *int64 `json:"total_bytes,omitempty"` +} + // TransferAttributionBreakdown defines model for TransferAttributionBreakdown. type TransferAttributionBreakdown struct { ActorEmail *string `json:"actor_email,omitempty"` @@ -185,6 +244,9 @@ type From = time.Time // Organization defines model for Organization. type Organization = string +// Path defines model for Path. +type Path = string + // Project defines model for Project. type Project = string @@ -197,6 +259,12 @@ type ReconciliationStatus = ProviderTransferReconciliationStatus // SHA256 defines model for SHA256. type SHA256 = string +// StorageOrganization defines model for StorageOrganization. +type StorageOrganization = string + +// StorageProject defines model for StorageProject. +type StorageProject = string + // To defines model for To. type To = time.Time @@ -234,6 +302,40 @@ type RecordProviderTransferEventsParams struct { Project *Project `form:"project,omitempty" json:"project,omitempty"` } +// ListStorageChildrenParams defines parameters for ListStorageChildren. +type ListStorageChildrenParams struct { + // Organization Organization scope for the storage subtree query. + Organization StorageOrganization `form:"organization" json:"organization"` + + // Project Project scope for the storage subtree query. + Project StorageProject `form:"project" json:"project"` + + // Path Optional storage subtree path. Omit for the project root. + Path *Path `form:"path,omitempty" json:"path,omitempty"` + Limit *int `form:"limit,omitempty" json:"limit,omitempty"` + Offset *int `form:"offset,omitempty" json:"offset,omitempty"` + SortBy *ListStorageChildrenParamsSortBy `form:"sort_by,omitempty" json:"sort_by,omitempty"` + SortOrder *ListStorageChildrenParamsSortOrder `form:"sort_order,omitempty" json:"sort_order,omitempty"` +} + +// ListStorageChildrenParamsSortBy defines parameters for ListStorageChildren. +type ListStorageChildrenParamsSortBy string + +// ListStorageChildrenParamsSortOrder defines parameters for ListStorageChildren. +type ListStorageChildrenParamsSortOrder string + +// GetStorageSummaryParams defines parameters for GetStorageSummary. +type GetStorageSummaryParams struct { + // Organization Organization scope for the storage subtree query. + Organization StorageOrganization `form:"organization" json:"organization"` + + // Project Project scope for the storage subtree query. + Project StorageProject `form:"project" json:"project"` + + // Path Optional storage subtree path. Omit for the project root. + Path *Path `form:"path,omitempty" json:"path,omitempty"` +} + // GetMetricsSummaryParams defines parameters for GetMetricsSummary. type GetMetricsSummaryParams struct { InactiveDays *int `form:"inactive_days,omitempty" json:"inactive_days,omitempty"` @@ -308,6 +410,12 @@ type ServerInterface interface { // (POST /index/v1/metrics/provider-transfer-events) RecordProviderTransferEvents(c fiber.Ctx, params RecordProviderTransferEventsParams) error + // (GET /index/v1/metrics/storage/children) + ListStorageChildren(c fiber.Ctx, params ListStorageChildrenParams) error + + // (GET /index/v1/metrics/storage/summary) + GetStorageSummary(c fiber.Ctx, params GetStorageSummaryParams) error + // (GET /index/v1/metrics/summary) GetMetricsSummary(c fiber.Ctx, params GetMetricsSummaryParams) error // Group transfer attribution events @@ -459,6 +567,142 @@ func (siw *ServerInterfaceWrapper) RecordProviderTransferEvents(c fiber.Ctx) err return siw.Handler.RecordProviderTransferEvents(c, params) } +// ListStorageChildren operation middleware +func (siw *ServerInterfaceWrapper) ListStorageChildren(c fiber.Ctx) error { + var err error + var params ListStorageChildrenParams + + // ------------- Required query parameter "organization" ------------- + if paramValue := c.Query("organization"); paramValue != "" { + + var value StorageOrganization + err = runtime.BindStyledParameterWithOptions("form", "organization", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter organization: %w", err).Error()) + } + params.Organization = value + + } else { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument organization is required, but not found").Error()) + } + // ------------- Required query parameter "project" ------------- + if paramValue := c.Query("project"); paramValue != "" { + + var value StorageProject + err = runtime.BindStyledParameterWithOptions("form", "project", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter project: %w", err).Error()) + } + params.Project = value + + } else { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument project is required, but not found").Error()) + } + // ------------- Optional query parameter "path" ------------- + if paramValue := c.Query("path"); paramValue != "" { + + var value Path + err = runtime.BindStyledParameterWithOptions("form", "path", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter path: %w", err).Error()) + } + params.Path = &value + + } + // ------------- Optional query parameter "limit" ------------- + if paramValue := c.Query("limit"); paramValue != "" { + + var value int + err = runtime.BindStyledParameterWithOptions("form", "limit", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter limit: %w", err).Error()) + } + params.Limit = &value + + } + // ------------- Optional query parameter "offset" ------------- + if paramValue := c.Query("offset"); paramValue != "" { + + var value int + err = runtime.BindStyledParameterWithOptions("form", "offset", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter offset: %w", err).Error()) + } + params.Offset = &value + + } + // ------------- Optional query parameter "sort_by" ------------- + if paramValue := c.Query("sort_by"); paramValue != "" { + + var value ListStorageChildrenParamsSortBy + err = runtime.BindStyledParameterWithOptions("form", "sort_by", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter sort_by: %w", err).Error()) + } + params.SortBy = &value + + } + // ------------- Optional query parameter "sort_order" ------------- + if paramValue := c.Query("sort_order"); paramValue != "" { + + var value ListStorageChildrenParamsSortOrder + err = runtime.BindStyledParameterWithOptions("form", "sort_order", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter sort_order: %w", err).Error()) + } + params.SortOrder = &value + + } + + return siw.Handler.ListStorageChildren(c, params) +} + +// GetStorageSummary operation middleware +func (siw *ServerInterfaceWrapper) GetStorageSummary(c fiber.Ctx) error { + var err error + var params GetStorageSummaryParams + + // ------------- Required query parameter "organization" ------------- + if paramValue := c.Query("organization"); paramValue != "" { + + var value StorageOrganization + err = runtime.BindStyledParameterWithOptions("form", "organization", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter organization: %w", err).Error()) + } + params.Organization = value + + } else { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument organization is required, but not found").Error()) + } + // ------------- Required query parameter "project" ------------- + if paramValue := c.Query("project"); paramValue != "" { + + var value StorageProject + err = runtime.BindStyledParameterWithOptions("form", "project", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter project: %w", err).Error()) + } + params.Project = value + + } else { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument project is required, but not found").Error()) + } + // ------------- Optional query parameter "path" ------------- + if paramValue := c.Query("path"); paramValue != "" { + + var value Path + err = runtime.BindStyledParameterWithOptions("form", "path", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter path: %w", err).Error()) + } + params.Path = &value + + } + + return siw.Handler.GetStorageSummary(c, params) +} + // GetMetricsSummary operation middleware func (siw *ServerInterfaceWrapper) GetMetricsSummary(c fiber.Ctx) error { var err error @@ -799,6 +1043,10 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Post(options.BaseURL+"/index/v1/metrics/provider-transfer-events", wrapper.RecordProviderTransferEvents) + router.Get(options.BaseURL+"/index/v1/metrics/storage/children", wrapper.ListStorageChildren) + + router.Get(options.BaseURL+"/index/v1/metrics/storage/summary", wrapper.GetStorageSummary) + router.Get(options.BaseURL+"/index/v1/metrics/summary", wrapper.GetMetricsSummary) router.Get(options.BaseURL+"/index/v1/metrics/transfers/breakdown", wrapper.GetTransferBreakdown) @@ -964,6 +1212,104 @@ func (response RecordProviderTransferEvents500Response) VisitRecordProviderTrans return nil } +type ListStorageChildrenRequestObject struct { + Params ListStorageChildrenParams +} + +type ListStorageChildrenResponseObject interface { + VisitListStorageChildrenResponse(ctx fiber.Ctx) error +} + +type ListStorageChildren200JSONResponse StoragePathChildrenResponse + +func (response ListStorageChildren200JSONResponse) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type ListStorageChildren400Response struct { +} + +func (response ListStorageChildren400Response) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type ListStorageChildren401Response struct { +} + +func (response ListStorageChildren401Response) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type ListStorageChildren403Response struct { +} + +func (response ListStorageChildren403Response) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type ListStorageChildren500Response struct { +} + +func (response ListStorageChildren500Response) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + +type GetStorageSummaryRequestObject struct { + Params GetStorageSummaryParams +} + +type GetStorageSummaryResponseObject interface { + VisitGetStorageSummaryResponse(ctx fiber.Ctx) error +} + +type GetStorageSummary200JSONResponse StoragePathSummary + +func (response GetStorageSummary200JSONResponse) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type GetStorageSummary400Response struct { +} + +func (response GetStorageSummary400Response) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type GetStorageSummary401Response struct { +} + +func (response GetStorageSummary401Response) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type GetStorageSummary403Response struct { +} + +func (response GetStorageSummary403Response) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type GetStorageSummary500Response struct { +} + +func (response GetStorageSummary500Response) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + type GetMetricsSummaryRequestObject struct { Params GetMetricsSummaryParams } @@ -1123,6 +1469,12 @@ type StrictServerInterface interface { // (POST /index/v1/metrics/provider-transfer-events) RecordProviderTransferEvents(ctx context.Context, request RecordProviderTransferEventsRequestObject) (RecordProviderTransferEventsResponseObject, error) + // (GET /index/v1/metrics/storage/children) + ListStorageChildren(ctx context.Context, request ListStorageChildrenRequestObject) (ListStorageChildrenResponseObject, error) + + // (GET /index/v1/metrics/storage/summary) + GetStorageSummary(ctx context.Context, request GetStorageSummaryRequestObject) (GetStorageSummaryResponseObject, error) + // (GET /index/v1/metrics/summary) GetMetricsSummary(ctx context.Context, request GetMetricsSummaryRequestObject) (GetMetricsSummaryResponseObject, error) // Group transfer attribution events @@ -1234,6 +1586,60 @@ func (sh *strictHandler) RecordProviderTransferEvents(ctx fiber.Ctx, params Reco return nil } +// ListStorageChildren operation middleware +func (sh *strictHandler) ListStorageChildren(ctx fiber.Ctx, params ListStorageChildrenParams) error { + var request ListStorageChildrenRequestObject + + request.Params = params + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.ListStorageChildren(ctx.Context(), request.(ListStorageChildrenRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "ListStorageChildren") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(ListStorageChildrenResponseObject); ok { + if err := validResponse.VisitListStorageChildrenResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + +// GetStorageSummary operation middleware +func (sh *strictHandler) GetStorageSummary(ctx fiber.Ctx, params GetStorageSummaryParams) error { + var request GetStorageSummaryRequestObject + + request.Params = params + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.GetStorageSummary(ctx.Context(), request.(GetStorageSummaryRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetStorageSummary") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(GetStorageSummaryResponseObject); ok { + if err := validResponse.VisitGetStorageSummaryResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + // GetMetricsSummary operation middleware func (sh *strictHandler) GetMetricsSummary(ctx fiber.Ctx, params GetMetricsSummaryParams) error { var request GetMetricsSummaryRequestObject diff --git a/client/client.go b/client/client.go index 4b9a6516..9ef1ccd2 100644 --- a/client/client.go +++ b/client/client.go @@ -47,6 +47,7 @@ type Client struct { drs *syfonclient.DRSService buckets *syfonclient.BucketsService metrics *syfonclient.MetricsService + repair *syfonclient.RepairService lfs *syfonclient.LFSService // Generated schema-specific clients @@ -208,6 +209,7 @@ func (c *Client) initServices() error { c.data = syfonclient.NewDataService(c.internalGen, c.requestor, l, c.drs) c.buckets = syfonclient.NewBucketsService(c.bucketGen) c.metrics = syfonclient.NewMetricsService(c.metricsGen) + c.repair = syfonclient.NewRepairService(c.requestor) return nil } @@ -226,6 +228,7 @@ func (c *Client) Index() *syfonclient.IndexService { return c.index } func (c *Client) DRS() *syfonclient.DRSService { return c.drs } func (c *Client) Buckets() *syfonclient.BucketsService { return c.buckets } func (c *Client) Metrics() *syfonclient.MetricsService { return c.metrics } +func (c *Client) Repair() *syfonclient.RepairService { return c.repair } func (c *Client) LFS() *syfonclient.LFSService { return c.lfs } // Schema-specific generated clients diff --git a/client/services/interface.go b/client/services/interface.go index 1c92105b..ffa5049d 100644 --- a/client/services/interface.go +++ b/client/services/interface.go @@ -8,5 +8,6 @@ type SyfonClient interface { DRS() *DRSService Buckets() *BucketsService Metrics() *MetricsService + Repair() *RepairService LFS() *LFSService } diff --git a/client/services/metrics_service.go b/client/services/metrics_service.go index e72b1001..6f0d0ad9 100644 --- a/client/services/metrics_service.go +++ b/client/services/metrics_service.go @@ -18,7 +18,10 @@ func NewMetricsService(gen metricsapi.ClientWithResponsesInterface) *MetricsServ } func (s *MetricsService) Summary(ctx context.Context, opts MetricsSummaryOptions) (metricsapi.FileUsageSummary, error) { - params := &metricsapi.GetMetricsSummaryParams{} + params := &metricsapi.GetMetricsSummaryParams{ + Organization: stringPtr[metricsapi.Organization](opts.Organization), + Project: stringPtr[metricsapi.Project](opts.ProjectID), + } if opts.InactiveDays > 0 { params.InactiveDays = &opts.InactiveDays } @@ -33,7 +36,10 @@ func (s *MetricsService) Summary(ctx context.Context, opts MetricsSummaryOptions } func (s *MetricsService) Files(ctx context.Context, opts MetricsFilesOptions) ([]metricsapi.FileUsage, error) { - params := &metricsapi.ListMetricsFilesParams{} + params := &metricsapi.ListMetricsFilesParams{ + Organization: stringPtr[metricsapi.Organization](opts.Organization), + Project: stringPtr[metricsapi.Project](opts.ProjectID), + } if opts.Limit > 0 { params.Limit = &opts.Limit } @@ -68,6 +74,46 @@ func (s *MetricsService) File(ctx context.Context, objectID string) (metricsapi. return *resp.JSON200, nil } +func (s *MetricsService) StorageSummary(ctx context.Context, opts StorageSummaryOptions) (metricsapi.StoragePathSummary, error) { + params := &metricsapi.GetStorageSummaryParams{ + Organization: metricsapi.StorageOrganization(opts.Organization), + Project: metricsapi.StorageProject(opts.ProjectID), + Path: stringPtr[metricsapi.Path](opts.Path), + } + resp, err := s.gen.GetStorageSummaryWithResponse(ctx, params) + if err != nil { + return metricsapi.StoragePathSummary{}, err + } + if resp.JSON200 == nil { + return metricsapi.StoragePathSummary{}, fmt.Errorf("failed to get storage metrics summary: %d", resp.StatusCode()) + } + return *resp.JSON200, nil +} + +func (s *MetricsService) StorageChildren(ctx context.Context, opts StorageChildrenOptions) (metricsapi.StoragePathChildrenResponse, error) { + params := &metricsapi.ListStorageChildrenParams{ + Organization: metricsapi.StorageOrganization(opts.Organization), + Project: metricsapi.StorageProject(opts.ProjectID), + Path: stringPtr[metricsapi.Path](opts.Path), + SortBy: stringPtr[metricsapi.ListStorageChildrenParamsSortBy](opts.SortBy), + SortOrder: stringPtr[metricsapi.ListStorageChildrenParamsSortOrder](opts.SortOrder), + } + if opts.Limit > 0 { + params.Limit = &opts.Limit + } + if opts.Offset > 0 { + params.Offset = &opts.Offset + } + resp, err := s.gen.ListStorageChildrenWithResponse(ctx, params) + if err != nil { + return metricsapi.StoragePathChildrenResponse{}, err + } + if resp.JSON200 == nil { + return metricsapi.StoragePathChildrenResponse{}, fmt.Errorf("failed to list storage metrics children: %d", resp.StatusCode()) + } + return *resp.JSON200, nil +} + func (s *MetricsService) TransferSummary(ctx context.Context, opts TransferMetricsOptions) (models.TransferAttributionSummary, error) { params, err := transferSummaryParams(opts) if err != nil { diff --git a/client/services/repair_service.go b/client/services/repair_service.go new file mode 100644 index 00000000..eea26218 --- /dev/null +++ b/client/services/repair_service.go @@ -0,0 +1,58 @@ +package services + +import ( + "context" + "net/http" + + "github.com/calypr/syfon/client/request" + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/repair" +) + +type RepairService struct { + requestor request.Requester +} + +func NewRepairService(r request.Requester) *RepairService { + return &RepairService{requestor: r} +} + +func (s *RepairService) StorageCleanupAudit(ctx context.Context, opts StorageCleanupAuditOptions) (repair.StorageCleanupReport, error) { + req := repair.StorageCleanupAuditRequest{ + Organization: opts.Organization, + Project: opts.ProjectID, + PathPrefix: opts.PathPrefix, + ExpectedPaths: append([]string(nil), opts.ExpectedPaths...), + CheckStorage: opts.CheckStorage, + } + var out repair.StorageCleanupReport + if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairCleanupAudit, req, &out); err != nil { + return repair.StorageCleanupReport{}, err + } + return out, nil +} + +func (s *RepairService) StorageCleanupApply(ctx context.Context, opts StorageCleanupApplyOptions) (repair.StorageCleanupApplyResult, error) { + kinds := make([]repair.FindingKind, 0, len(opts.SelectedFindingKinds)) + for _, kind := range opts.SelectedFindingKinds { + kinds = append(kinds, repair.FindingKind(kind)) + } + req := repair.StorageCleanupApplyRequest{ + Organization: opts.Organization, + Project: opts.ProjectID, + PathPrefix: opts.PathPrefix, + ExpectedPaths: append([]string(nil), opts.ExpectedPaths...), + DeleteStaleDuplicates: opts.DeleteStaleDuplicates, + DeleteRepoOrphans: opts.DeleteRepoOrphans, + DryRun: opts.DryRun, + SelectedPaths: append([]string(nil), opts.SelectedPaths...), + SelectedObjectIDs: append([]string(nil), opts.SelectedObjectIDs...), + SelectedFindingKinds: kinds, + CheckStorage: opts.CheckStorage, + } + var out repair.StorageCleanupApplyResult + if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairCleanupApply, req, &out); err != nil { + return repair.StorageCleanupApplyResult{}, err + } + return out, nil +} diff --git a/client/services/services_test.go b/client/services/services_test.go index 24c5492c..b07db86f 100644 --- a/client/services/services_test.go +++ b/client/services/services_test.go @@ -130,15 +130,21 @@ func (f *fakeBucketClient) DeleteProjectDataWithResponse(ctx context.Context, or } type fakeMetricsClient struct { - summaryResp *metricsapi.GetMetricsSummaryResponse - summaryErr error - summaryParams *metricsapi.GetMetricsSummaryParams - filesResp *metricsapi.ListMetricsFilesResponse - filesErr error - filesParams *metricsapi.ListMetricsFilesParams - fileResp *metricsapi.GetMetricsFileResponse - fileErr error - fileObjectID string + summaryResp *metricsapi.GetMetricsSummaryResponse + summaryErr error + summaryParams *metricsapi.GetMetricsSummaryParams + storageSummaryResp *metricsapi.GetStorageSummaryResponse + storageSummaryErr error + storageSummaryParams *metricsapi.GetStorageSummaryParams + filesResp *metricsapi.ListMetricsFilesResponse + filesErr error + filesParams *metricsapi.ListMetricsFilesParams + storageChildrenResp *metricsapi.ListStorageChildrenResponse + storageChildrenErr error + storageChildrenParams *metricsapi.ListStorageChildrenParams + fileResp *metricsapi.GetMetricsFileResponse + fileErr error + fileObjectID string } func (f *fakeMetricsClient) ListMetricsFilesWithResponse(ctx context.Context, params *metricsapi.ListMetricsFilesParams, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.ListMetricsFilesResponse, error) { @@ -156,6 +162,16 @@ func (f *fakeMetricsClient) GetMetricsSummaryWithResponse(ctx context.Context, p return f.summaryResp, f.summaryErr } +func (f *fakeMetricsClient) GetStorageSummaryWithResponse(ctx context.Context, params *metricsapi.GetStorageSummaryParams, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.GetStorageSummaryResponse, error) { + f.storageSummaryParams = params + return f.storageSummaryResp, f.storageSummaryErr +} + +func (f *fakeMetricsClient) ListStorageChildrenWithResponse(ctx context.Context, params *metricsapi.ListStorageChildrenParams, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.ListStorageChildrenResponse, error) { + f.storageChildrenParams = params + return f.storageChildrenResp, f.storageChildrenErr +} + func (f *fakeMetricsClient) RecordProviderTransferEventsWithBodyWithResponse(ctx context.Context, params *metricsapi.RecordProviderTransferEventsParams, contentType string, body io.Reader, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.RecordProviderTransferEventsResponse, error) { return &metricsapi.RecordProviderTransferEventsResponse{HTTPResponse: &http.Response{StatusCode: http.StatusNotImplemented}}, nil } diff --git a/client/services/types.go b/client/services/types.go index a920b794..09c06cb6 100644 --- a/client/services/types.go +++ b/client/services/types.go @@ -50,6 +50,22 @@ type MetricsSummaryOptions struct { ProjectID string } +type StorageSummaryOptions struct { + Organization string + ProjectID string + Path string +} + +type StorageChildrenOptions struct { + Organization string + ProjectID string + Path string + Limit int + Offset int + SortBy string + SortOrder string +} + type TransferMetricsOptions struct { Organization string ProjectID string @@ -64,3 +80,25 @@ type TransferMetricsOptions struct { ReconciliationStatus string AllowStale bool } + +type StorageCleanupAuditOptions struct { + Organization string + ProjectID string + PathPrefix string + ExpectedPaths []string + CheckStorage *bool +} + +type StorageCleanupApplyOptions struct { + Organization string + ProjectID string + PathPrefix string + ExpectedPaths []string + DeleteStaleDuplicates bool + DeleteRepoOrphans bool + DryRun bool + SelectedPaths []string + SelectedObjectIDs []string + SelectedFindingKinds []string + CheckStorage *bool +} diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index 9fd3bb65..ac1d9f27 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -41,6 +41,8 @@ func RegisterInternalRoutes(router fiber.Router, om *core.ObjectManager) { router.Post(common.RouteInternalBulkCreate, handleInternalBulkCreateFiber(om)) router.Post(common.RouteInternalBulkDocs, handleInternalBulkDocumentsFiber(om)) router.Post(common.RouteInternalBulkDeleteHashes, handleInternalBulkDeleteFiber(om)) + router.Post(common.RouteInternalRepairCleanupAudit, handleInternalStorageCleanupAuditFiber(om)) + router.Post(common.RouteInternalRepairCleanupApply, handleInternalStorageCleanupApplyFiber(om)) registerInternalTransferRoutes(router, om) } @@ -321,7 +323,6 @@ func handleInternalBulkDocumentsFiber(om *core.ObjectManager) fiber.Handler { } if obj, err := req.AsBulkDocumentsRequest1(); err == nil { ids = append(ids, common.DerefStringSlice(obj.Ids)...) - ids = append(ids, common.DerefStringSlice(obj.Dids)...) } if len(ids) == 0 { return c.Status(fiber.StatusBadRequest).SendString("Invalid request body: ids are required") diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go new file mode 100644 index 00000000..3e73741d --- /dev/null +++ b/internal/api/internaldrs/repair_cleanup.go @@ -0,0 +1,83 @@ +package internaldrs + +import ( + "context" + "strings" + + sycommon "github.com/calypr/syfon/common" + "github.com/calypr/syfon/internal/api/apiutil" + apimiddleware "github.com/calypr/syfon/internal/api/middleware" + "github.com/calypr/syfon/internal/authz" + intcommon "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/repair" + "github.com/gofiber/fiber/v3" +) + +func handleInternalStorageCleanupAuditFiber(om *core.ObjectManager) fiber.Handler { + svc := repair.NewStorageCleanupService(om) + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req repair.StorageCleanupAuditRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + req.Organization = strings.TrimSpace(req.Organization) + req.Project = strings.TrimSpace(req.Project) + req.PathPrefix = strings.TrimSpace(req.PathPrefix) + if req.Organization == "" || req.Project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read"); err != nil { + return apiutil.HandleError(c, err) + } + report, err := svc.Audit(c.Context(), req) + if err != nil { + return apiutil.HandleError(c, err) + } + return c.JSON(report) + } +} + +func handleInternalStorageCleanupApplyFiber(om *core.ObjectManager) fiber.Handler { + svc := repair.NewStorageCleanupService(om) + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req repair.StorageCleanupApplyRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + req.Organization = strings.TrimSpace(req.Organization) + req.Project = strings.TrimSpace(req.Project) + req.PathPrefix = strings.TrimSpace(req.PathPrefix) + if req.Organization == "" || req.Project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read", "delete"); err != nil { + return apiutil.HandleError(c, err) + } + result, err := svc.Apply(c.Context(), req) + if err != nil { + return apiutil.HandleError(c, err) + } + return c.JSON(result) + } +} + +func authorizeStorageCleanupScope(ctx context.Context, organization, project string, methods ...string) error { + if !authz.IsAuthzEnforced(ctx) { + return nil + } + resource, err := sycommon.ResourcePath(organization, project) + if err != nil { + return err + } + if authz.HasMethodAccess(ctx, methods[0], []string{"/programs", "/data_file"}) || authz.HasAnyMethodAccess(ctx, []string{resource}, methods...) { + return nil + } + return intcommon.ErrUnauthorized +} diff --git a/internal/api/metrics/storage_summary.go b/internal/api/metrics/storage_summary.go new file mode 100644 index 00000000..f4075990 --- /dev/null +++ b/internal/api/metrics/storage_summary.go @@ -0,0 +1,140 @@ +package metrics + +import ( + "context" + "net/http" + "strings" + + "github.com/calypr/syfon/apigen/server/metricsapi" + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/models" + "github.com/calypr/syfon/internal/storagemetrics" +) + +func (s *MetricsServer) GetStorageSummary(ctx context.Context, request metricsapi.GetStorageSummaryRequestObject) (metricsapi.GetStorageSummaryResponseObject, error) { + access, statusCode, ok := s.checkAuth(ctx) + if !ok { + return getStorageSummaryAuthResponse(statusCode), nil + } + if !access.isScoped() || strings.TrimSpace(access.project) == "" { + return metricsapi.GetStorageSummary400Response{}, nil + } + normalizedPath, _, err := common.NormalizeBrowsePath(pointerString(request.Params.Path)) + if err != nil { + return metricsapi.GetStorageSummary400Response{}, nil + } + summary, err := s.database.GetStoragePathSummary(ctx, access.organization, access.project, normalizedPath) + if err != nil { + return metricsapi.GetStorageSummary500Response{}, nil + } + return metricsapi.GetStorageSummary200JSONResponse(storageSummaryToGenerated(summary)), nil +} + +func (s *MetricsServer) ListStorageChildren(ctx context.Context, request metricsapi.ListStorageChildrenRequestObject) (metricsapi.ListStorageChildrenResponseObject, error) { + limit := 200 + if request.Params.Limit != nil { + limit = *request.Params.Limit + } + offset := 0 + if request.Params.Offset != nil { + offset = *request.Params.Offset + } + if limit < 1 || limit > 1000 || offset < 0 { + return metricsapi.ListStorageChildren400Response{}, nil + } + access, statusCode, ok := s.checkAuth(ctx) + if !ok { + return getStorageChildrenAuthResponse(statusCode), nil + } + if !access.isScoped() || strings.TrimSpace(access.project) == "" { + return metricsapi.ListStorageChildren400Response{}, nil + } + normalizedPath, _, err := common.NormalizeBrowsePath(pointerString(request.Params.Path)) + if err != nil { + return metricsapi.ListStorageChildren400Response{}, nil + } + sortBy, sortOrder, err := storagemetrics.NormalizeStorageChildrenSort(pointerString(request.Params.SortBy), pointerString(request.Params.SortOrder)) + if err != nil { + return metricsapi.ListStorageChildren400Response{}, nil + } + items, err := s.database.ListStoragePathChildren(ctx, access.organization, access.project, normalizedPath, limit, offset, sortBy, sortOrder) + if err != nil { + return metricsapi.ListStorageChildren500Response{}, nil + } + return metricsapi.ListStorageChildren200JSONResponse(storageChildrenToGenerated(access.organization, access.project, normalizedPath, items)), nil +} + +func getStorageSummaryAuthResponse(statusCode int) metricsapi.GetStorageSummaryResponseObject { + switch statusCode { + case http.StatusUnauthorized: + return metricsapi.GetStorageSummary401Response{} + case http.StatusForbidden: + return metricsapi.GetStorageSummary403Response{} + default: + return metricsapi.GetStorageSummary400Response{} + } +} + +func getStorageChildrenAuthResponse(statusCode int) metricsapi.ListStorageChildrenResponseObject { + switch statusCode { + case http.StatusUnauthorized: + return metricsapi.ListStorageChildren401Response{} + case http.StatusForbidden: + return metricsapi.ListStorageChildren403Response{} + default: + return metricsapi.ListStorageChildren400Response{} + } +} + +func storageSummaryToGenerated(v models.StoragePathSummary) metricsapi.StoragePathSummary { + return metricsapi.StoragePathSummary{ + Organization: &v.Organization, + Project: &v.Project, + Path: &v.Path, + FileCount: &v.FileCount, + RecordCount: &v.RecordCount, + TotalBytes: &v.TotalBytes, + DownloadCount: &v.DownloadCount, + DirectChildCount: &v.DirectChildCount, + DuplicatePathCount: &v.DuplicatePathCount, + LatestUpdateTime: v.LatestUpdateTime, + LastDownloadTime: v.LastDownloadTime, + } +} + +func storageChildrenToGenerated(organization, project, path string, items []models.StoragePathChild) metricsapi.StoragePathChildrenResponse { + out := metricsapi.StoragePathChildrenResponse{ + Organization: &organization, + Project: &project, + Path: &path, + } + if len(items) == 0 { + empty := make([]metricsapi.StoragePathChild, 0) + out.Items = &empty + return out + } + generated := make([]metricsapi.StoragePathChild, 0, len(items)) + for _, item := range items { + childType := metricsapi.StoragePathChildType(item.Type) + generated = append(generated, metricsapi.StoragePathChild{ + Name: &item.Name, + Path: &item.Path, + Type: &childType, + FileCount: &item.FileCount, + RecordCount: &item.RecordCount, + TotalBytes: &item.TotalBytes, + LatestUpdateTime: item.LatestUpdateTime, + DownloadCount: &item.DownloadCount, + LastDownloadTime: item.LastDownloadTime, + }) + } + out.Items = &generated + return out +} + +func pointerString[T ~string](v *T) string { + if v == nil { + return "" + } + return string(*v) +} diff --git a/internal/api/metrics/storage_summary_test.go b/internal/api/metrics/storage_summary_test.go new file mode 100644 index 00000000..a7b95861 --- /dev/null +++ b/internal/api/metrics/storage_summary_test.go @@ -0,0 +1,131 @@ +package metrics + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/calypr/syfon/apigen/server/drs" + "github.com/calypr/syfon/apigen/server/metricsapi" + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/models" + "github.com/calypr/syfon/internal/testutils" + "github.com/gofiber/fiber/v3" +) + +func TestStorageMetricsRoutes_SummaryAndChildren(t *testing.T) { + now := time.Now().UTC() + db := &testutils.MockDatabase{ + Objects: map[string]*drs.DrsObject{ + "obj-1": {Id: "obj-1", Name: common.Ptr("data/a.txt"), Size: 10, UpdatedTime: &now}, + "obj-2": {Id: "obj-2", Name: common.Ptr("data/a.txt"), Size: 20, UpdatedTime: &now}, + "obj-3": {Id: "obj-3", Name: common.Ptr("data/nested/b.txt"), Size: 30, UpdatedTime: &now}, + }, + ObjectAuthz: map[string]map[string][]string{ + "obj-1": {"cbds": {"end_to_end_test"}}, + "obj-2": {"cbds": {"end_to_end_test"}}, + "obj-3": {"cbds": {"end_to_end_test"}}, + }, + Usage: map[string]models.FileUsage{ + "obj-1": {ObjectID: "obj-1", DownloadCount: 2, LastDownloadTime: timePtr(now.Add(-3 * time.Hour))}, + "obj-2": {ObjectID: "obj-2", DownloadCount: 4, LastDownloadTime: timePtr(now.Add(-time.Hour))}, + "obj-3": {ObjectID: "obj-3", DownloadCount: 1, LastDownloadTime: timePtr(now.Add(-2 * time.Hour))}, + }, + } + app := fiber.New() + app.Use(func(c fiber.Ctx) error { + if mode := c.Get("X-Test-Auth-Mode"); mode != "" { + var privs map[string]map[string]bool + if privsJSON := c.Get("X-Test-Privileges"); privsJSON != "" { + _ = json.Unmarshal([]byte(privsJSON), &privs) + } + ctx := metricsTestContext(c.Context(), mode, c.Get("X-Test-Auth-Header") == "true", c.Get("X-Test-Auth-Header") == "true", privs) + c.SetContext(ctx) + } + return c.Next() + }) + RegisterMetricsRoutes(app, db) + + privs, _ := json.Marshal(map[string]map[string]bool{ + "/programs/cbds/projects/end_to_end_test": {"read": true}, + }) + + t.Run("summary", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/index/v1/metrics/storage/summary?organization=cbds&project=end_to_end_test&path=data", nil) + req.Header.Set("X-Test-Auth-Mode", "gen3") + req.Header.Set("X-Test-Auth-Header", "true") + req.Header.Set("X-Test-Privileges", string(privs)) + httpResp, err := app.Test(req) + if err != nil { + t.Fatalf("test request failed: %v", err) + } + body, _ := io.ReadAll(httpResp.Body) + if httpResp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", httpResp.StatusCode, string(body)) + } + var resp metricsapi.StoragePathSummary + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.RecordCount == nil || *resp.RecordCount != 3 || resp.FileCount == nil || *resp.FileCount != 2 || resp.DuplicatePathCount == nil || *resp.DuplicatePathCount != 1 { + t.Fatalf("unexpected storage summary: %+v", resp) + } + if resp.DownloadCount == nil || *resp.DownloadCount != 7 { + t.Fatalf("unexpected download count: %+v", resp) + } + if resp.LastDownloadTime == nil || !resp.LastDownloadTime.Equal(now.Add(-time.Hour)) { + t.Fatalf("unexpected last download time: %+v", resp.LastDownloadTime) + } + }) + + t.Run("children", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/index/v1/metrics/storage/children?organization=cbds&project=end_to_end_test&path=data&sort_by=bytes&sort_order=desc", nil) + req.Header.Set("X-Test-Auth-Mode", "gen3") + req.Header.Set("X-Test-Auth-Header", "true") + req.Header.Set("X-Test-Privileges", string(privs)) + httpResp, err := app.Test(req) + if err != nil { + t.Fatalf("test request failed: %v", err) + } + body, _ := io.ReadAll(httpResp.Body) + if httpResp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", httpResp.StatusCode, string(body)) + } + var resp metricsapi.StoragePathChildrenResponse + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.Items == nil || len(*resp.Items) != 2 { + t.Fatalf("unexpected children response: %+v", resp) + } + first := (*resp.Items)[0] + if first.Name == nil || *first.Name != "nested" || first.Type == nil || *first.Type != metricsapi.Directory { + t.Fatalf("unexpected first child: %+v", first) + } + if first.DownloadCount == nil || *first.DownloadCount != 1 { + t.Fatalf("unexpected first child download count: %+v", first) + } + }) +} + +func TestStorageMetricsRoutes_Validation(t *testing.T) { + app := fiber.New() + RegisterMetricsRoutes(app, &testutils.MockDatabase{}) + + req := httptest.NewRequest(http.MethodGet, "/index/v1/metrics/storage/children?organization=cbds&project=end_to_end_test&path=../bad", nil) + httpResp, err := app.Test(req) + if err != nil { + t.Fatalf("test request failed: %v", err) + } + if httpResp.StatusCode != http.StatusBadRequest { + body, _ := io.ReadAll(httpResp.Body) + t.Fatalf("expected 400, got %d body=%s", httpResp.StatusCode, string(body)) + } +} + +func timePtr(t time.Time) *time.Time { + return &t +} diff --git a/internal/common/routes.go b/internal/common/routes.go index 569fc2b9..b4539251 100644 --- a/internal/common/routes.go +++ b/internal/common/routes.go @@ -1,19 +1,21 @@ package common const ( - RouteInternalDownload = "/data/download/{file_id}" - RouteInternalDownloadPart = "/data/download/{file_id}/part" - RouteInternalUpload = "/data/upload" - RouteInternalUploadURL = "/data/upload/{file_id}" - RouteInternalUploadBulk = "/data/upload/bulk" - RouteInternalMultipartInit = "/data/multipart/init" - RouteInternalMultipartUpload = "/data/multipart/upload" - RouteInternalMultipartComplete = "/data/multipart/complete" - RouteInternalInspectObject = "/data/inspect" - RouteInternalBuckets = "/data/buckets" - RouteInternalBucketDetail = "/data/buckets/{bucket}" - RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" - RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" + RouteInternalDownload = "/data/download/{file_id}" + RouteInternalDownloadPart = "/data/download/{file_id}/part" + RouteInternalUpload = "/data/upload" + RouteInternalUploadURL = "/data/upload/{file_id}" + RouteInternalUploadBulk = "/data/upload/bulk" + RouteInternalMultipartInit = "/data/multipart/init" + RouteInternalMultipartUpload = "/data/multipart/upload" + RouteInternalMultipartComplete = "/data/multipart/complete" + RouteInternalInspectObject = "/data/inspect" + RouteInternalBuckets = "/data/buckets" + RouteInternalBucketDetail = "/data/buckets/{bucket}" + RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" + RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" + RouteInternalRepairCleanupAudit = "/data/repair/storage-cleanup/audit" + RouteInternalRepairCleanupApply = "/data/repair/storage-cleanup/apply" RouteInternalIndex = "/index" RouteInternalIndexDetail = "/index/{id}" diff --git a/internal/core/storage_cleanup.go b/internal/core/storage_cleanup.go new file mode 100644 index 00000000..47abce89 --- /dev/null +++ b/internal/core/storage_cleanup.go @@ -0,0 +1,38 @@ +package core + +import ( + "context" + "fmt" + "strings" + + sycommon "github.com/calypr/syfon/common" + "github.com/calypr/syfon/internal/authz" + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/db" + "github.com/calypr/syfon/internal/models" +) + +func (m *ObjectManager) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + store, ok := m.db.(db.StorageMetricsStore) + if !ok { + return nil, fmt.Errorf("storage cleanup listing not supported by database") + } + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" || project == "" { + return nil, fmt.Errorf("organization and project are required") + } + if authz.IsAuthzEnforced(ctx) { + if authz.IsGen3Mode(ctx) && !authz.HasAuthHeader(ctx) { + return nil, common.ErrUnauthorized + } + resource, err := sycommon.ResourcePath(organization, project) + if err != nil { + return nil, err + } + if !authz.HasMethodAccess(ctx, objectMethodRead, []string{"/programs", "/data_file"}) && !authz.HasAnyMethodAccess(ctx, []string{resource}, objectMethodRead) { + return nil, common.ErrUnauthorized + } + } + return store.ListStorageCleanupRecords(ctx, organization, project, pathPrefix) +} diff --git a/internal/db/interface.go b/internal/db/interface.go index 30791773..fe70dad7 100644 --- a/internal/db/interface.go +++ b/internal/db/interface.go @@ -65,6 +65,12 @@ type FileUsageScopedLister interface { GetFileUsageSummaryByResources(ctx context.Context, resources []string, includeUnscoped bool, inactiveSince *time.Time) (models.FileUsageSummary, error) } +type StorageMetricsStore interface { + GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) + ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) + ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) +} + type TransferAttributionScopedStore interface { GetTransferAttributionSummaryByResources(ctx context.Context, filter models.TransferAttributionFilter, resources []string) (models.TransferAttributionSummary, error) GetTransferAttributionBreakdownByResources(ctx context.Context, filter models.TransferAttributionFilter, groupBy string, resources []string) ([]models.TransferAttributionBreakdown, error) @@ -134,6 +140,9 @@ type MetricsStore interface { ListFileUsageByObjectIDs(ctx context.Context, ids []string) ([]models.FileUsage, error) ListFileUsage(ctx context.Context, limit, offset int, inactiveSince *time.Time) ([]models.FileUsage, error) GetFileUsageSummary(ctx context.Context, inactiveSince *time.Time) (models.FileUsageSummary, error) + GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) + ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) + ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) } // LFSStore is the minimum storage surface needed by the LFS API. @@ -151,4 +160,5 @@ type DatabaseInterface interface { CredentialStore PendingLFSMetaStore UsageStore + StorageMetricsStore } diff --git a/internal/db/postgres/metrics.go b/internal/db/postgres/metrics.go index 8e7138b0..5e7c66e1 100644 --- a/internal/db/postgres/metrics.go +++ b/internal/db/postgres/metrics.go @@ -12,6 +12,7 @@ import ( sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" + "github.com/calypr/syfon/internal/storagemetrics" "github.com/lib/pq" ) @@ -274,6 +275,81 @@ func (db *PostgresDB) GetFileUsage(ctx context.Context, objectID string) (*model return &usage, nil } +func (db *PostgresDB) GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) { + if err := db.flushObjectUsageEvents(ctx); err != nil { + return models.StoragePathSummary{}, err + } + rows, err := db.listStorageMetricRowsByScope(ctx, organization, project) + if err != nil { + return models.StoragePathSummary{}, err + } + return storagemetrics.AggregateStoragePathSummary(strings.TrimSpace(organization), strings.TrimSpace(project), path, rows) +} + +func (db *PostgresDB) ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) { + if err := db.flushObjectUsageEvents(ctx); err != nil { + return nil, err + } + rows, err := db.listStorageMetricRowsByScope(ctx, organization, project) + if err != nil { + return nil, err + } + return storagemetrics.AggregateStoragePathChildren(path, rows, limit, offset, sortBy, sortOrder) +} + +func (db *PostgresDB) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + if err := db.flushObjectUsageEvents(ctx); err != nil { + return nil, err + } + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return nil, err + } + _, prefixSegments, err := common.NormalizeBrowsePath(pathPrefix) + if err != nil { + return nil, err + } + rows, err := db.db.QueryContext(ctx, ` + SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, + COALESCE(u.download_count, 0), u.last_download_time + FROM drs_object_browse_index bi + JOIN drs_object o ON o.id = bi.object_id + LEFT JOIN object_usage u ON u.object_id = bi.object_id + WHERE bi.resource = $1 + ORDER BY bi.normalized_path ASC, bi.object_id ASC + `, resource) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.StorageCleanupRecord, 0) + for rows.Next() { + var row models.StorageCleanupRecord + var lastDownload sql.NullTime + if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { + return nil, err + } + info, ok, err := common.BrowsePathInfoFromName(row.NormalizedPath) + if err != nil { + return nil, err + } + if !ok || !common.HasBrowsePathPrefix(info.Segments, prefixSegments) { + continue + } + row.NormalizedPath = info.Normalized + if lastDownload.Valid { + t := lastDownload.Time.UTC() + row.LastDownloadTime = &t + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func (db *PostgresDB) ListFileUsageByObjectIDs(ctx context.Context, ids []string) ([]models.FileUsage, error) { if len(ids) == 0 { return []models.FileUsage{}, nil @@ -397,6 +473,44 @@ func scanFileUsageRows(rows *sql.Rows, capacity int) ([]models.FileUsage, error) return out, nil } +func (db *PostgresDB) listStorageMetricRowsByScope(ctx context.Context, organization, project string) ([]models.DrsObjectRecord, error) { + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return nil, err + } + rows, err := db.db.QueryContext(ctx, ` + SELECT bi.object_id, o.size, o.updated_time, + COALESCE(u.download_count, 0), u.last_download_time, + bi.normalized_path, bi.normalized_path + FROM drs_object_browse_index bi + JOIN drs_object o ON o.id = bi.object_id + LEFT JOIN object_usage u ON u.object_id = bi.object_id + WHERE bi.resource = $1 + ORDER BY bi.object_id + `, resource) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]models.DrsObjectRecord, 0) + for rows.Next() { + var row models.DrsObjectRecord + var lastDownload sql.NullTime + if err := rows.Scan(&row.ID, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload, &row.Name, &row.FileName); err != nil { + return nil, err + } + if lastDownload.Valid { + t := lastDownload.Time.UTC() + row.LastDownloadTime = &t + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func (db *PostgresDB) listScopedFileUsagePage(ctx context.Context, resources []string, includeUnscoped bool, limit, offset int, inactiveSince *time.Time) ([]models.FileUsage, error) { if err := db.flushObjectUsageEvents(ctx); err != nil { return nil, err diff --git a/internal/db/postgres/metrics_sqlmock_test.go b/internal/db/postgres/metrics_sqlmock_test.go index b1f92628..423f756c 100644 --- a/internal/db/postgres/metrics_sqlmock_test.go +++ b/internal/db/postgres/metrics_sqlmock_test.go @@ -109,5 +109,45 @@ func TestRecordProviderTransferEvents_EmptyInput(t *testing.T) { } } +func TestGetStoragePathSummary(t *testing.T) { + pg, mock, rawDB := newMockPostgresDB(t) + defer rawDB.Close() - + now := time.Now().UTC() + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT DISTINCT e.object_id + FROM object_usage_event e + JOIN drs_object o ON o.id = e.object_id + `)).WillReturnRows(sqlmock.NewRows([]string{"object_id"})) + mock.ExpectCommit() + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT bi.object_id, o.size, o.updated_time, + COALESCE(u.download_count, 0), u.last_download_time, + bi.normalized_path, bi.normalized_path + FROM drs_object_browse_index bi + JOIN drs_object o ON o.id = bi.object_id + LEFT JOIN object_usage u ON u.object_id = bi.object_id + WHERE bi.resource = $1 + ORDER BY bi.object_id + `)). + WithArgs("/organization/org/project/proj"). + WillReturnRows(sqlmock.NewRows([]string{"object_id", "size", "updated_time", "download_count", "last_download_time", "normalized_path", "normalized_path"}). + AddRow("obj-1", int64(10), now.Add(-time.Hour), int64(2), now.Add(-3*time.Hour), "data/a.txt", "data/a.txt"). + AddRow("obj-2", int64(20), now, int64(4), now.Add(-30*time.Minute), "data/a.txt", "data/a.txt"). + AddRow("obj-3", int64(30), now.Add(-2*time.Hour), int64(1), now.Add(-90*time.Minute), "data/nested/b.txt", "data/nested/b.txt")) + + summary, err := pg.GetStoragePathSummary(context.Background(), "org", "proj", "data") + if err != nil { + t.Fatalf("GetStoragePathSummary returned error: %v", err) + } + if summary.RecordCount != 3 || summary.FileCount != 2 || summary.TotalBytes != 60 || summary.DuplicatePathCount != 1 || summary.DownloadCount != 7 { + t.Fatalf("unexpected summary: %+v", summary) + } + if summary.LastDownloadTime == nil || !summary.LastDownloadTime.Equal(now.Add(-30*time.Minute)) { + t.Fatalf("unexpected last download time: %+v", summary.LastDownloadTime) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } +} diff --git a/internal/db/sqlite/metrics.go b/internal/db/sqlite/metrics.go index b331a43f..d1a97f56 100644 --- a/internal/db/sqlite/metrics.go +++ b/internal/db/sqlite/metrics.go @@ -13,6 +13,7 @@ import ( sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" + "github.com/calypr/syfon/internal/storagemetrics" ) func (db *SqliteDB) flushObjectUsageEventsForIDsTx(ctx context.Context, tx *sql.Tx, ids []string) error { @@ -265,6 +266,81 @@ func (db *SqliteDB) GetTransferAttributionBreakdown(ctx context.Context, filter return scanTransferAttributionBreakdown(rows) } +func (db *SqliteDB) GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) { + if err := db.flushObjectUsageEvents(ctx); err != nil { + return models.StoragePathSummary{}, err + } + rows, err := db.listStorageMetricRowsByScope(ctx, organization, project) + if err != nil { + return models.StoragePathSummary{}, err + } + return storagemetrics.AggregateStoragePathSummary(strings.TrimSpace(organization), strings.TrimSpace(project), path, rows) +} + +func (db *SqliteDB) ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) { + if err := db.flushObjectUsageEvents(ctx); err != nil { + return nil, err + } + rows, err := db.listStorageMetricRowsByScope(ctx, organization, project) + if err != nil { + return nil, err + } + return storagemetrics.AggregateStoragePathChildren(path, rows, limit, offset, sortBy, sortOrder) +} + +func (db *SqliteDB) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + if err := db.flushObjectUsageEvents(ctx); err != nil { + return nil, err + } + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return nil, err + } + _, prefixSegments, err := common.NormalizeBrowsePath(pathPrefix) + if err != nil { + return nil, err + } + rows, err := db.db.QueryContext(ctx, ` + SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, + COALESCE(u.download_count, 0), u.last_download_time + FROM drs_object_browse_index bi + JOIN drs_object o ON o.id = bi.object_id + LEFT JOIN object_usage u ON u.object_id = bi.object_id + WHERE bi.resource = ? + ORDER BY bi.normalized_path ASC, bi.object_id ASC + `, resource) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.StorageCleanupRecord, 0) + for rows.Next() { + var row models.StorageCleanupRecord + var lastDownload sql.NullTime + if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { + return nil, err + } + info, ok, err := common.BrowsePathInfoFromName(row.NormalizedPath) + if err != nil { + return nil, err + } + if !ok || !common.HasBrowsePathPrefix(info.Segments, prefixSegments) { + continue + } + row.NormalizedPath = info.Normalized + if lastDownload.Valid { + t := lastDownload.Time.UTC() + row.LastDownloadTime = &t + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func (db *SqliteDB) GetTransferAttributionSummaryByResources(ctx context.Context, filter models.TransferAttributionFilter, resources []string) (models.TransferAttributionSummary, error) { where, args := transferAttributionWhereByResources(filter, resources) var out models.TransferAttributionSummary @@ -496,6 +572,44 @@ func scanFileUsageRows(rows *sql.Rows, capacity int) ([]models.FileUsage, error) return out, nil } +func (db *SqliteDB) listStorageMetricRowsByScope(ctx context.Context, organization, project string) ([]models.DrsObjectRecord, error) { + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return nil, err + } + rows, err := db.db.QueryContext(ctx, ` + SELECT bi.object_id, o.size, o.updated_time, + COALESCE(u.download_count, 0), u.last_download_time, + bi.normalized_path, bi.normalized_path + FROM drs_object_browse_index bi + JOIN drs_object o ON o.id = bi.object_id + LEFT JOIN object_usage u ON u.object_id = bi.object_id + WHERE bi.resource = ? + ORDER BY bi.object_id + `, resource) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]models.DrsObjectRecord, 0) + for rows.Next() { + var row models.DrsObjectRecord + var lastDownload sql.NullTime + if err := rows.Scan(&row.ID, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload, &row.Name, &row.FileName); err != nil { + return nil, err + } + if lastDownload.Valid { + t := lastDownload.Time.UTC() + row.LastDownloadTime = &t + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func (db *SqliteDB) listScopedFileUsagePage(ctx context.Context, resources []string, includeUnscoped bool, limit, offset int, inactiveSince *time.Time) ([]models.FileUsage, error) { if err := db.flushObjectUsageEvents(ctx); err != nil { return nil, err diff --git a/internal/db/sqlite/sqlite_test.go b/internal/db/sqlite/sqlite_test.go index f1f62876..e771a948 100644 --- a/internal/db/sqlite/sqlite_test.go +++ b/internal/db/sqlite/sqlite_test.go @@ -1344,6 +1344,85 @@ func TestSqliteDB_TransferAttributionByResources(t *testing.T) { } } +func TestSqliteDB_StoragePathMetricsByScope(t *testing.T) { + ctx := context.Background() + db, err := NewSqliteDB(":memory:") + if err != nil { + t.Fatalf("failed to create db: %v", err) + } + now := time.Now().UTC() + for _, obj := range []models.InternalObject{ + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "obj-1", + Name: common.Ptr("data/a.txt"), + Size: 10, + CreatedTime: now, + UpdatedTime: &now, + }, + }, + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "obj-2", + Name: common.Ptr("data/a.txt"), + Size: 20, + CreatedTime: now, + UpdatedTime: &now, + }, + }, + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "obj-3", + Name: common.Ptr("data/nested/b.txt"), + Size: 30, + CreatedTime: now, + UpdatedTime: &now, + }, + }, + } { + if err := db.CreateObject(ctx, &obj); err != nil { + t.Fatalf("CreateObject failed: %v", err) + } + } + if err := db.RecordFileDownload(ctx, "obj-1"); err != nil { + t.Fatalf("RecordFileDownload failed: %v", err) + } + if err := db.RecordFileDownload(ctx, "obj-2"); err != nil { + t.Fatalf("RecordFileDownload failed: %v", err) + } + if err := db.RecordFileDownload(ctx, "obj-2"); err != nil { + t.Fatalf("RecordFileDownload failed: %v", err) + } + + summary, err := db.GetStoragePathSummary(ctx, "org", "p1", "data") + if err != nil { + t.Fatalf("GetStoragePathSummary failed: %v", err) + } + if summary.RecordCount != 3 || summary.FileCount != 2 || summary.TotalBytes != 60 || summary.DuplicatePathCount != 1 || summary.DownloadCount != 3 { + t.Fatalf("unexpected storage summary: %+v", summary) + } + if summary.LastDownloadTime == nil { + t.Fatalf("expected last download time in summary: %+v", summary) + } + + items, err := db.ListStoragePathChildren(ctx, "org", "p1", "data", 10, 0, "bytes", "desc") + if err != nil { + t.Fatalf("ListStoragePathChildren failed: %v", err) + } + if len(items) != 2 { + t.Fatalf("expected 2 storage children, got %+v", items) + } + if items[0].Name != "nested" || items[0].Type != "directory" || items[0].TotalBytes != 30 || items[0].DownloadCount != 0 { + t.Fatalf("unexpected first child: %+v", items[0]) + } + if items[1].Name != "a.txt" || items[1].DownloadCount != 3 || items[1].LastDownloadTime == nil { + t.Fatalf("unexpected second child: %+v", items[1]) + } +} + func TestSqliteDB_ListBucketVisibilityRows(t *testing.T) { ctx := context.Background() db, err := NewSqliteDB(":memory:") diff --git a/internal/models/storage.go b/internal/models/storage.go index 6d4b3e94..6b7c12d0 100644 --- a/internal/models/storage.go +++ b/internal/models/storage.go @@ -54,6 +54,50 @@ type FileUsageSummary struct { InactiveFileCount int64 } +type StoragePathSummary struct { + Organization string + Project string + Path string + FileCount int64 + RecordCount int64 + TotalBytes int64 + DownloadCount int64 + DirectChildCount int64 + DuplicatePathCount int64 + LatestUpdateTime *time.Time + LastDownloadTime *time.Time +} + +type StoragePathChild struct { + Name string + Path string + Type string + FileCount int64 + RecordCount int64 + TotalBytes int64 + LatestUpdateTime *time.Time + DownloadCount int64 + LastDownloadTime *time.Time +} + +type StoragePathChildrenResponse struct { + Organization string + Project string + Path string + Items []StoragePathChild +} + +// StorageCleanupRecord is the browse-index-backed projection used by storage +// cleanup audit/apply flows. +type StorageCleanupRecord struct { + ObjectID string + NormalizedPath string + Size int64 + UpdatedTime time.Time + DownloadCount int64 + LastDownloadTime *time.Time +} + // BucketVisibilityRow is the minimum storage projection needed to build bucket // visibility responses without hydrating full objects. type BucketVisibilityRow struct { @@ -214,12 +258,14 @@ type TransferAttributionBreakdown struct { // DrsObjectRecord mirrors the subset of drs_object columns returned by storage queries. type DrsObjectRecord struct { - ID string - Size int64 - CreatedTime time.Time - UpdatedTime time.Time - Name string - FileName string - Version string - Description string + ID string + Size int64 + CreatedTime time.Time + UpdatedTime time.Time + DownloadCount int64 + LastDownloadTime *time.Time + Name string + FileName string + Version string + Description string } diff --git a/internal/repair/format.go b/internal/repair/format.go new file mode 100644 index 00000000..8e6acfcb --- /dev/null +++ b/internal/repair/format.go @@ -0,0 +1,84 @@ +package repair + +import ( + "encoding/json" + "fmt" + "io" +) + +func WriteAudit(w io.Writer, format string, report Report) error { + if format == "json" { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(report) + } + if _, err := fmt.Fprintf(w, "scanned=%d findings=%d\n", report.Scanned, len(report.Objects)); err != nil { + return err + } + for _, obj := range report.Objects { + if _, err := fmt.Fprintf(w, "%s sha256=%s scope=%s/%s autofix=%t\n", obj.ObjectID, obj.SHA256, obj.Organization, obj.Project, obj.AutoFixable); err != nil { + return err + } + for _, finding := range obj.Findings { + if _, err := fmt.Fprintf(w, " - %s [%s] %s\n", finding.Kind, finding.Severity, finding.Message); err != nil { + return err + } + } + } + return nil +} + +func WriteApply(w io.Writer, format string, result ApplyResult) error { + if format == "json" { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(result) + } + if err := WriteAudit(w, format, result.Report); err != nil { + return err + } + _, err := fmt.Fprintf(w, "mutated=%d skipped=%d auto_fixable=%d\n", result.Mutated, result.Skipped, result.AutoFixable) + return err +} + +func WriteStorageCleanupAudit(w io.Writer, format string, report StorageCleanupReport) error { + if format == "json" { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(report) + } + if _, err := fmt.Fprintf(w, "scope=%s/%s scanned=%d findings=%d\n", report.Organization, report.Project, report.Scanned, len(report.Findings)); err != nil { + return err + } + for _, finding := range report.Findings { + if _, err := fmt.Fprintf(w, "%s [%s] %s action=%s\n", finding.NormalizedPath, finding.Kind, finding.Message, finding.RecommendedAction); err != nil { + return err + } + for _, record := range finding.Records { + if _, err := fmt.Fprintf(w, " - %s storage=%s downloads=%d\n", record.ObjectID, record.StorageStatus, record.DownloadCount); err != nil { + return err + } + } + } + return nil +} + +func WriteStorageCleanupApply(w io.Writer, format string, result StorageCleanupApplyResult) error { + if format == "json" { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(result) + } + if err := WriteStorageCleanupAudit(w, format, result.Report); err != nil { + return err + } + if _, err := fmt.Fprintf(w, "dry_run=%t deleted=%d skipped=%d repo_delete_paths=%d\n", result.DryRun, len(result.DeletedRecordIDs), len(result.Skipped), len(result.RepoDeletePaths)); err != nil { + return err + } + for _, skipped := range result.Skipped { + if _, err := fmt.Fprintf(w, " - skipped %s %s: %s\n", skipped.Kind, skipped.ObjectID, skipped.Reason); err != nil { + return err + } + } + return nil +} diff --git a/internal/repair/service.go b/internal/repair/service.go new file mode 100644 index 00000000..765401d8 --- /dev/null +++ b/internal/repair/service.go @@ -0,0 +1,502 @@ +package repair + +import ( + "context" + "fmt" + "net/http" + "net/url" + "sort" + "strings" + + drsapi "github.com/calypr/syfon/apigen/client/drs" + "github.com/calypr/syfon/apigen/client/internalapi" + "github.com/calypr/syfon/client/request" + syfoncommon "github.com/calypr/syfon/common" + intcommon "github.com/calypr/syfon/internal/common" +) + +const defaultPageSize = 500 + +func (s *Service) Audit(ctx context.Context, opts Options) (Report, error) { + state, err := s.audit(ctx, opts) + if err != nil { + return Report{}, err + } + return state.report, nil +} + +func (s *Service) Apply(ctx context.Context, opts Options) (ApplyResult, error) { + opts.Organization = strings.TrimSpace(opts.Organization) + opts.Project = strings.TrimSpace(opts.Project) + if opts.Organization == "" || opts.Project == "" { + return ApplyResult{}, fmt.Errorf("apply requires --organization and --project") + } + + state, err := s.audit(ctx, opts) + if err != nil { + return ApplyResult{}, err + } + + result := ApplyResult{Report: state.report} + for _, obj := range state.objects { + if obj.updated == nil { + continue + } + result.AutoFixable++ + if _, err := s.index.Update(ctx, obj.record.Did, *obj.updated); err != nil { + result.Skipped++ + continue + } + result.Mutated++ + } + return result, nil +} + +func (s *Service) audit(ctx context.Context, opts Options) (*auditState, error) { + scopes, err := s.loadScopeTargets(ctx) + if err != nil { + return nil, err + } + records, scanned, err := s.listRecords(ctx, opts) + if err != nil { + return nil, err + } + + state := &auditState{ + report: Report{ + Organization: strings.TrimSpace(opts.Organization), + Project: strings.TrimSpace(opts.Project), + Scanned: scanned, + }, + } + for _, rec := range records { + obj, include := s.auditRecord(ctx, rec, scopes, opts) + if !include { + continue + } + state.objects = append(state.objects, obj) + } + s.addDuplicateFindings(state.objects) + sort.Slice(state.objects, func(i, j int) bool { return state.objects[i].record.Did < state.objects[j].record.Did }) + for _, obj := range state.objects { + if len(obj.findings) == 0 { + continue + } + state.report.Objects = append(state.report.Objects, ObjectReport{ + ObjectID: obj.record.Did, + SHA256: obj.sha256, + Organization: obj.scope.Organization, + Project: obj.scope.Project, + CurrentAccessURLs: append([]string(nil), obj.currentURLs...), + ProposedCanonicalURL: obj.canonicalURL, + AutoFixable: obj.updated != nil, + Findings: append([]Finding(nil), obj.findings...), + }) + } + return state, nil +} + +func (s *Service) listRecords(ctx context.Context, opts Options) ([]internalapi.InternalRecord, int, error) { + pageSize := opts.PageSize + if pageSize <= 0 { + pageSize = defaultPageSize + } + var out []internalapi.InternalRecord + start := "" + scanned := 0 + for { + limit := pageSize + if opts.Limit > 0 && opts.Limit-scanned < limit { + limit = opts.Limit - scanned + } + if limit <= 0 && opts.Limit > 0 { + break + } + resp, err := s.index.List(ctx, ListRecordsOptions{Limit: limit, Start: start}) + if err != nil { + return nil, scanned, err + } + if resp.Records == nil || len(*resp.Records) == 0 { + break + } + records := *resp.Records + out = append(out, records...) + scanned += len(records) + start = strings.TrimSpace(records[len(records)-1].Did) + if len(records) < limit || start == "" { + break + } + } + return out, scanned, nil +} + +func (s *Service) auditRecord(ctx context.Context, rec internalapi.InternalRecord, scopes map[string][]scopeTarget, opts Options) (*auditedObject, bool) { + sha := "" + if rec.Hashes != nil { + sha = syfoncommon.NormalizeChecksum((*rec.Hashes)["sha256"]) + } + obj := &auditedObject{ + record: rec, + sha256: sha, + currentURLs: accessMethodURLs(rec.AccessMethods), + } + + resource, known, ambiguous := inferRecordResource(rec, sha, scopes) + obj.scopeKnown = known + obj.scopeAmbiguous = ambiguous + obj.inferredScope = resource + if known { + obj.scope = scopes[resource][0] + obj.canonicalURL = canonicalAccessURL(obj.scope, rec.Did, sha) + } + + targetResource := "" + if strings.TrimSpace(opts.Organization) != "" && strings.TrimSpace(opts.Project) != "" { + targetResource, _ = syfoncommon.ResourcePath(opts.Organization, opts.Project) + } + if targetResource != "" && !recordMatchesResource(rec, targetResource) && obj.inferredScope != targetResource { + return obj, false + } + + if targetResource != "" && !recordMatchesResource(rec, targetResource) && obj.inferredScope == targetResource && sha != "" { + obj.findings = append(obj.findings, newFinding(FindingMissingControlledAccess, SeverityWarn, rec, sha, obj.currentURLs, obj.canonicalURL, true, "missing controlled_access row recoverable from deterministic scope")) + updated := cloneRecord(rec) + updated.ControlledAccess = addControlledAccess(updated.ControlledAccess, targetResource) + obj.updated = &updated + } + if obj.scopeKnown && obj.canonicalURL != "" { + s.classifyAccessMethods(obj) + } + if opts.CheckStorage { + s.addStorageFindings(ctx, obj) + } + return obj, true +} + +func (s *Service) classifyAccessMethods(obj *auditedObject) { + if obj.record.AccessMethods == nil { + return + } + methods := cloneAccessMethods(*obj.record.AccessMethods) + hasCanonical := false + for _, raw := range obj.currentURLs { + if raw == obj.canonicalURL { + hasCanonical = true + break + } + } + + changed := false + for i := range methods { + raw := accessMethodURL(methods[i]) + if raw == "" || raw == obj.canonicalURL { + continue + } + if isLegacyPathStyleURL(raw) { + if hasCanonical { + obj.findings = append(obj.findings, newFinding(FindingLegacyAccessURLRemovable, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, obj.canonicalURL, true, "legacy META/CONFIG URL has canonical sibling")) + methods[i].AccessUrl = nil + changed = true + continue + } + obj.findings = append(obj.findings, newFinding(FindingLegacyAccessURLRewritable, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, obj.canonicalURL, true, "legacy META/CONFIG URL can be rewritten to canonical scoped URL")) + setAccessMethodURL(&methods[i], obj.canonicalURL) + changed = true + continue + } + obj.findings = append(obj.findings, newFinding(FindingNonCanonicalAccessURL, SeverityInfo, obj.record, obj.sha256, obj.currentURLs, obj.canonicalURL, false, "access URL does not match canonical scoped target")) + } + + if !changed { + return + } + filtered := make([]drsapi.AccessMethod, 0, len(methods)) + for _, method := range methods { + if accessMethodURL(method) == "" { + continue + } + filtered = append(filtered, method) + } + updated := cloneRecord(obj.record) + updated.AccessMethods = &filtered + if obj.updated != nil && obj.updated.ControlledAccess != nil { + updated.ControlledAccess = obj.updated.ControlledAccess + } + obj.updated = &updated +} + +func (s *Service) addStorageFindings(ctx context.Context, obj *auditedObject) { + for _, raw := range obj.currentURLs { + req := storageInspectRequest{ + Organization: obj.scope.Organization, + Project: obj.scope.Project, + ObjectURL: raw, + } + var resp storageInspectResponse + err := s.requestor.Do(ctx, http.MethodPost, "/data/inspect", req, &resp) + if err == nil { + continue + } + kind := FindingStorageProbeError + severity := SeverityWarn + message := err.Error() + if respErr, ok := err.(*request.ResponseError); ok && respErr.Status == http.StatusNotFound { + kind = FindingStorageObjectMissing + severity = SeverityError + message = "storage object not found" + } + obj.findings = append(obj.findings, newFinding(kind, severity, obj.record, obj.sha256, []string{raw}, obj.canonicalURL, false, message)) + } +} + +func (s *Service) loadScopeTargets(ctx context.Context) (map[string][]scopeTarget, error) { + buckets, err := s.buckets.List(ctx) + if err != nil { + return nil, fmt.Errorf("list buckets: %w", err) + } + out := map[string][]scopeTarget{} + for bucket := range buckets.S3BUCKETS { + scopes, err := s.buckets.ListScopes(ctx, bucket) + if err != nil { + return nil, fmt.Errorf("list bucket scopes for %s: %w", bucket, err) + } + for _, scope := range scopes { + resource, err := syfoncommon.ResourcePath(strings.TrimSpace(scope.Organization), strings.TrimSpace(scope.ProjectId)) + if err != nil || resource == "" { + continue + } + target := scopeTarget{ + Resource: resource, + Organization: strings.TrimSpace(scope.Organization), + Project: strings.TrimSpace(scope.ProjectId), + Bucket: bucket, + } + if scope.Path != nil { + parsed, err := url.Parse(strings.TrimSpace(*scope.Path)) + if err == nil { + if host := strings.TrimSpace(parsed.Host); host != "" { + target.Bucket = host + } + target.Prefix = strings.Trim(strings.TrimSpace(parsed.Path), "/") + } + } + out[resource] = append(out[resource], target) + } + } + return out, nil +} + +func (s *Service) addDuplicateFindings(objects []*auditedObject) { + byKey := map[string][]*auditedObject{} + for _, obj := range objects { + if obj.sha256 == "" { + continue + } + for _, resource := range recordProjectResources(obj.record, obj.inferredScope) { + byKey[resource+"|"+obj.sha256] = append(byKey[resource+"|"+obj.sha256], obj) + } + } + for key, group := range byKey { + if len(group) < 2 { + continue + } + resource := strings.SplitN(key, "|", 2)[0] + scope := intcommon.ParseResourcePath(resource) + for _, obj := range group { + obj.findings = append(obj.findings, newFinding(FindingDuplicateSHA256Sibling, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, obj.canonicalURL, false, "same sha256 appears in multiple DIDs for this scope")) + if obj.scope.Organization == "" { + obj.scope.Organization = scope.Organization + obj.scope.Project = scope.Project + } + } + } +} + +func inferRecordResource(rec internalapi.InternalRecord, sha string, scopes map[string][]scopeTarget) (string, bool, bool) { + projectResources := recordProjectResources(rec, "") + if len(projectResources) == 1 { + return projectResources[0], true, false + } + if len(projectResources) > 1 { + return "", false, true + } + if strings.TrimSpace(sha) == "" { + return "", false, false + } + matches := []string{} + for resource, targets := range scopes { + if len(targets) == 0 || strings.TrimSpace(targets[0].Project) == "" { + continue + } + minted, err := intcommon.MintObjectIDFromChecksum(sha, []string{resource}) + if err != nil { + continue + } + if minted == strings.TrimSpace(rec.Did) { + matches = append(matches, resource) + } + } + if len(matches) == 1 { + return matches[0], true, false + } + if len(matches) > 1 { + return "", false, true + } + return "", false, false +} + +func recordProjectResources(rec internalapi.InternalRecord, inferred string) []string { + seen := map[string]struct{}{} + var out []string + if rec.ControlledAccess != nil { + for _, resource := range syfoncommon.NormalizeAccessResources(*rec.ControlledAccess) { + org, project, ok := syfoncommon.ResourceScope(resource) + if !ok || org == "" || project == "" { + continue + } + if _, exists := seen[resource]; exists { + continue + } + seen[resource] = struct{}{} + out = append(out, resource) + } + } + if inferred != "" { + if _, exists := seen[inferred]; !exists { + out = append(out, inferred) + } + } + sort.Strings(out) + return out +} + +func recordMatchesResource(rec internalapi.InternalRecord, resource string) bool { + if rec.ControlledAccess == nil { + return false + } + for _, candidate := range syfoncommon.NormalizeAccessResources(*rec.ControlledAccess) { + if candidate == resource { + return true + } + } + return false +} + +func accessMethodURLs(methods *[]drsapi.AccessMethod) []string { + if methods == nil { + return nil + } + var out []string + for _, method := range *methods { + if raw := accessMethodURL(method); raw != "" { + out = append(out, raw) + } + } + return out +} + +func accessMethodURL(method drsapi.AccessMethod) string { + if method.AccessUrl == nil { + return "" + } + return strings.TrimSpace(method.AccessUrl.Url) +} + +func setAccessMethodURL(method *drsapi.AccessMethod, raw string) { + if method.AccessUrl == nil { + method.AccessUrl = &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{} + } + method.AccessUrl.Url = raw +} + +func cloneAccessMethods(in []drsapi.AccessMethod) []drsapi.AccessMethod { + out := make([]drsapi.AccessMethod, len(in)) + copy(out, in) + return out +} + +func cloneRecord(rec internalapi.InternalRecord) internalapi.InternalRecord { + out := rec + if rec.AccessMethods != nil { + methods := cloneAccessMethods(*rec.AccessMethods) + out.AccessMethods = &methods + } + if rec.ControlledAccess != nil { + controlled := append([]string(nil), (*rec.ControlledAccess)...) + out.ControlledAccess = &controlled + } + if rec.Hashes != nil { + hashes := make(internalapi.HashInfo, len(*rec.Hashes)) + for k, v := range *rec.Hashes { + hashes[k] = v + } + out.Hashes = &hashes + } + return out +} + +func addControlledAccess(controlled *[]string, resource string) *[]string { + resources := append([]string(nil), syfoncommon.NormalizeAccessResources(derefStrings(controlled))...) + resources = append(resources, resource) + normalized := syfoncommon.NormalizeAccessResources(resources) + return &normalized +} + +func derefStrings(values *[]string) []string { + if values == nil { + return nil + } + return *values +} + +func canonicalAccessURL(target scopeTarget, did, sha string) string { + if strings.TrimSpace(target.Bucket) == "" || strings.TrimSpace(did) == "" || strings.TrimSpace(sha) == "" { + return "" + } + parts := []string{} + if prefix := strings.Trim(target.Prefix, "/"); prefix != "" { + parts = append(parts, prefix) + } + parts = append(parts, strings.TrimSpace(did), strings.TrimSpace(sha)) + return "s3://" + strings.TrimSpace(target.Bucket) + "/" + strings.Join(parts, "/") +} + +func isLegacyPathStyleURL(raw string) bool { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || !strings.EqualFold(parsed.Scheme, "s3") { + return false + } + path := strings.Trim(strings.TrimSpace(parsed.Path), "/") + if path == "" { + return false + } + head := strings.ToUpper(strings.Split(path, "/")[0]) + return head == "META" || head == "CONFIG" +} + +func newFinding(kind FindingKind, severity Severity, rec internalapi.InternalRecord, sha string, currentURLs []string, canonical string, autoFixable bool, msg string) Finding { + finding := Finding{ + Kind: kind, + Severity: severity, + ObjectID: rec.Did, + SHA256: sha, + CurrentAccessURLs: append([]string(nil), currentURLs...), + ProposedCanonicalURL: canonical, + AutoFixable: autoFixable, + Message: msg, + } + if rec.ControlledAccess != nil { + for _, resource := range syfoncommon.NormalizeAccessResources(*rec.ControlledAccess) { + org, project, ok := syfoncommon.ResourceScope(resource) + if ok && org != "" { + finding.Organization = org + finding.Project = project + break + } + } + } + return finding +} diff --git a/internal/repair/service_test.go b/internal/repair/service_test.go new file mode 100644 index 00000000..9ca2ca11 --- /dev/null +++ b/internal/repair/service_test.go @@ -0,0 +1,268 @@ +package repair + +import ( + "context" + "net/http" + "testing" + + "github.com/calypr/syfon/apigen/client/bucketapi" + drsapi "github.com/calypr/syfon/apigen/client/drs" + "github.com/calypr/syfon/apigen/client/internalapi" + "github.com/calypr/syfon/client/request" + syfoncommon "github.com/calypr/syfon/common" + intcommon "github.com/calypr/syfon/internal/common" +) + +func TestLegacyAccessURLWithCanonicalSiblingIsRemovable(t *testing.T) { + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{ + recordWithMethods(t, "did-1", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{ + "s3://bforepc/META/file.ndjson", + "s3://bforepc/bforepc-prod/did-1/abc", + }), + }}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 1 { + t.Fatalf("expected one object report, got %d", len(report.Objects)) + } + obj := report.Objects[0] + if !obj.AutoFixable { + t.Fatal("expected autofixable object") + } + if obj.Findings[0].Kind != FindingLegacyAccessURLRemovable { + t.Fatalf("expected removable finding, got %s", obj.Findings[0].Kind) + } +} + +func TestLegacyAccessURLWithoutCanonicalSiblingIsRewritable(t *testing.T) { + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{ + recordWithMethods(t, "did-1", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{ + "s3://bforepc/META/file.ndjson", + }), + }}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if got := report.Objects[0].Findings[0].Kind; got != FindingLegacyAccessURLRewritable { + t.Fatalf("expected rewritable finding, got %s", got) + } +} + +func TestMissingControlledAccessRecoverableFromDeterministicScope(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + did, err := intcommon.MintObjectIDFromChecksum("abc", []string{resource}) + if err != nil { + t.Fatalf("mint did: %v", err) + } + rec := recordWithMethods(t, did, "abc", nil, []string{"s3://bforepc/bforepc-prod/" + did + "/abc"}) + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{rec}}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if got := report.Objects[0].Findings[0].Kind; got != FindingMissingControlledAccess { + t.Fatalf("expected missing controlled access finding, got %s", got) + } + if !report.Objects[0].AutoFixable { + t.Fatal("expected autofixable object") + } +} + +func TestDuplicateSHAReportedWithoutAutofix(t *testing.T) { + rec1 := recordWithMethods(t, "did-1", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{"s3://bforepc/bforepc-prod/did-1/abc"}) + rec2 := recordWithMethods(t, "did-2", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{"s3://bforepc/bforepc-prod/did-2/abc"}) + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{rec1, rec2}}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 2 { + t.Fatalf("expected two object reports, got %d", len(report.Objects)) + } + for _, obj := range report.Objects { + found := false + for _, finding := range obj.Findings { + if finding.Kind == FindingDuplicateSHA256Sibling { + found = true + if finding.AutoFixable { + t.Fatal("duplicate sibling should not be autofixable") + } + } + } + if !found { + t.Fatalf("expected duplicate finding for %s", obj.ObjectID) + } + } +} + +func TestApplyUpdatesAccessMethodsAndControlledAccess(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + did, err := intcommon.MintObjectIDFromChecksum("abc", []string{resource}) + if err != nil { + t.Fatalf("mint did: %v", err) + } + rec := recordWithMethods(t, did, "abc", nil, []string{"s3://bforepc/META/file.ndjson"}) + idx := &fakeIndex{records: []internalapi.InternalRecord{rec}} + svc := NewService(idx, fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), &fakeRequester{}) + + result, err := svc.Apply(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if result.Mutated != 1 { + t.Fatalf("expected one mutation, got %d", result.Mutated) + } + if len(idx.updated) != 1 { + t.Fatalf("expected one updated record, got %d", len(idx.updated)) + } + updated := idx.updated[0] + if updated.ControlledAccess == nil || len(*updated.ControlledAccess) != 1 || (*updated.ControlledAccess)[0] != resource { + t.Fatalf("expected controlled access backfill, got %#v", updated.ControlledAccess) + } + if got := accessMethodURLs(updated.AccessMethods); len(got) != 1 || got[0] != "s3://bforepc/bforepc-prod/"+did+"/abc" { + t.Fatalf("unexpected access methods: %#v", got) + } +} + +func TestCheckStorageAddsMissingObjectFinding(t *testing.T) { + rec := recordWithMethods(t, "did-1", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{"s3://bforepc/bforepc-prod/did-1/abc"}) + req := &fakeRequester{ + err: &request.ResponseError{Method: http.MethodPost, URL: "/data/inspect", Status: http.StatusNotFound}, + } + svc := NewService(&fakeIndex{records: []internalapi.InternalRecord{rec}}, fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), req) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC", CheckStorage: true}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + found := false + for _, finding := range report.Objects[0].Findings { + if finding.Kind == FindingStorageObjectMissing { + found = true + } + } + if !found { + t.Fatal("expected storage object missing finding") + } + if req.calls == 0 { + t.Fatal("expected storage inspect call") + } +} + +type fakeIndex struct { + records []internalapi.InternalRecord + updated []internalapi.InternalRecord + listErr error + updateErr error +} + +func (f *fakeIndex) List(ctx context.Context, opts ListRecordsOptions) (internalapi.ListRecordsResponse, error) { + if f.listErr != nil { + return internalapi.ListRecordsResponse{}, f.listErr + } + records := append([]internalapi.InternalRecord(nil), f.records...) + return internalapi.ListRecordsResponse{Records: &records}, nil +} + +func (f *fakeIndex) Update(ctx context.Context, did string, rec internalapi.InternalRecord) (internalapi.InternalRecordResponse, error) { + if f.updateErr != nil { + return internalapi.InternalRecordResponse{}, f.updateErr + } + f.updated = append(f.updated, rec) + return internalapi.InternalRecordResponse{Did: did}, nil +} + +type fakeBuckets struct { + list bucketapi.BucketsResponse + scopes map[string][]bucketapi.BucketScopeResponse +} + +func (f *fakeBuckets) List(ctx context.Context) (bucketapi.BucketsResponse, error) { + return f.list, nil +} + +func (f *fakeBuckets) ListScopes(ctx context.Context, bucket string) ([]bucketapi.BucketScopeResponse, error) { + return f.scopes[bucket], nil +} + +type fakeRequester struct { + err error + calls int +} + +func (f *fakeRequester) Do(ctx context.Context, method, path string, body, out any, opts ...request.RequestOption) error { + f.calls++ + if f.err != nil { + return f.err + } + return nil +} + +func recordWithMethods(t *testing.T, did, sha string, controlled []string, urls []string) internalapi.InternalRecord { + t.Helper() + hashes := internalapi.HashInfo{"sha256": sha} + methods := make([]drsapi.AccessMethod, 0, len(urls)) + for _, raw := range urls { + accessURL := raw + methods = append(methods, drsapi.AccessMethod{ + Type: drsapi.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: accessURL}, + }) + } + rec := internalapi.InternalRecord{ + Did: did, + Hashes: &hashes, + AccessMethods: &methods, + } + if controlled != nil { + c := append([]string(nil), controlled...) + rec.ControlledAccess = &c + } + return rec +} + +func fakeBucketsForScope(t *testing.T, org, project, rawPath string) *fakeBuckets { + t.Helper() + return &fakeBuckets{ + list: bucketapi.BucketsResponse{ + S3BUCKETS: map[string]bucketapi.BucketMetadata{ + "bforepc": {}, + }, + }, + scopes: map[string][]bucketapi.BucketScopeResponse{ + "bforepc": {{ + Organization: org, + ProjectId: project, + Path: &rawPath, + }}, + }, + } +} + +var _ request.Requester = (*fakeRequester)(nil) diff --git a/internal/repair/storage_cleanup_service.go b/internal/repair/storage_cleanup_service.go new file mode 100644 index 00000000..c990f5f3 --- /dev/null +++ b/internal/repair/storage_cleanup_service.go @@ -0,0 +1,421 @@ +package repair + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + drsapi "github.com/calypr/syfon/apigen/server/drs" + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/models" +) + +type storageCleanupManager interface { + ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) + GetBulkObjects(ctx context.Context, ids []string, requiredMethod string) ([]models.InternalObject, error) + InspectStorageObject(ctx context.Context, req core.InspectStorageRequest) (*core.StorageObjectMetadata, error) + DeleteObjectWithOptions(ctx context.Context, id string, opts core.DeleteOptions) error +} + +type StorageCleanupService struct { + om storageCleanupManager +} + +func NewStorageCleanupService(om storageCleanupManager) *StorageCleanupService { + return &StorageCleanupService{om: om} +} + +type storageCleanupPathState struct { + Path string + Records []StorageCleanupRecordAudit +} + +func (s *StorageCleanupService) Audit(ctx context.Context, req StorageCleanupAuditRequest) (StorageCleanupReport, error) { + checkStorage := true + if req.CheckStorage != nil { + checkStorage = *req.CheckStorage + } + report := StorageCleanupReport{ + Organization: strings.TrimSpace(req.Organization), + Project: strings.TrimSpace(req.Project), + PathPrefix: strings.TrimSpace(req.PathPrefix), + Summary: map[FindingKind]int{}, + } + expected, hasExpected, err := normalizeExpectedPaths(req.ExpectedPaths) + if err != nil { + return StorageCleanupReport{}, err + } + rows, err := s.om.ListStorageCleanupRecords(ctx, report.Organization, report.Project, report.PathPrefix) + if err != nil { + return StorageCleanupReport{}, err + } + report.Scanned = len(rows) + + ids := uniqueCleanupIDs(rows) + objects, err := s.om.GetBulkObjects(ctx, ids, "read") + if err != nil { + return StorageCleanupReport{}, err + } + objectByID := make(map[string]models.InternalObject, len(objects)) + for _, obj := range objects { + objectByID[strings.TrimSpace(obj.Id)] = obj + } + + grouped := make(map[string]*storageCleanupPathState) + for _, row := range rows { + state := grouped[row.NormalizedPath] + if state == nil { + state = &storageCleanupPathState{Path: row.NormalizedPath} + grouped[row.NormalizedPath] = state + } + record := StorageCleanupRecordAudit{ + ObjectID: row.ObjectID, + NormalizedPath: row.NormalizedPath, + Size: row.Size, + DownloadCount: row.DownloadCount, + LastDownloadTime: utcTimePtr(row.LastDownloadTime), + UpdatedTime: utcTimeValue(row.UpdatedTime), + } + if obj, ok := objectByID[row.ObjectID]; ok && obj.AccessMethods != nil { + record.CurrentAccessURLs = storageCleanupAccessMethodURLs(*obj.AccessMethods) + } + if checkStorage { + record.StorageStatus, record.StorageMessage = s.inspectCleanupRecord(ctx, report.Organization, report.Project, record.CurrentAccessURLs) + } else { + record.StorageStatus = StorageProbeStatusUnknown + } + state.Records = append(state.Records, record) + } + + paths := make([]string, 0, len(grouped)) + for path := range grouped { + paths = append(paths, path) + } + sort.Strings(paths) + + for _, path := range paths { + state := grouped[path] + sort.Slice(state.Records, func(i, j int) bool { + return state.Records[i].ObjectID < state.Records[j].ObjectID + }) + findings := classifyStorageCleanupPath(state, hasExpected, expected[state.Path]) + for _, finding := range findings { + report.Summary[finding.Kind]++ + report.Findings = append(report.Findings, finding) + } + } + if len(report.Summary) == 0 { + report.Summary = nil + } + return report, nil +} + +func (s *StorageCleanupService) Apply(ctx context.Context, req StorageCleanupApplyRequest) (StorageCleanupApplyResult, error) { + checkStorage := req.CheckStorage + report, err := s.Audit(ctx, StorageCleanupAuditRequest{ + Organization: req.Organization, + Project: req.Project, + PathPrefix: req.PathPrefix, + ExpectedPaths: req.ExpectedPaths, + CheckStorage: checkStorage, + }) + if err != nil { + return StorageCleanupApplyResult{}, err + } + result := StorageCleanupApplyResult{Report: report, DryRun: req.DryRun} + actions, repoPaths := buildStorageCleanupActions(report, req) + result.RepoDeletePaths = repoPaths + if req.DryRun { + return result, nil + } + ids := make([]string, 0, len(actions)) + for id := range actions { + ids = append(ids, id) + } + sort.Strings(ids) + for _, objectID := range ids { + action := actions[objectID] + if err := s.om.DeleteObjectWithOptions(ctx, objectID, core.DeleteOptions{DeleteStorageData: action.DeleteStorageData}); err != nil { + result.Skipped = append(result.Skipped, StorageCleanupSkipped{ + NormalizedPath: action.Path, + ObjectID: objectID, + Kind: action.Kind, + Reason: err.Error(), + }) + if action.DeleteStorageData { + result.StoragePurgeResults = append(result.StoragePurgeResults, StorageCleanupStoragePurgeResult{ + ObjectID: objectID, + Purged: false, + Message: err.Error(), + }) + } + continue + } + result.DeletedRecordIDs = append(result.DeletedRecordIDs, objectID) + if action.DeleteStorageData { + result.StoragePurgeResults = append(result.StoragePurgeResults, StorageCleanupStoragePurgeResult{ + ObjectID: objectID, + Purged: true, + }) + } + } + return result, nil +} + +func (s *StorageCleanupService) inspectCleanupRecord(ctx context.Context, organization, project string, urls []string) (StorageProbeStatus, string) { + if len(urls) == 0 { + return StorageProbeStatusUnknown, "record has no access URLs" + } + var sawMissing bool + var lastErr error + for _, raw := range urls { + if strings.TrimSpace(raw) == "" { + continue + } + _, err := s.om.InspectStorageObject(ctx, core.InspectStorageRequest{ + Organization: organization, + Project: project, + ObjectURL: raw, + }) + if err == nil { + return StorageProbeStatusLive, "" + } + if inspectErr, ok := err.(*core.StorageInspectError); ok && inspectErr.Kind == core.StorageInspectObjectNotFound { + sawMissing = true + lastErr = err + continue + } + lastErr = err + } + if sawMissing { + return StorageProbeStatusMissing, "storage object not found" + } + if lastErr != nil { + return StorageProbeStatusError, lastErr.Error() + } + return StorageProbeStatusUnknown, "" +} + +func classifyStorageCleanupPath(state *storageCleanupPathState, hasExpected, inExpectedSet bool) []StorageCleanupFinding { + liveCount, missingCount, errorCount := 0, 0, 0 + for _, rec := range state.Records { + switch rec.StorageStatus { + case StorageProbeStatusLive: + liveCount++ + case StorageProbeStatusMissing: + missingCount++ + case StorageProbeStatusError: + errorCount++ + } + } + findings := make([]StorageCleanupFinding, 0, 2) + if len(state.Records) > 1 { + switch { + case liveCount > 1: + findings = append(findings, StorageCleanupFinding{ + Kind: FindingLiveDuplicateConflict, + Severity: SeverityWarn, + NormalizedPath: state.Path, + Message: fmt.Sprintf("multiple records for %s still resolve in storage", state.Path), + RecommendedAction: "manual_review", + Records: cloneCleanupRecords(state.Records), + }) + case liveCount >= 1 && missingCount >= 1 && errorCount == 0: + findings = append(findings, StorageCleanupFinding{ + Kind: FindingStaleDuplicateRecord, + Severity: SeverityWarn, + NormalizedPath: state.Path, + Message: fmt.Sprintf("duplicate path %s has stale records that no longer resolve", state.Path), + RecommendedAction: "delete_stale_duplicates", + Records: cloneCleanupRecords(state.Records), + }) + case errorCount > 0: + findings = append(findings, StorageCleanupFinding{ + Kind: FindingStorageProbeError, + Severity: SeverityWarn, + NormalizedPath: state.Path, + Message: fmt.Sprintf("duplicate path %s could not be fully verified in storage", state.Path), + RecommendedAction: "manual_review", + Records: cloneCleanupRecords(state.Records), + }) + } + } + if hasExpected && !inExpectedSet { + switch { + case liveCount > 0: + findings = append(findings, StorageCleanupFinding{ + Kind: FindingRepoOrphanLiveObject, + Severity: SeverityWarn, + NormalizedPath: state.Path, + Message: fmt.Sprintf("path %s is absent from expected repo contents but still resolves in storage", state.Path), + RecommendedAction: "delete_repo_orphans", + RepoDeleteCandidate: true, + Records: cloneCleanupRecords(state.Records), + }) + case missingCount > 0 && errorCount == 0: + findings = append(findings, StorageCleanupFinding{ + Kind: FindingRepoOrphanStaleRecord, + Severity: SeverityWarn, + NormalizedPath: state.Path, + Message: fmt.Sprintf("path %s is absent from expected repo contents and storage is already missing", state.Path), + RecommendedAction: "delete_repo_orphans", + RepoDeleteCandidate: true, + Records: cloneCleanupRecords(state.Records), + }) + case errorCount > 0: + findings = append(findings, StorageCleanupFinding{ + Kind: FindingStorageProbeError, + Severity: SeverityWarn, + NormalizedPath: state.Path, + Message: fmt.Sprintf("path %s is absent from expected repo contents but storage verification failed", state.Path), + RecommendedAction: "manual_review", + RepoDeleteCandidate: true, + Records: cloneCleanupRecords(state.Records), + }) + } + } + return findings +} + +type storageCleanupAction struct { + Path string + Kind FindingKind + DeleteStorageData bool +} + +func buildStorageCleanupActions(report StorageCleanupReport, req StorageCleanupApplyRequest) (map[string]storageCleanupAction, []string) { + pathFilter := make(map[string]bool, len(req.SelectedPaths)) + for _, path := range req.SelectedPaths { + if normalized, _, err := common.NormalizeBrowsePath(path); err == nil { + pathFilter[normalized] = true + } + } + objectFilter := make(map[string]bool, len(req.SelectedObjectIDs)) + for _, objectID := range req.SelectedObjectIDs { + objectID = strings.TrimSpace(objectID) + if objectID != "" { + objectFilter[objectID] = true + } + } + kindFilter := make(map[FindingKind]bool, len(req.SelectedFindingKinds)) + for _, kind := range req.SelectedFindingKinds { + kindFilter[kind] = true + } + repoPaths := make(map[string]bool) + actions := map[string]storageCleanupAction{} + for _, finding := range report.Findings { + if len(pathFilter) > 0 && !pathFilter[finding.NormalizedPath] { + continue + } + if len(kindFilter) > 0 && !kindFilter[finding.Kind] { + continue + } + switch finding.Kind { + case FindingStaleDuplicateRecord: + if !req.DeleteStaleDuplicates { + continue + } + for _, rec := range finding.Records { + if rec.StorageStatus != StorageProbeStatusMissing { + continue + } + if len(objectFilter) > 0 && !objectFilter[rec.ObjectID] { + continue + } + actions[rec.ObjectID] = storageCleanupAction{Path: finding.NormalizedPath, Kind: finding.Kind} + } + case FindingRepoOrphanLiveObject, FindingRepoOrphanStaleRecord: + if !req.DeleteRepoOrphans { + continue + } + repoPaths[finding.NormalizedPath] = true + for _, rec := range finding.Records { + if len(objectFilter) > 0 && !objectFilter[rec.ObjectID] { + continue + } + switch rec.StorageStatus { + case StorageProbeStatusLive: + actions[rec.ObjectID] = storageCleanupAction{Path: finding.NormalizedPath, Kind: finding.Kind, DeleteStorageData: true} + case StorageProbeStatusMissing: + if current, ok := actions[rec.ObjectID]; !ok || !current.DeleteStorageData { + actions[rec.ObjectID] = storageCleanupAction{Path: finding.NormalizedPath, Kind: finding.Kind} + } + } + } + } + } + outPaths := make([]string, 0, len(repoPaths)) + for path := range repoPaths { + outPaths = append(outPaths, path) + } + sort.Strings(outPaths) + return actions, outPaths +} + +func normalizeExpectedPaths(values []string) (map[string]bool, bool, error) { + if len(values) == 0 { + return nil, false, nil + } + out := make(map[string]bool, len(values)) + for _, raw := range values { + normalized, _, err := common.NormalizeBrowsePath(raw) + if err != nil { + return nil, false, err + } + out[normalized] = true + } + return out, true, nil +} + +func uniqueCleanupIDs(rows []models.StorageCleanupRecord) []string { + seen := make(map[string]bool, len(rows)) + out := make([]string, 0, len(rows)) + for _, row := range rows { + if seen[row.ObjectID] { + continue + } + seen[row.ObjectID] = true + out = append(out, row.ObjectID) + } + sort.Strings(out) + return out +} + +func storageCleanupAccessMethodURLs(methods []drsapi.AccessMethod) []string { + out := make([]string, 0, len(methods)) + for _, method := range methods { + if method.AccessUrl == nil { + continue + } + raw := strings.TrimSpace(method.AccessUrl.Url) + if raw != "" { + out = append(out, raw) + } + } + return out +} + +func cloneCleanupRecords(in []StorageCleanupRecordAudit) []StorageCleanupRecordAudit { + out := make([]StorageCleanupRecordAudit, len(in)) + copy(out, in) + return out +} + +func utcTimeValue(v time.Time) *time.Time { + if v.IsZero() { + return nil + } + t := v.UTC() + return &t +} + +func utcTimePtr(v *time.Time) *time.Time { + if v == nil { + return nil + } + t := v.UTC() + return &t +} diff --git a/internal/repair/storage_cleanup_service_test.go b/internal/repair/storage_cleanup_service_test.go new file mode 100644 index 00000000..bc28b68e --- /dev/null +++ b/internal/repair/storage_cleanup_service_test.go @@ -0,0 +1,207 @@ +package repair + +import ( + "context" + "testing" + + drsapi "github.com/calypr/syfon/apigen/server/drs" + "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/models" +) + +func TestStorageCleanupAuditStaleDuplicateRecord(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-live", NormalizedPath: "data/file.tsv", Size: 10}, + {ObjectID: "obj-missing", NormalizedPath: "data/file.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "obj-live": cleanupObject("obj-live", "s3://bucket/live/file.tsv"), + "obj-missing": cleanupObject("obj-missing", "s3://bucket/missing/file.tsv"), + }, + } + manager.inspectByObject = map[string]error{ + "obj-live": nil, + "obj-missing": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, + } + + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if len(report.Findings) != 1 { + t.Fatalf("expected one finding, got %d", len(report.Findings)) + } + if report.Findings[0].Kind != FindingStaleDuplicateRecord { + t.Fatalf("expected stale duplicate finding, got %s", report.Findings[0].Kind) + } +} + +func TestStorageCleanupAuditLiveDuplicateConflict(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-1", NormalizedPath: "data/file.tsv", Size: 10}, + {ObjectID: "obj-2", NormalizedPath: "data/file.tsv", Size: 11}, + }, + objects: map[string]models.InternalObject{ + "obj-1": cleanupObject("obj-1", "s3://bucket/object-1/file.tsv"), + "obj-2": cleanupObject("obj-2", "s3://bucket/object-2/file.tsv"), + }, + } + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{Organization: "org", Project: "proj"}) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if len(report.Findings) != 1 || report.Findings[0].Kind != FindingLiveDuplicateConflict { + t.Fatalf("expected live duplicate conflict, got %+v", report.Findings) + } +} + +func TestStorageCleanupAuditRepoOrphanStatuses(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-live", NormalizedPath: "data/live.tsv", Size: 10}, + {ObjectID: "obj-stale", NormalizedPath: "data/stale.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "obj-live": cleanupObject("obj-live", "s3://bucket/data/live.tsv"), + "obj-stale": cleanupObject("obj-stale", "s3://bucket/data/stale.tsv"), + }, + inspectByObject: map[string]error{ + "obj-live": nil, + "obj-stale": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, + }, + } + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + ExpectedPaths: []string{"data/expected.tsv"}, + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if len(report.Findings) != 2 { + t.Fatalf("expected two orphan findings, got %d", len(report.Findings)) + } + if report.Findings[0].Kind != FindingRepoOrphanLiveObject || report.Findings[1].Kind != FindingRepoOrphanStaleRecord { + t.Fatalf("unexpected findings: %+v", report.Findings) + } +} + +func TestStorageCleanupApplyDeletesExpectedRecords(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "dup-live", NormalizedPath: "data/dup.tsv", Size: 10}, + {ObjectID: "dup-missing", NormalizedPath: "data/dup.tsv", Size: 10}, + {ObjectID: "orphan-live", NormalizedPath: "data/orphan.tsv", Size: 10}, + {ObjectID: "orphan-stale", NormalizedPath: "data/stale.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "dup-live": cleanupObject("dup-live", "s3://bucket/dup/live.tsv"), + "dup-missing": cleanupObject("dup-missing", "s3://bucket/dup/missing.tsv"), + "orphan-live": cleanupObject("orphan-live", "s3://bucket/data/orphan.tsv"), + "orphan-stale": cleanupObject("orphan-stale", "s3://bucket/data/stale.tsv"), + }, + inspectByObject: map[string]error{ + "dup-live": nil, + "dup-missing": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, + "orphan-live": nil, + "orphan-stale": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, + }, + } + svc := NewStorageCleanupService(manager) + result, err := svc.Apply(context.Background(), StorageCleanupApplyRequest{ + Organization: "org", + Project: "proj", + ExpectedPaths: []string{"data/dup.tsv", "data/keep.tsv"}, + DeleteStaleDuplicates: true, + DeleteRepoOrphans: true, + }) + if err != nil { + t.Fatalf("Apply failed: %v", err) + } + if len(result.DeletedRecordIDs) != 3 { + t.Fatalf("expected three deleted records, got %d", len(result.DeletedRecordIDs)) + } + if got := manager.deleteOpts["dup-missing"]; got.DeleteStorageData { + t.Fatal("stale duplicate delete should not purge storage") + } + if got := manager.deleteOpts["orphan-live"]; !got.DeleteStorageData { + t.Fatal("live orphan delete should purge storage") + } + if got := manager.deleteOpts["orphan-stale"]; got.DeleteStorageData { + t.Fatal("stale orphan delete should not purge storage") + } +} + +type fakeCleanupManager struct { + rows []models.StorageCleanupRecord + objects map[string]models.InternalObject + inspectByObject map[string]error + deleteOpts map[string]core.DeleteOptions +} + +func (f *fakeCleanupManager) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + return append([]models.StorageCleanupRecord(nil), f.rows...), nil +} + +func (f *fakeCleanupManager) GetBulkObjects(ctx context.Context, ids []string, requiredMethod string) ([]models.InternalObject, error) { + out := make([]models.InternalObject, 0, len(ids)) + for _, id := range ids { + if obj, ok := f.objects[id]; ok { + out = append(out, obj) + } + } + return out, nil +} + +func (f *fakeCleanupManager) InspectStorageObject(ctx context.Context, req core.InspectStorageRequest) (*core.StorageObjectMetadata, error) { + for id, obj := range f.objects { + for _, method := range derefServerMethods(obj.AccessMethods) { + if method.AccessUrl != nil && method.AccessUrl.Url == req.ObjectURL { + if err, ok := f.inspectByObject[id]; ok { + return nil, err + } + return &core.StorageObjectMetadata{ObjectURL: req.ObjectURL}, nil + } + } + } + return &core.StorageObjectMetadata{ObjectURL: req.ObjectURL}, nil +} + +func (f *fakeCleanupManager) DeleteObjectWithOptions(ctx context.Context, id string, opts core.DeleteOptions) error { + if f.deleteOpts == nil { + f.deleteOpts = map[string]core.DeleteOptions{} + } + f.deleteOpts[id] = opts + return nil +} + +func cleanupObject(id, url string) models.InternalObject { + methods := []drsapi.AccessMethod{{ + Type: drsapi.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: url}, + }} + return models.InternalObject{ + DrsObject: drsapi.DrsObject{ + Id: id, + AccessMethods: &methods, + }, + } +} + +func derefServerMethods(methods *[]drsapi.AccessMethod) []drsapi.AccessMethod { + if methods == nil { + return nil + } + return *methods +} diff --git a/internal/repair/storage_cleanup_types.go b/internal/repair/storage_cleanup_types.go new file mode 100644 index 00000000..16151850 --- /dev/null +++ b/internal/repair/storage_cleanup_types.go @@ -0,0 +1,94 @@ +package repair + +import "time" + +type StorageProbeStatus string + +const ( + StorageProbeStatusUnknown StorageProbeStatus = "unknown" + StorageProbeStatusLive StorageProbeStatus = "live" + StorageProbeStatusMissing StorageProbeStatus = "missing" + StorageProbeStatusError StorageProbeStatus = "error" +) + +const ( + FindingStaleDuplicateRecord FindingKind = "stale_duplicate_record" + FindingLiveDuplicateConflict FindingKind = "live_duplicate_conflict" + FindingRepoOrphanLiveObject FindingKind = "repo_orphan_live_object" + FindingRepoOrphanStaleRecord FindingKind = "repo_orphan_stale_record" +) + +type StorageCleanupRecordAudit struct { + ObjectID string `json:"object_id"` + NormalizedPath string `json:"normalized_path"` + Size int64 `json:"size"` + UpdatedTime *time.Time `json:"updated_time,omitempty"` + DownloadCount int64 `json:"download_count"` + LastDownloadTime *time.Time `json:"last_download_time,omitempty"` + CurrentAccessURLs []string `json:"current_access_urls,omitempty"` + StorageStatus StorageProbeStatus `json:"storage_status"` + StorageMessage string `json:"storage_message,omitempty"` +} + +type StorageCleanupFinding struct { + Kind FindingKind `json:"kind"` + Severity Severity `json:"severity"` + NormalizedPath string `json:"normalized_path"` + Message string `json:"message"` + RecommendedAction string `json:"recommended_action,omitempty"` + RepoDeleteCandidate bool `json:"repo_delete_candidate"` + Records []StorageCleanupRecordAudit `json:"records,omitempty"` +} + +type StorageCleanupReport struct { + Organization string `json:"organization"` + Project string `json:"project"` + PathPrefix string `json:"path_prefix,omitempty"` + Scanned int `json:"scanned"` + Summary map[FindingKind]int `json:"summary,omitempty"` + Findings []StorageCleanupFinding `json:"findings,omitempty"` +} + +type StorageCleanupAuditRequest struct { + Organization string `json:"organization"` + Project string `json:"project"` + PathPrefix string `json:"path_prefix,omitempty"` + ExpectedPaths []string `json:"expected_paths,omitempty"` + CheckStorage *bool `json:"check_storage,omitempty"` +} + +type StorageCleanupApplyRequest struct { + Organization string `json:"organization"` + Project string `json:"project"` + PathPrefix string `json:"path_prefix,omitempty"` + ExpectedPaths []string `json:"expected_paths,omitempty"` + DeleteStaleDuplicates bool `json:"delete_stale_duplicates"` + DeleteRepoOrphans bool `json:"delete_repo_orphans"` + DryRun bool `json:"dry_run"` + SelectedPaths []string `json:"selected_paths,omitempty"` + SelectedObjectIDs []string `json:"selected_object_ids,omitempty"` + SelectedFindingKinds []FindingKind `json:"selected_finding_kinds,omitempty"` + CheckStorage *bool `json:"check_storage,omitempty"` +} + +type StorageCleanupSkipped struct { + NormalizedPath string `json:"normalized_path"` + ObjectID string `json:"object_id,omitempty"` + Kind FindingKind `json:"kind"` + Reason string `json:"reason"` +} + +type StorageCleanupStoragePurgeResult struct { + ObjectID string `json:"object_id"` + Purged bool `json:"purged"` + Message string `json:"message,omitempty"` +} + +type StorageCleanupApplyResult struct { + Report StorageCleanupReport `json:"report"` + DryRun bool `json:"dry_run"` + DeletedRecordIDs []string `json:"deleted_record_ids,omitempty"` + StoragePurgeResults []StorageCleanupStoragePurgeResult `json:"storage_purge_results,omitempty"` + RepoDeletePaths []string `json:"repo_delete_paths,omitempty"` + Skipped []StorageCleanupSkipped `json:"skipped,omitempty"` +} diff --git a/internal/repair/types.go b/internal/repair/types.go new file mode 100644 index 00000000..b23324d1 --- /dev/null +++ b/internal/repair/types.go @@ -0,0 +1,137 @@ +package repair + +import ( + "context" + + "github.com/calypr/syfon/apigen/client/bucketapi" + "github.com/calypr/syfon/apigen/client/internalapi" + "github.com/calypr/syfon/client/request" +) + +type IndexAPI interface { + List(ctx context.Context, opts ListRecordsOptions) (internalapi.ListRecordsResponse, error) + Update(ctx context.Context, did string, rec internalapi.InternalRecord) (internalapi.InternalRecordResponse, error) +} + +type ListRecordsOptions struct { + Limit int + Start string +} + +type BucketsAPI interface { + List(ctx context.Context) (bucketapi.BucketsResponse, error) + ListScopes(ctx context.Context, bucket string) ([]bucketapi.BucketScopeResponse, error) +} + +type Options struct { + Organization string + Project string + CheckStorage bool + Format string + Limit int + PageSize int +} + +type FindingKind string + +const ( + FindingLegacyAccessURLRemovable FindingKind = "legacy_access_url_removable" + FindingLegacyAccessURLRewritable FindingKind = "legacy_access_url_rewritable" + FindingNonCanonicalAccessURL FindingKind = "non_canonical_access_url" + FindingMissingControlledAccess FindingKind = "missing_controlled_access" + FindingDuplicateSHA256Sibling FindingKind = "duplicate_sha256_sibling" + FindingStorageObjectMissing FindingKind = "storage_object_missing" + FindingStorageProbeError FindingKind = "storage_probe_error" +) + +type Severity string + +const ( + SeverityInfo Severity = "info" + SeverityWarn Severity = "warn" + SeverityError Severity = "error" +) + +type Finding struct { + Kind FindingKind `json:"kind"` + Severity Severity `json:"severity"` + ObjectID string `json:"object_id"` + SHA256 string `json:"sha256,omitempty"` + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + CurrentAccessURLs []string `json:"current_access_urls,omitempty"` + ProposedCanonicalURL string `json:"proposed_canonical_url,omitempty"` + AutoFixable bool `json:"auto_fixable"` + Message string `json:"message,omitempty"` +} + +type ObjectReport struct { + ObjectID string `json:"object_id"` + SHA256 string `json:"sha256,omitempty"` + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + CurrentAccessURLs []string `json:"current_access_urls,omitempty"` + ProposedCanonicalURL string `json:"proposed_canonical_url,omitempty"` + AutoFixable bool `json:"auto_fixable"` + Findings []Finding `json:"findings,omitempty"` +} + +type Report struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + Scanned int `json:"scanned"` + Objects []ObjectReport `json:"objects,omitempty"` +} + +type ApplyResult struct { + Report Report `json:"report"` + Mutated int `json:"mutated"` + Skipped int `json:"skipped"` + AutoFixable int `json:"auto_fixable"` +} + +type Service struct { + index IndexAPI + buckets BucketsAPI + requestor request.Requester +} + +func NewService(index IndexAPI, buckets BucketsAPI, requestor request.Requester) *Service { + return &Service{index: index, buckets: buckets, requestor: requestor} +} + +type scopeTarget struct { + Resource string + Organization string + Project string + Bucket string + Prefix string +} + +type storageInspectRequest struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + ObjectURL string `json:"object_url,omitempty"` +} + +type storageInspectResponse struct { + ObjectURL string `json:"object_url"` +} + +type auditState struct { + report Report + objects []*auditedObject +} + +type auditedObject struct { + record internalapi.InternalRecord + sha256 string + currentURLs []string + scope scopeTarget + scopeKnown bool + scopeAmbiguous bool + inferredScope string + canonicalURL string + findings []Finding + updated *internalapi.InternalRecord +} diff --git a/internal/storagemetrics/storage_metrics.go b/internal/storagemetrics/storage_metrics.go new file mode 100644 index 00000000..20c6814b --- /dev/null +++ b/internal/storagemetrics/storage_metrics.go @@ -0,0 +1,248 @@ +package storagemetrics + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/models" +) + +const ( + StoragePathChildTypeFile = "file" + StoragePathChildTypeDirectory = "directory" +) + +func AggregateStoragePathSummary(organization, project, path string, rows []models.DrsObjectRecord) (models.StoragePathSummary, error) { + normalizedPath, requestedSegments, err := common.NormalizeBrowsePath(path) + if err != nil { + return models.StoragePathSummary{}, err + } + summary := models.StoragePathSummary{ + Organization: organization, + Project: project, + Path: normalizedPath, + } + distinctPaths := make(map[string]struct{}) + directChildren := make(map[string]struct{}) + recordsByPath := make(map[string]int64) + for _, row := range rows { + info, ok := storageMetricPathInfo(row) + if !ok || !common.HasBrowsePathPrefix(info.Segments, requestedSegments) { + continue + } + if len(info.Segments) > len(requestedSegments) { + childPath := strings.Join(info.Segments[:len(requestedSegments)+1], "/") + directChildren[childPath] = struct{}{} + } + summary.RecordCount++ + summary.TotalBytes += row.Size + summary.DownloadCount += row.DownloadCount + distinctPaths[info.Normalized] = struct{}{} + recordsByPath[info.Normalized]++ + if row.LastDownloadTime != nil && (summary.LastDownloadTime == nil || row.LastDownloadTime.After(*summary.LastDownloadTime)) { + t := row.LastDownloadTime.UTC() + summary.LastDownloadTime = &t + } + if row.UpdatedTime.IsZero() { + continue + } + if summary.LatestUpdateTime == nil || row.UpdatedTime.After(*summary.LatestUpdateTime) { + t := row.UpdatedTime.UTC() + summary.LatestUpdateTime = &t + } + } + summary.FileCount = int64(len(distinctPaths)) + summary.DirectChildCount = int64(len(directChildren)) + for _, count := range recordsByPath { + if count > 1 { + summary.DuplicatePathCount++ + } + } + return summary, nil +} + +func AggregateStoragePathChildren(path string, rows []models.DrsObjectRecord, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) { + _, requestedSegments, err := common.NormalizeBrowsePath(path) + if err != nil { + return nil, err + } + type childAggregate struct { + item models.StoragePathChild + distinctPaths map[string]struct{} + isDirectory bool + } + children := make(map[string]*childAggregate) + for _, row := range rows { + info, ok := storageMetricPathInfo(row) + if !ok || !common.HasBrowsePathPrefix(info.Segments, requestedSegments) || len(info.Segments) <= len(requestedSegments) { + continue + } + childSegments := info.Segments[:len(requestedSegments)+1] + childPath := strings.Join(childSegments, "/") + agg, exists := children[childPath] + if !exists { + agg = &childAggregate{ + item: models.StoragePathChild{ + Name: childSegments[len(childSegments)-1], + Path: childPath, + Type: StoragePathChildTypeFile, + }, + distinctPaths: make(map[string]struct{}), + } + children[childPath] = agg + } + if len(info.Segments) > len(requestedSegments)+1 { + agg.isDirectory = true + agg.item.Type = StoragePathChildTypeDirectory + } + agg.item.RecordCount++ + agg.item.TotalBytes += row.Size + agg.item.DownloadCount += row.DownloadCount + agg.distinctPaths[info.Normalized] = struct{}{} + if row.LastDownloadTime != nil && (agg.item.LastDownloadTime == nil || row.LastDownloadTime.After(*agg.item.LastDownloadTime)) { + t := row.LastDownloadTime.UTC() + agg.item.LastDownloadTime = &t + } + if row.UpdatedTime.IsZero() { + continue + } + if agg.item.LatestUpdateTime == nil || row.UpdatedTime.After(*agg.item.LatestUpdateTime) { + t := row.UpdatedTime.UTC() + agg.item.LatestUpdateTime = &t + } + } + items := make([]models.StoragePathChild, 0, len(children)) + for _, agg := range children { + agg.item.FileCount = int64(len(agg.distinctPaths)) + if agg.isDirectory { + agg.item.Type = StoragePathChildTypeDirectory + } + items = append(items, agg.item) + } + sortStoragePathChildren(items, sortBy, sortOrder) + if offset < 0 { + offset = 0 + } + if offset >= len(items) { + return []models.StoragePathChild{}, nil + } + if limit <= 0 { + return items[offset:], nil + } + end := offset + limit + if end > len(items) { + end = len(items) + } + return items[offset:end], nil +} + +func storageMetricPathInfo(row models.DrsObjectRecord) (common.BrowsePathInfo, bool) { + raw := strings.TrimSpace(row.FileName) + if raw == "" { + raw = strings.TrimSpace(row.Name) + } + info, ok, err := common.BrowsePathInfoFromName(raw) + if err != nil || !ok { + return common.BrowsePathInfo{}, false + } + return info, true +} + +func sortStoragePathChildren(items []models.StoragePathChild, sortBy, sortOrder string) { + desc := strings.EqualFold(strings.TrimSpace(sortOrder), "desc") + key := strings.TrimSpace(sortBy) + if key == "" { + key = "name" + } + sort.Slice(items, func(i, j int) bool { + left := items[i] + right := items[j] + less := false + switch key { + case "bytes": + less = left.TotalBytes < right.TotalBytes + case "updated_time": + less = compareTimePtr(left.LatestUpdateTime, right.LatestUpdateTime) + case "records": + less = left.RecordCount < right.RecordCount + case "name": + fallthrough + default: + less = strings.ToLower(left.Name) < strings.ToLower(right.Name) + } + if equivalentSortValue(left, right, key) { + if strings.EqualFold(left.Name, right.Name) { + if desc { + return left.Path > right.Path + } + return left.Path < right.Path + } + if desc { + return strings.ToLower(left.Name) > strings.ToLower(right.Name) + } + return strings.ToLower(left.Name) < strings.ToLower(right.Name) + } + if desc { + return !less + } + return less + }) +} + +func equivalentSortValue(left, right models.StoragePathChild, key string) bool { + switch key { + case "bytes": + return left.TotalBytes == right.TotalBytes + case "updated_time": + return sameTimePtr(left.LatestUpdateTime, right.LatestUpdateTime) + case "records": + return left.RecordCount == right.RecordCount + default: + return strings.EqualFold(left.Name, right.Name) + } +} + +func compareTimePtr(left, right *time.Time) bool { + if left == nil && right == nil { + return false + } + if left == nil { + return true + } + if right == nil { + return false + } + return left.Before(*right) +} + +func sameTimePtr(left, right *time.Time) bool { + if left == nil || right == nil { + return left == right + } + return left.Equal(*right) +} + +func NormalizeStorageChildrenSort(sortBy, sortOrder string) (string, string, error) { + sortBy = strings.TrimSpace(sortBy) + if sortBy == "" { + sortBy = "name" + } + switch sortBy { + case "bytes", "name", "updated_time", "records": + default: + return "", "", fmt.Errorf("invalid sort_by") + } + sortOrder = strings.TrimSpace(sortOrder) + if sortOrder == "" { + sortOrder = "asc" + } + switch sortOrder { + case "asc", "desc": + default: + return "", "", fmt.Errorf("invalid sort_order") + } + return sortBy, sortOrder, nil +} diff --git a/internal/storagemetrics/storage_metrics_test.go b/internal/storagemetrics/storage_metrics_test.go new file mode 100644 index 00000000..2e116e97 --- /dev/null +++ b/internal/storagemetrics/storage_metrics_test.go @@ -0,0 +1,96 @@ +package storagemetrics + +import ( + "testing" + "time" + + "github.com/calypr/syfon/internal/models" +) + +func TestAggregateStoragePathSummary(t *testing.T) { + now := time.Now().UTC() + rows := []models.DrsObjectRecord{ + {Name: "data/a.txt", Size: 10, UpdatedTime: now.Add(-time.Hour), DownloadCount: 2, LastDownloadTime: timePtr(now.Add(-3 * time.Hour))}, + {FileName: "data/a.txt", Size: 20, UpdatedTime: now, DownloadCount: 4, LastDownloadTime: timePtr(now.Add(-30 * time.Minute))}, + {Name: "data/nested/b.txt", Size: 30, UpdatedTime: now.Add(-2 * time.Hour), DownloadCount: 1, LastDownloadTime: timePtr(now.Add(-90 * time.Minute))}, + {Name: "other/c.txt", Size: 40, UpdatedTime: now.Add(-3 * time.Hour)}, + {Name: "bad/../path", Size: 50, UpdatedTime: now}, + } + + summary, err := AggregateStoragePathSummary("org", "proj", "data", rows) + if err != nil { + t.Fatalf("AggregateStoragePathSummary returned error: %v", err) + } + if summary.Path != "data" { + t.Fatalf("expected normalized path data, got %q", summary.Path) + } + if summary.FileCount != 2 || summary.RecordCount != 3 || summary.TotalBytes != 60 { + t.Fatalf("unexpected counts: %+v", summary) + } + if summary.DirectChildCount != 2 { + t.Fatalf("expected 2 direct children, got %+v", summary) + } + if summary.DuplicatePathCount != 1 { + t.Fatalf("expected one duplicate path, got %+v", summary) + } + if summary.DownloadCount != 7 { + t.Fatalf("unexpected download count: %+v", summary) + } + if summary.LastDownloadTime == nil || !summary.LastDownloadTime.Equal(now.Add(-30*time.Minute)) { + t.Fatalf("unexpected last download time: %+v", summary.LastDownloadTime) + } + if summary.LatestUpdateTime == nil || !summary.LatestUpdateTime.Equal(now) { + t.Fatalf("unexpected latest update time: %+v", summary.LatestUpdateTime) + } +} + +func TestAggregateStoragePathChildren(t *testing.T) { + now := time.Now().UTC() + rows := []models.DrsObjectRecord{ + {Name: ` data\\alpha.txt `, Size: 10, UpdatedTime: now.Add(-time.Hour), DownloadCount: 2, LastDownloadTime: timePtr(now.Add(-10 * time.Hour))}, + {Name: "data/dir/one.txt", Size: 15, UpdatedTime: now.Add(-2 * time.Hour), DownloadCount: 3, LastDownloadTime: timePtr(now.Add(-4 * time.Hour))}, + {Name: "data/dir/two.txt", Size: 25, UpdatedTime: now, DownloadCount: 5, LastDownloadTime: timePtr(now.Add(-15 * time.Minute))}, + {Name: "data/alpha.txt", Size: 5, UpdatedTime: now.Add(-30 * time.Minute), DownloadCount: 7, LastDownloadTime: timePtr(now.Add(-2 * time.Hour))}, + {Name: "skip/elsewhere.txt", Size: 99, UpdatedTime: now}, + } + + items, err := AggregateStoragePathChildren("data", rows, 10, 0, "bytes", "desc") + if err != nil { + t.Fatalf("AggregateStoragePathChildren returned error: %v", err) + } + if len(items) != 2 { + t.Fatalf("expected 2 items, got %+v", items) + } + if items[0].Name != "dir" || items[0].Type != StoragePathChildTypeDirectory || items[0].FileCount != 2 || items[0].RecordCount != 2 || items[0].TotalBytes != 40 || items[0].DownloadCount != 8 { + t.Fatalf("unexpected directory aggregate: %+v", items[0]) + } + if items[0].LastDownloadTime == nil || !items[0].LastDownloadTime.Equal(now.Add(-15*time.Minute)) { + t.Fatalf("unexpected directory last download time: %+v", items[0].LastDownloadTime) + } + if items[1].Name != "alpha.txt" || items[1].Type != StoragePathChildTypeFile || items[1].FileCount != 1 || items[1].RecordCount != 2 || items[1].TotalBytes != 15 || items[1].DownloadCount != 9 { + t.Fatalf("unexpected file aggregate: %+v", items[1]) + } + if items[1].LastDownloadTime == nil || !items[1].LastDownloadTime.Equal(now.Add(-2*time.Hour)) { + t.Fatalf("unexpected file last download time: %+v", items[1].LastDownloadTime) + } +} + +func TestNormalizeStorageChildrenSort(t *testing.T) { + sortBy, sortOrder, err := NormalizeStorageChildrenSort("", "") + if err != nil { + t.Fatalf("NormalizeStorageChildrenSort returned error: %v", err) + } + if sortBy != "name" || sortOrder != "asc" { + t.Fatalf("unexpected defaults: %q %q", sortBy, sortOrder) + } + if _, _, err := NormalizeStorageChildrenSort("nope", "asc"); err == nil { + t.Fatal("expected invalid sort_by error") + } + if _, _, err := NormalizeStorageChildrenSort("name", "down"); err == nil { + t.Fatal("expected invalid sort_order error") + } +} + +func timePtr(t time.Time) *time.Time { + return &t +} diff --git a/internal/testutils/mocks.go b/internal/testutils/mocks.go index d91b54e9..a2d4d26c 100644 --- a/internal/testutils/mocks.go +++ b/internal/testutils/mocks.go @@ -12,6 +12,7 @@ import ( sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" + "github.com/calypr/syfon/internal/storagemetrics" "github.com/calypr/syfon/internal/urlmanager" ) @@ -151,6 +152,60 @@ func (m *MockDatabase) ListObjectIDsByScope(ctx context.Context, organization, p return ids, nil } +func (m *MockDatabase) GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) { + return storagemetrics.AggregateStoragePathSummary(strings.TrimSpace(organization), strings.TrimSpace(project), path, m.storageMetricRowsForScope(strings.TrimSpace(organization), strings.TrimSpace(project))) +} + +func (m *MockDatabase) ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) { + return storagemetrics.AggregateStoragePathChildren(path, m.storageMetricRowsForScope(strings.TrimSpace(organization), strings.TrimSpace(project)), limit, offset, sortBy, sortOrder) +} + +func (m *MockDatabase) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + _, prefixSegments, err := common.NormalizeBrowsePath(pathPrefix) + if err != nil { + return nil, err + } + rows := make([]models.StorageCleanupRecord, 0, len(m.Objects)) + for id, obj := range m.Objects { + authz := map[string][]string{} + if m.ObjectAuthz != nil { + authz = m.ObjectAuthz[id] + } + projects, ok := authz[organization] + if !ok { + continue + } + matchesProject := strings.TrimSpace(project) == "" + for _, candidate := range projects { + if candidate == project { + matchesProject = true + break + } + } + if !matchesProject || obj.Name == nil { + continue + } + info, ok, err := common.BrowsePathInfoFromName(*obj.Name) + if err != nil || !ok || !common.HasBrowsePathPrefix(info.Segments, prefixSegments) { + continue + } + row := models.StorageCleanupRecord{ + ObjectID: id, + NormalizedPath: info.Normalized, + Size: obj.Size, + } + if obj.UpdatedTime != nil { + row.UpdatedTime = obj.UpdatedTime.UTC() + } + if u, ok := m.Usage[id]; ok { + row.DownloadCount = u.DownloadCount + row.LastDownloadTime = u.LastDownloadTime + } + rows = append(rows, row) + } + return rows, nil +} + func (m *MockDatabase) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { normalizedPath, pathSegments, err := common.NormalizeBrowsePath(path) if err != nil { @@ -235,6 +290,68 @@ func (m *MockDatabase) ListObjectIDsPageByPath(ctx context.Context, organization return ids[offset:end], directories, nil } +func (m *MockDatabase) storageMetricRows() []models.DrsObjectRecord { + rows := make([]models.DrsObjectRecord, 0, len(m.Objects)) + for id, obj := range m.Objects { + row := models.DrsObjectRecord{ + ID: id, + Size: obj.Size, + } + if obj.Name != nil { + row.Name = *obj.Name + } + if obj.UpdatedTime != nil { + row.UpdatedTime = obj.UpdatedTime.UTC() + } + if u, ok := m.Usage[id]; ok { + row.DownloadCount = u.DownloadCount + row.LastDownloadTime = u.LastDownloadTime + } + rows = append(rows, row) + } + return rows +} + +func (m *MockDatabase) storageMetricRowsForScope(organization, project string) []models.DrsObjectRecord { + rows := make([]models.DrsObjectRecord, 0, len(m.Objects)) + for id, obj := range m.Objects { + authz := map[string][]string{} + if m.ObjectAuthz != nil { + authz = m.ObjectAuthz[id] + } + projects, ok := authz[organization] + if !ok { + continue + } + matchesProject := strings.TrimSpace(project) == "" + for _, candidate := range projects { + if candidate == project { + matchesProject = true + break + } + } + if !matchesProject { + continue + } + row := models.DrsObjectRecord{ + ID: id, + Size: obj.Size, + } + if obj.Name != nil { + row.Name = *obj.Name + } + if obj.UpdatedTime != nil { + row.UpdatedTime = obj.UpdatedTime.UTC() + } + if u, ok := m.Usage[id]; ok { + row.DownloadCount = u.DownloadCount + row.LastDownloadTime = u.LastDownloadTime + } + rows = append(rows, row) + } + return rows +} + func (m *MockDatabase) ListObjectIDsByResources(ctx context.Context, resources []string, includeUnscoped bool) ([]string, error) { allowed := map[string]struct{}{} for _, resource := range resources { From 7691882a6c20c4b428631238a4d8ab4243ff7752 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 25 Jun 2026 15:48:10 -0700 Subject: [PATCH 02/24] add additional features to support observability of storage issues --- apigen/client/internalapi/internal.gen.go | 63 +- apigen/openapi/internal.openapi.yaml | 35 + apigen/server/internalapi/internal.gen.go | 63 +- client/services/repair_service.go | 15 + client/services/types.go | 8 + cmd/repair/main.go | 306 ++++++++ cmd/repair/main_test.go | 53 ++ docs/frontend-handoff-storage-metrics.md | 272 ++++++++ docs/implementation-plan-storage-summary.md | 414 +++++++++++ internal/api/internaldrs/index.go | 1 + internal/api/internaldrs/repair_cleanup.go | 27 + internal/common/routes.go | 1 + internal/core/bucket_catalog.go | 4 + internal/core/object_storage_inspect.go | 142 +++- internal/core/service_test.go | 71 +- internal/core/storage_cleanup.go | 11 + internal/db/interface.go | 2 + internal/db/postgres/metrics.go | 82 ++- internal/db/postgres/metrics_sqlmock_test.go | 83 +++ internal/db/sqlite/metrics.go | 82 ++- internal/repair/format.go | 9 +- internal/repair/project_diff_service.go | 166 +++++ internal/repair/project_diff_service_test.go | 90 +++ internal/repair/project_diff_types.go | 46 ++ internal/repair/storage_cleanup_service.go | 660 ++++++++++++++++-- .../repair/storage_cleanup_service_test.go | 532 +++++++++++++- internal/repair/storage_cleanup_types.go | 71 +- internal/testutils/mocks.go | 18 + 28 files changed, 3136 insertions(+), 191 deletions(-) create mode 100644 cmd/repair/main.go create mode 100644 cmd/repair/main_test.go create mode 100644 docs/frontend-handoff-storage-metrics.md create mode 100644 docs/implementation-plan-storage-summary.md create mode 100644 internal/repair/project_diff_service.go create mode 100644 internal/repair/project_diff_service_test.go create mode 100644 internal/repair/project_diff_types.go diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index de99bfa2..516ca0e2 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -240,36 +240,55 @@ type StorageCleanupAuditRequest struct { // StorageCleanupFinding defines model for StorageCleanupFinding. type StorageCleanupFinding struct { - Kind *string `json:"kind,omitempty"` - Message *string `json:"message,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - RecommendedAction *string `json:"recommended_action,omitempty"` - Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` - RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` - Severity *string `json:"severity,omitempty"` + CleanupScope *string `json:"cleanup_scope,omitempty"` + ChecksumCount *int `json:"checksum_count,omitempty"` + Kind *string `json:"kind,omitempty"` + LegacyUrlTemplateDetected *bool `json:"legacy_url_template_detected,omitempty"` + Message *string `json:"message,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + RecommendedAction *string `json:"recommended_action,omitempty"` + Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` + RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` + SizeCount *int `json:"size_count,omitempty"` + StructuralReason *string `json:"structural_reason,omitempty"` + Severity *string `json:"severity,omitempty"` +} + +// StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. +type StorageCleanupAccessProbe struct { + Bucket *string `json:"bucket,omitempty"` + ErrorKind *string `json:"error_kind,omitempty"` + StorageMessage *string `json:"storage_message,omitempty"` + StorageStatus *string `json:"storage_status,omitempty"` + Url *string `json:"url,omitempty"` } // StorageCleanupRecordAudit defines model for StorageCleanupRecordAudit. type StorageCleanupRecordAudit struct { - CurrentAccessUrls *[]string `json:"current_access_urls,omitempty"` - DownloadCount *int64 `json:"download_count,omitempty"` - LastDownloadTime *time.Time `json:"last_download_time,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - ObjectId *string `json:"object_id,omitempty"` - Size *int64 `json:"size,omitempty"` - StorageMessage *string `json:"storage_message,omitempty"` - StorageStatus *string `json:"storage_status,omitempty"` - UpdatedTime *time.Time `json:"updated_time,omitempty"` + AccessProbes *[]StorageCleanupAccessProbe `json:"access_probes,omitempty"` + CleanupScope *string `json:"cleanup_scope,omitempty"` + CurrentAccessUrls *[]string `json:"current_access_urls,omitempty"` + DownloadCount *int64 `json:"download_count,omitempty"` + LastDownloadTime *time.Time `json:"last_download_time,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + ObjectId *string `json:"object_id,omitempty"` + Size *int64 `json:"size,omitempty"` + StorageMessage *string `json:"storage_message,omitempty"` + StorageStatus *string `json:"storage_status,omitempty"` + UpdatedTime *time.Time `json:"updated_time,omitempty"` } // StorageCleanupReport defines model for StorageCleanupReport. type StorageCleanupReport struct { - Findings *[]StorageCleanupFinding `json:"findings,omitempty"` - Organization *string `json:"organization,omitempty"` - PathPrefix *string `json:"path_prefix,omitempty"` - Project *string `json:"project,omitempty"` - Scanned *int `json:"scanned,omitempty"` - Summary *map[string]int `json:"summary,omitempty"` + ClassifiedPaths *int `json:"classified_paths,omitempty"` + Findings *[]StorageCleanupFinding `json:"findings,omitempty"` + Organization *string `json:"organization,omitempty"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project *string `json:"project,omitempty"` + Scanned *int `json:"scanned,omitempty"` + ScannedPaths *int `json:"scanned_paths,omitempty"` + Summary *map[string]int `json:"summary,omitempty"` + UnclassifiedPaths *int `json:"unclassified_paths,omitempty"` } // StorageCleanupSkipped defines model for StorageCleanupSkipped. diff --git a/apigen/openapi/internal.openapi.yaml b/apigen/openapi/internal.openapi.yaml index 522a5a94..4a996744 100644 --- a/apigen/openapi/internal.openapi.yaml +++ b/apigen/openapi/internal.openapi.yaml @@ -877,6 +877,12 @@ components: type: string scanned: type: integer + scanned_paths: + type: integer + classified_paths: + type: integer + unclassified_paths: + type: integer summary: type: object additionalProperties: @@ -900,10 +906,33 @@ components: type: string repo_delete_candidate: type: boolean + cleanup_scope: + type: string + structural_reason: + type: string + checksum_count: + type: integer + size_count: + type: integer + legacy_url_template_detected: + type: boolean records: type: array items: $ref: '#/components/schemas/StorageCleanupRecordAudit' + StorageCleanupAccessProbe: + type: object + properties: + url: + type: string + bucket: + type: string + storage_status: + type: string + storage_message: + type: string + error_kind: + type: string StorageCleanupRecordAudit: type: object properties: @@ -927,6 +956,12 @@ components: type: array items: type: string + access_probes: + type: array + items: + $ref: '#/components/schemas/StorageCleanupAccessProbe' + cleanup_scope: + type: string storage_status: type: string storage_message: diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 79b0e569..4f3df1ce 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -238,36 +238,55 @@ type StorageCleanupAuditRequest struct { // StorageCleanupFinding defines model for StorageCleanupFinding. type StorageCleanupFinding struct { - Kind *string `json:"kind,omitempty"` - Message *string `json:"message,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - RecommendedAction *string `json:"recommended_action,omitempty"` - Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` - RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` - Severity *string `json:"severity,omitempty"` + CleanupScope *string `json:"cleanup_scope,omitempty"` + ChecksumCount *int `json:"checksum_count,omitempty"` + Kind *string `json:"kind,omitempty"` + LegacyUrlTemplateDetected *bool `json:"legacy_url_template_detected,omitempty"` + Message *string `json:"message,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + RecommendedAction *string `json:"recommended_action,omitempty"` + Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` + RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` + SizeCount *int `json:"size_count,omitempty"` + StructuralReason *string `json:"structural_reason,omitempty"` + Severity *string `json:"severity,omitempty"` +} + +// StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. +type StorageCleanupAccessProbe struct { + Bucket *string `json:"bucket,omitempty"` + ErrorKind *string `json:"error_kind,omitempty"` + StorageMessage *string `json:"storage_message,omitempty"` + StorageStatus *string `json:"storage_status,omitempty"` + Url *string `json:"url,omitempty"` } // StorageCleanupRecordAudit defines model for StorageCleanupRecordAudit. type StorageCleanupRecordAudit struct { - CurrentAccessUrls *[]string `json:"current_access_urls,omitempty"` - DownloadCount *int64 `json:"download_count,omitempty"` - LastDownloadTime *time.Time `json:"last_download_time,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - ObjectId *string `json:"object_id,omitempty"` - Size *int64 `json:"size,omitempty"` - StorageMessage *string `json:"storage_message,omitempty"` - StorageStatus *string `json:"storage_status,omitempty"` - UpdatedTime *time.Time `json:"updated_time,omitempty"` + AccessProbes *[]StorageCleanupAccessProbe `json:"access_probes,omitempty"` + CleanupScope *string `json:"cleanup_scope,omitempty"` + CurrentAccessUrls *[]string `json:"current_access_urls,omitempty"` + DownloadCount *int64 `json:"download_count,omitempty"` + LastDownloadTime *time.Time `json:"last_download_time,omitempty"` + NormalizedPath *string `json:"normalized_path,omitempty"` + ObjectId *string `json:"object_id,omitempty"` + Size *int64 `json:"size,omitempty"` + StorageMessage *string `json:"storage_message,omitempty"` + StorageStatus *string `json:"storage_status,omitempty"` + UpdatedTime *time.Time `json:"updated_time,omitempty"` } // StorageCleanupReport defines model for StorageCleanupReport. type StorageCleanupReport struct { - Findings *[]StorageCleanupFinding `json:"findings,omitempty"` - Organization *string `json:"organization,omitempty"` - PathPrefix *string `json:"path_prefix,omitempty"` - Project *string `json:"project,omitempty"` - Scanned *int `json:"scanned,omitempty"` - Summary *map[string]int `json:"summary,omitempty"` + ClassifiedPaths *int `json:"classified_paths,omitempty"` + Findings *[]StorageCleanupFinding `json:"findings,omitempty"` + Organization *string `json:"organization,omitempty"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project *string `json:"project,omitempty"` + Scanned *int `json:"scanned,omitempty"` + ScannedPaths *int `json:"scanned_paths,omitempty"` + Summary *map[string]int `json:"summary,omitempty"` + UnclassifiedPaths *int `json:"unclassified_paths,omitempty"` } // StorageCleanupSkipped defines model for StorageCleanupSkipped. diff --git a/client/services/repair_service.go b/client/services/repair_service.go index eea26218..6e5bcd9d 100644 --- a/client/services/repair_service.go +++ b/client/services/repair_service.go @@ -17,12 +17,27 @@ func NewRepairService(r request.Requester) *RepairService { return &RepairService{requestor: r} } +func (s *RepairService) ProjectDiffAudit(ctx context.Context, opts ProjectDiffAuditOptions) (repair.ProjectDiffReport, error) { + req := repair.ProjectDiffAuditRequest{ + Organization: opts.Organization, + Project: opts.ProjectID, + PathPrefix: opts.PathPrefix, + ExpectedPaths: append([]string(nil), opts.ExpectedPaths...), + } + var out repair.ProjectDiffReport + if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairProjectDiff, req, &out); err != nil { + return repair.ProjectDiffReport{}, err + } + return out, nil +} + func (s *RepairService) StorageCleanupAudit(ctx context.Context, opts StorageCleanupAuditOptions) (repair.StorageCleanupReport, error) { req := repair.StorageCleanupAuditRequest{ Organization: opts.Organization, Project: opts.ProjectID, PathPrefix: opts.PathPrefix, ExpectedPaths: append([]string(nil), opts.ExpectedPaths...), + SelectedPaths: append([]string(nil), opts.SelectedPaths...), CheckStorage: opts.CheckStorage, } var out repair.StorageCleanupReport diff --git a/client/services/types.go b/client/services/types.go index 09c06cb6..5e5cb81f 100644 --- a/client/services/types.go +++ b/client/services/types.go @@ -86,9 +86,17 @@ type StorageCleanupAuditOptions struct { ProjectID string PathPrefix string ExpectedPaths []string + SelectedPaths []string CheckStorage *bool } +type ProjectDiffAuditOptions struct { + Organization string + ProjectID string + PathPrefix string + ExpectedPaths []string +} + type StorageCleanupApplyOptions struct { Organization string ProjectID string diff --git a/cmd/repair/main.go b/cmd/repair/main.go new file mode 100644 index 00000000..2da77ad7 --- /dev/null +++ b/cmd/repair/main.go @@ -0,0 +1,306 @@ +package repair + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + syclient "github.com/calypr/syfon/client" + "github.com/calypr/syfon/client/services" + "github.com/calypr/syfon/cmd/cliauth" + sycommon "github.com/calypr/syfon/internal/common" + repairsvc "github.com/calypr/syfon/internal/repair" + "github.com/spf13/cobra" +) + +type auditRunner func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error +type applyRunner func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error + +type commandOptions struct { + Organization string + Project string + PathPrefix string + ExpectedPathsFile string + RepoRoot string + CheckStorage bool + Format string + DeleteStaleDuplicates bool + DeleteRepoOrphans bool + DryRun bool + SelectedPaths []string + SelectedObjectIDs []string + SelectedFindingKinds []string +} + +var Cmd = newCommand(runAudit, runApply) + +func newCommand(audit auditRunner, apply applyRunner) *cobra.Command { + var opts commandOptions + cmd := &cobra.Command{ + Use: "repair", + Short: "Audit and apply Syfon storage cleanup actions", + } + addSharedFlags(cmd, &opts) + cmd.AddCommand(&cobra.Command{ + Use: "audit", + Short: "Audit duplicate and orphaned storage paths", + RunE: func(cmd *cobra.Command, args []string) error { + req, err := normalizedAuditOptions(opts) + if err != nil { + return err + } + return audit(cmd, req) + }, + }) + cmd.AddCommand(&cobra.Command{ + Use: "apply", + Short: "Apply storage cleanup actions", + PreRunE: func(cmd *cobra.Command, args []string) error { + req, err := normalizedApplyOptions(opts) + if err != nil { + return err + } + if req.Organization == "" || req.Project == "" { + return fmt.Errorf("repair apply requires --organization and --project") + } + if !req.DeleteStaleDuplicates && !req.DeleteRepoOrphans { + return fmt.Errorf("repair apply requires --delete-stale-duplicates and/or --delete-repo-orphans") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + req, err := normalizedApplyOptions(opts) + if err != nil { + return err + } + return apply(cmd, req) + }, + }) + return cmd +} + +func addSharedFlags(cmd *cobra.Command, opts *commandOptions) { + cmd.PersistentFlags().StringVar(&opts.Organization, "organization", "", "Organization to scope the cleanup") + cmd.PersistentFlags().StringVar(&opts.Project, "project", "", "Project to scope the cleanup") + cmd.PersistentFlags().StringVar(&opts.PathPrefix, "path-prefix", "", "Optional path prefix under the project") + cmd.PersistentFlags().StringVar(&opts.ExpectedPathsFile, "expected-paths-file", "", "File containing expected normalized paths (newline-delimited or JSON array)") + cmd.PersistentFlags().StringVar(&opts.RepoRoot, "repo-root", "", "Local checkout root to walk for expected paths") + cmd.PersistentFlags().BoolVar(&opts.CheckStorage, "check-storage", true, "Probe storage object existence for candidate records") + cmd.PersistentFlags().StringVar(&opts.Format, "format", "text", "Output format: text or json") + cmd.PersistentFlags().BoolVar(&opts.DeleteStaleDuplicates, "delete-stale-duplicates", false, "Delete stale duplicate Syfon records") + cmd.PersistentFlags().BoolVar(&opts.DeleteRepoOrphans, "delete-repo-orphans", false, "Delete Syfon records and purge storage for repo-orphaned live objects") + cmd.PersistentFlags().BoolVar(&opts.DryRun, "dry-run", false, "Report actions without deleting records") + cmd.PersistentFlags().StringSliceVar(&opts.SelectedPaths, "selected-path", nil, "Limit apply to one or more normalized paths") + cmd.PersistentFlags().StringSliceVar(&opts.SelectedObjectIDs, "selected-object-id", nil, "Limit apply to one or more Syfon object IDs") + cmd.PersistentFlags().StringSliceVar(&opts.SelectedFindingKinds, "selected-finding-kind", nil, "Limit apply to one or more finding kinds") +} + +func normalizedAuditOptions(opts commandOptions) (repairsvc.StorageCleanupAuditRequest, error) { + expected, err := loadExpectedPaths(opts.ExpectedPathsFile, opts.RepoRoot) + if err != nil { + return repairsvc.StorageCleanupAuditRequest{}, err + } + checkStorage := opts.CheckStorage + return repairsvc.StorageCleanupAuditRequest{ + Organization: strings.TrimSpace(opts.Organization), + Project: strings.TrimSpace(opts.Project), + PathPrefix: strings.TrimSpace(opts.PathPrefix), + ExpectedPaths: expected, + CheckStorage: &checkStorage, + }, nil +} + +func normalizedApplyOptions(opts commandOptions) (repairsvc.StorageCleanupApplyRequest, error) { + expected, err := loadExpectedPaths(opts.ExpectedPathsFile, opts.RepoRoot) + if err != nil { + return repairsvc.StorageCleanupApplyRequest{}, err + } + checkStorage := opts.CheckStorage + kinds := make([]repairsvc.FindingKind, 0, len(opts.SelectedFindingKinds)) + for _, kind := range opts.SelectedFindingKinds { + kinds = append(kinds, repairsvc.FindingKind(strings.TrimSpace(kind))) + } + return repairsvc.StorageCleanupApplyRequest{ + Organization: strings.TrimSpace(opts.Organization), + Project: strings.TrimSpace(opts.Project), + PathPrefix: strings.TrimSpace(opts.PathPrefix), + ExpectedPaths: expected, + DeleteStaleDuplicates: opts.DeleteStaleDuplicates, + DeleteRepoOrphans: opts.DeleteRepoOrphans, + DryRun: opts.DryRun, + SelectedPaths: trimNonEmpty(opts.SelectedPaths), + SelectedObjectIDs: trimNonEmpty(opts.SelectedObjectIDs), + SelectedFindingKinds: kinds, + CheckStorage: &checkStorage, + }, nil +} + +func runAudit(cmd *cobra.Command, req repairsvc.StorageCleanupAuditRequest) error { + client, err := buildClient(cmd) + if err != nil { + return err + } + report, err := client.Repair().StorageCleanupAudit(cmd.Context(), servicesAuditOptions(req)) + if err != nil { + return err + } + return repairsvc.WriteStorageCleanupAudit(cmd.OutOrStdout(), normalizedFormat(cmd), report) +} + +func runApply(cmd *cobra.Command, req repairsvc.StorageCleanupApplyRequest) error { + client, err := buildClient(cmd) + if err != nil { + return err + } + result, err := client.Repair().StorageCleanupApply(cmd.Context(), servicesApplyOptions(req)) + if err != nil { + return err + } + return repairsvc.WriteStorageCleanupApply(cmd.OutOrStdout(), normalizedFormat(cmd), result) +} + +func buildClient(cmd *cobra.Command) (*syclient.Client, error) { + client, err := cliauth.NewServerClient(cmd) + if err != nil { + return nil, err + } + concrete, ok := client.(*syclient.Client) + if !ok { + return nil, fmt.Errorf("unexpected syfon client type %T", client) + } + return concrete, nil +} + +func normalizedFormat(cmd *cobra.Command) string { + format, _ := cmd.Flags().GetString("format") + format = strings.ToLower(strings.TrimSpace(format)) + if format == "" { + return "text" + } + return format +} + +func servicesAuditOptions(req repairsvc.StorageCleanupAuditRequest) services.StorageCleanupAuditOptions { + return services.StorageCleanupAuditOptions{ + Organization: req.Organization, + ProjectID: req.Project, + PathPrefix: req.PathPrefix, + ExpectedPaths: append([]string(nil), req.ExpectedPaths...), + CheckStorage: req.CheckStorage, + } +} + +func servicesApplyOptions(req repairsvc.StorageCleanupApplyRequest) services.StorageCleanupApplyOptions { + kinds := make([]string, 0, len(req.SelectedFindingKinds)) + for _, kind := range req.SelectedFindingKinds { + kinds = append(kinds, string(kind)) + } + return services.StorageCleanupApplyOptions{ + Organization: req.Organization, + ProjectID: req.Project, + PathPrefix: req.PathPrefix, + ExpectedPaths: append([]string(nil), req.ExpectedPaths...), + DeleteStaleDuplicates: req.DeleteStaleDuplicates, + DeleteRepoOrphans: req.DeleteRepoOrphans, + DryRun: req.DryRun, + SelectedPaths: append([]string(nil), req.SelectedPaths...), + SelectedObjectIDs: append([]string(nil), req.SelectedObjectIDs...), + SelectedFindingKinds: kinds, + CheckStorage: req.CheckStorage, + } +} + +func loadExpectedPaths(filePath, repoRoot string) ([]string, error) { + filePath = strings.TrimSpace(filePath) + repoRoot = strings.TrimSpace(repoRoot) + if filePath != "" && repoRoot != "" { + return nil, fmt.Errorf("--expected-paths-file and --repo-root are mutually exclusive") + } + switch { + case filePath != "": + return loadExpectedPathsFile(filePath) + case repoRoot != "": + return walkExpectedPaths(repoRoot) + default: + return nil, nil + } +} + +func loadExpectedPathsFile(path string) ([]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var raw []string + if json.Unmarshal(data, &raw) == nil { + return normalizeManifestPaths(raw) + } + lines := strings.Split(string(data), "\n") + return normalizeManifestPaths(lines) +} + +func walkExpectedPaths(root string) ([]string, error) { + root, err := filepath.Abs(root) + if err != nil { + return nil, err + } + out := make([]string, 0, 256) + err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == ".git" { + return filepath.SkipDir + } + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + normalized := filepath.ToSlash(rel) + out = append(out, normalized) + return nil + }) + if err != nil { + return nil, err + } + return normalizeManifestPaths(out) +} + +func normalizeManifestPaths(values []string) ([]string, error) { + out := make([]string, 0, len(values)) + seen := map[string]bool{} + for _, raw := range values { + raw = strings.TrimSpace(raw) + if raw == "" { + continue + } + normalized, _, err := sycommon.NormalizeBrowsePath(raw) + if err != nil { + return nil, err + } + if seen[normalized] { + continue + } + seen[normalized] = true + out = append(out, normalized) + } + sort.Strings(out) + return out, nil +} + +func trimNonEmpty(values []string) []string { + out := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + out = append(out, value) + } + } + return out +} diff --git a/cmd/repair/main_test.go b/cmd/repair/main_test.go new file mode 100644 index 00000000..19b99872 --- /dev/null +++ b/cmd/repair/main_test.go @@ -0,0 +1,53 @@ +package repair + +import ( + "testing" + + repairsvc "github.com/calypr/syfon/internal/repair" + "github.com/spf13/cobra" +) + +func TestApplyRequiresScope(t *testing.T) { + cmd := newCommand( + func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, + ) + cmd.SetArgs([]string{"apply", "--delete-stale-duplicates"}) + err := cmd.Execute() + if err == nil || err.Error() != "repair apply requires --organization and --project" { + t.Fatalf("expected scope validation error, got %v", err) + } +} + +func TestAuditCollectsCheckStorageFlag(t *testing.T) { + called := false + cmd := newCommand( + func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { + called = true + if opts.CheckStorage == nil || !*opts.CheckStorage { + t.Fatal("expected check-storage true") + } + return nil + }, + func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, + ) + cmd.SetArgs([]string{"audit", "--check-storage"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("audit failed: %v", err) + } + if !called { + t.Fatal("expected audit runner call") + } +} + +func TestApplyRequiresDeleteMode(t *testing.T) { + cmd := newCommand( + func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, + ) + cmd.SetArgs([]string{"apply", "--organization", "org", "--project", "proj"}) + err := cmd.Execute() + if err == nil || err.Error() != "repair apply requires --delete-stale-duplicates and/or --delete-repo-orphans" { + t.Fatalf("expected delete mode validation error, got %v", err) + } +} diff --git a/docs/frontend-handoff-storage-metrics.md b/docs/frontend-handoff-storage-metrics.md new file mode 100644 index 00000000..c2599532 --- /dev/null +++ b/docs/frontend-handoff-storage-metrics.md @@ -0,0 +1,272 @@ +# Frontend Handoff: Syfon Path-Level Storage Metrics + +This note describes the backend storage-metrics work that was just added in +`syfon` so the frontend can stop recursively walking `/index` and computing +directory totals client-side. + +## What Was Added + +Two new metrics endpoints now exist under the metrics namespace: + +1. `GET /index/v1/metrics/storage/summary` +2. `GET /index/v1/metrics/storage/children` + +These are project-scoped path metrics backed by Syfon’s existing +`drs_object_browse_index` path model plus `drs_object.size` and +`drs_object.updated_time`. + +The important part: the backend is reusing the same normalized browse-path +semantics as the existing path browsing queries, not inventing a second tree +definition. + +## Why Frontend Should Use This + +Old behavior: + +- page through all `/index` results for a project +- rebuild a directory tree in the client +- sum bytes and counts locally + +New behavior: + +- ask Syfon for one path summary +- ask Syfon for direct children of the current path +- render the result directly + +This removes: + +- full-project pagination just to render a folder row +- repeated client aggregation work +- large response payloads for simple summary views + +## Endpoint 1: Storage Summary + +Route: + +`GET /index/v1/metrics/storage/summary` + +Required query params: + +- `organization` +- `project` + +Optional query params: + +- `path` + +If `path` is omitted, the summary is for the project root. + +Example: + +```http +GET /index/v1/metrics/storage/summary?organization=cbds&project=end_to_end_test&path=data +``` + +Response shape: + +```json +{ + "organization": "cbds", + "project": "end_to_end_test", + "path": "data", + "file_count": 2, + "record_count": 3, + "total_bytes": 60, + "direct_child_count": 2, + "duplicate_path_count": 1, + "latest_update_time": "2026-06-24T22:00:00Z" +} +``` + +Field meanings: + +- `file_count`: distinct normalized file paths under the subtree +- `record_count`: total indexed records under the subtree +- `total_bytes`: sum of `drs_object.size` for all matching records +- `direct_child_count`: number of immediate children under the requested path +- `duplicate_path_count`: count of normalized file paths with more than one indexed record +- `latest_update_time`: max `drs_object.updated_time` seen in the subtree + +## Endpoint 2: Storage Children + +Route: + +`GET /index/v1/metrics/storage/children` + +Required query params: + +- `organization` +- `project` + +Optional query params: + +- `path` +- `limit` +- `offset` +- `sort_by` +- `sort_order` + +Valid `sort_by` values: + +- `name` +- `bytes` +- `updated_time` +- `records` + +Valid `sort_order` values: + +- `asc` +- `desc` + +Example: + +```http +GET /index/v1/metrics/storage/children?organization=cbds&project=end_to_end_test&path=data&sort_by=bytes&sort_order=desc +``` + +Response shape: + +```json +{ + "organization": "cbds", + "project": "end_to_end_test", + "path": "data", + "items": [ + { + "name": "nested", + "path": "data/nested", + "type": "directory", + "file_count": 1, + "record_count": 1, + "total_bytes": 30, + "latest_update_time": "2026-06-24T22:00:00Z" + }, + { + "name": "a.txt", + "path": "data/a.txt", + "type": "file", + "file_count": 1, + "record_count": 2, + "total_bytes": 30, + "latest_update_time": "2026-06-24T22:00:00Z" + } + ] +} +``` + +Field meanings: + +- `name`: direct child name only +- `path`: normalized full child path +- `type`: `file` or `directory` +- `file_count`: distinct normalized file paths represented by that child +- `record_count`: total records represented by that child +- `total_bytes`: summed `drs_object.size` +- `latest_update_time`: max update time across that child subtree + +## Path Semantics + +These endpoints intentionally follow Syfon’s existing browse-index path logic. + +Important rules: + +- path normalization strips duplicate `/` +- backslashes are normalized to `/` +- empty path means root +- `.` and `..` are rejected +- path aggregation is driven by the existing browse index + +So the frontend should treat returned `path` values as canonical and should not +try to re-normalize them differently. + +## Duplicate Records + +The backend does not pretend duplicate-path records are one file. + +If multiple indexed objects resolve to the same normalized path: + +- all contribute to `record_count` +- all contribute to `total_bytes` +- `file_count` still counts the distinct path once +- duplicate occupancy is surfaced through `duplicate_path_count` + +Frontend implication: + +- if `record_count > file_count`, there are duplicate indexed records inside that subtree +- if a child row has `record_count > file_count`, that child has duplicate path occupancy + +## Auth / Scope Requirements + +These routes are not global rollups. + +They currently require a concrete project scope: + +- `organization` must be set +- `project` must be set + +Do not call them for org-wide rollups yet. + +## Suggested Frontend Wiring + +For a file explorer/tree view: + +1. On entering a project root, call `/metrics/storage/summary` with no `path`. +2. Also call `/metrics/storage/children` with no `path`. +3. On clicking a directory, call both endpoints again with that directory `path`. +4. Use `summary.direct_child_count` for quick folder metadata if useful. +5. Use `children.items` to render rows without reconstructing the subtree client-side. + +For a summary card: + +- use `/metrics/storage/summary` +- render `file_count`, `record_count`, `total_bytes`, and optionally `duplicate_path_count` + +## Recommended UI Interpretation + +For a directory row: + +- primary size: `total_bytes` +- secondary count: `file_count` +- warning state if `record_count > file_count` + +For a file row: + +- size: `total_bytes` +- warning if `record_count > 1` + +## What This Does Not Do + +This is not a precomputed materialized quota table. + +It is still query-time aggregation, but it is much cheaper than loading every +project record into the frontend and aggregating there. + +This also does not add: + +- org-wide grouped rollups +- deletion/remediation workflows +- duplicate cleanup + +## Backend Files Touched + +Useful reference points if the other agent wants to inspect behavior: + +- `apigen/openapi/metrics.openapi.yaml` +- `internal/api/metrics/storage_summary.go` +- `internal/storagemetrics/storage_metrics.go` +- `internal/db/sqlite/metrics.go` +- `internal/db/postgres/metrics.go` +- `client/services/metrics_service.go` + +## Short Pasteable Summary + +Use Syfon’s new endpoints instead of recursively traversing `/index`: + +- `GET /index/v1/metrics/storage/summary?organization=ORG&project=PROJ&path=OPTIONAL` +- `GET /index/v1/metrics/storage/children?organization=ORG&project=PROJ&path=OPTIONAL&sort_by=name|bytes|updated_time|records&sort_order=asc|desc&limit=N&offset=N` + +These routes reuse the same normalized browse-index path semantics as existing +path browsing. `total_bytes` is summed from `drs_object.size`. `record_count` +counts all matching indexed records. `file_count` counts distinct normalized +paths. Duplicate path occupancy shows up when `record_count > file_count`, and +the summary also exposes `duplicate_path_count`. diff --git a/docs/implementation-plan-storage-summary.md b/docs/implementation-plan-storage-summary.md new file mode 100644 index 00000000..7a41fd9b --- /dev/null +++ b/docs/implementation-plan-storage-summary.md @@ -0,0 +1,414 @@ +# Implementation Plan: Path-Level Storage Summary Metrics + +## Goal + +Add native Syfon metrics endpoints for storage summaries so clients can ask for: + +- directory size under `{organization, project, path}` +- direct-child breakdown for breadcrumb/tree navigation +- project-level totals without traversing every `/index` page +- organization/project-scoped storage metrics with documented semantics + +This plan replaces the current frontend workaround that recursively walks +`/index` and computes aggregates client-side. + +## Why This Is Needed + +Current metrics support in Syfon is real, but it is focused on: + +- per-object file usage counts +- transfer attribution and provider-observed transfer events +- scoped usage summaries by object + +It does **not** currently provide: + +- path-level storage rollups +- directory summaries by prefix +- direct-child storage listings for a tree UI +- a metrics model that returns bytes grouped by canonical path + +That gap forces the frontend to re-enumerate the object index and rebuild a +directory tree on every request, which is the wrong ownership boundary. + +## Scope + +This plan covers new metrics read APIs and the storage-side aggregation needed +to power them. + +This plan does **not** include: + +- bucket/object deletion workflows +- automatic duplicate-object remediation +- quota enforcement +- billing reconciliation changes beyond reusing existing file size/object scope data + +## Required Product Behaviors + +The resulting API should support these use cases efficiently: + +1. Project summary card: + - total files + - total bytes + - total distinct direct children at the current path + +2. Breadcrumb/tree view: + - list direct children under a path + - show each child type (`file` or `directory`) + - show aggregate bytes for each child + - show file coverage count for each child + - show latest update timestamp for each child + +3. Organization/project summary views: + - project totals without full `/index` traversal + - optional grouped rollups for organization-wide reporting later + +## Canonical Data Semantics + +These must be fixed before implementation to avoid frontend/backend drift. + +### Path identity + +Use a single canonical path field for storage-tree aggregation. + +Recommended rule: + +- canonical path = normalized `file_name` when present +- fallback = normalized `name` only when `file_name` is empty +- if neither is a usable path, exclude from path metrics and surface in logs/diagnostics + +Rationale: + +- `file_name` is already the closest thing to path-like user-facing placement +- `name` is too ambiguous to be the primary tree key +- access URLs should not define logical browse paths + +### Size semantics + +Use the indexed object size already stored in Syfon metadata. + +- `total_bytes` = sum of `drs_object.size` for objects included in the filtered scope +- do not call cloud storage to compute live bucket bytes for interactive reads + +Rationale: + +- consistent with existing Syfon object metadata +- cheap to query +- avoids coupling UI latency to provider APIs + +### Object/version semantics + +Count every indexed object that is currently visible in the scope/path query. + +That means: + +- if two distinct objects resolve to the same canonical path, both contribute +- duplicate-path objects should be visible as `record_count > 1` +- deletion/supersession semantics remain whatever the object index currently exposes + +Rationale: + +- this is a storage summary of Syfon’s indexed state, not an inferred “clean file tree” +- remediation decisions should happen after the API can prove duplicate path occupancy + +## Proposed API + +Add a new storage-summary family under the metrics namespace. + +### 1. Path summary + +`GET /index/v1/metrics/storage/summary` + +Query params: + +- `organization` required +- `project` required +- `path` optional, default root + +Response shape: + +```json +{ + "organization": "Ellrott_Lab", + "project": "embedding_rotation", + "path": "data", + "file_count": 6, + "record_count": 6, + "total_bytes": 50497716224, + "direct_child_count": 6, + "duplicate_path_count": 3, + "latest_update_time": "2026-05-27T22:00:00Z" +} +``` + +### 2. Path children + +`GET /index/v1/metrics/storage/children` + +Query params: + +- `organization` required +- `project` required +- `path` optional, default root +- `limit` optional +- `offset` optional +- `sort_by` optional: `bytes`, `name`, `updated_time`, `records` +- `sort_order` optional: `asc`, `desc` + +Response shape: + +```json +{ + "organization": "Ellrott_Lab", + "project": "embedding_rotation", + "path": "data", + "items": [ + { + "name": "tcga_HM450.hdf5", + "path": "data/tcga_HM450.hdf5", + "type": "file", + "file_count": 1, + "record_count": 2, + "total_bytes": 39943195852, + "latest_update_time": "2026-05-27T22:00:00Z" + }, + { + "name": "nested", + "path": "data/nested", + "type": "directory", + "file_count": 42, + "record_count": 42, + "total_bytes": 90123123, + "latest_update_time": "2026-05-20T18:14:00Z" + } + ] +} +``` + +### 3. Optional later: grouped project rollup + +`GET /index/v1/metrics/storage/projects` + +This is useful later for explorer summary tables, but it is not necessary to +replace the FileSummary workaround. Keep it out of phase 1 unless it is nearly +free once the query layer exists. + +## Backend Design + +### New model types + +Add new metrics models in `internal/models`, for example: + +- `StoragePathSummary` +- `StoragePathChild` +- `StoragePathChildrenResponse` + +Fields should include: + +- scope: `organization`, `project`, `path` +- counts: `file_count`, `record_count`, `direct_child_count`, `duplicate_path_count` +- bytes: `total_bytes` +- timestamps: `latest_update_time` + +### New DB interface + +Extend `internal/db/interface.go` with a dedicated storage metrics contract. + +Suggested interface: + +```go +type StorageMetricsStore interface { + GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) + ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) +} +``` + +This should remain separate from `FileUsageScopedLister`. + +Reason: + +- file usage metrics answer “how often was this object used?” +- storage summary metrics answer “how much indexed data is under this logical path?” + +Those are different dimensions and should not be conflated. + +### Query strategy + +Do the aggregation in SQL against indexed metadata, not by hydrating every +object through `ObjectManager`. + +The query layer needs to: + +1. resolve visible objects in `{organization, project}` +2. normalize canonical browse path +3. filter to the requested prefix +4. compute: + - direct child name + - whether the child is a file or directory + - aggregate bytes + - file count + - record count + - latest update time + +### Postgres / SQLite parity + +Implement in both: + +- `internal/db/postgres/metrics.go` +- `internal/db/sqlite/metrics.go` + +Keep one behavioral contract and verify with matching tests. + +## Normalization Rules + +Path normalization should be shared and explicit. + +Recommended helper package behavior: + +- trim whitespace +- strip leading `/` +- collapse repeated `/` +- treat empty path as root +- direct child of root: + - `foo.txt` -> file child `foo.txt` + - `dir/a.txt` -> directory child `dir` + +If a record has canonical path equal to the requested path exactly, treat it as +a file child only when querying the parent directory. Do not let a path +summarize itself as its own descendant. + +## Authorization + +Use the same auth pattern as the existing metrics server: + +- require auth under Gen3 mode +- allow scoped reads only for readable resources +- require exact `organization/project` scope for path storage endpoints in phase 1 + +Do **not** implement organization-wide aggregation across arbitrary readable +scopes in the first pass. The storage monitor page already works at exact +project scope, and that keeps the SQL simpler. + +## Phased Rollout + +### Phase 1: backend path summary for exact project scope + +Deliver: + +- OpenAPI additions for: + - `GET /index/v1/metrics/storage/summary` + - `GET /index/v1/metrics/storage/children` +- generated metrics bindings via `make gen` +- new model types +- DB interface additions +- Postgres + SQLite implementations +- metrics server handlers +- unit/integration tests + +Success criterion: + +- the frontend can replace recursive `/index` traversal for a single project/path + +### Phase 2: frontend migration + +Deliver: + +- replace `FileSummary` recursive fetch logic with metrics calls +- preserve duplicate-path visibility with `record_count` +- keep `/index` only for drill-down features that need raw record identity + +Success criterion: + +- `FileSummary` no longer walks every record to compute path totals + +### Phase 3: broader rollups and diagnostics + +Optional additions: + +- organization-wide grouped project totals +- duplicate-path diagnostic endpoint +- “top N duplicate paths” for cleanup/admin views + +Success criterion: + +- explorer/admin UIs can build storage summaries without re-aggregating client-side + +## Testing Plan + +Add tests at three layers. + +### DB tests + +For both Postgres and SQLite: + +- root path summary +- nested path summary +- direct-child breakdown +- duplicate canonical path aggregation +- mixed file/directory children +- empty path result + +### API tests + +Under `internal/api/metrics`: + +- auth required / forbidden scope +- bad query validation +- summary response shape +- children response shape +- pagination and sorting + +### Server parity tests + +Update endpoint registration tests to include the new routes. + +## Risks + +### Path ambiguity + +If `file_name` is missing or inconsistent, tree metrics will be noisy. + +Mitigation: + +- define canonical path rules once +- add tests covering `file_name` vs `name` +- optionally add a diagnostic counter for records excluded from path metrics + +### Duplicate-path confusion + +Aggregating by path can hide the fact that multiple objects occupy one path. + +Mitigation: + +- return both `file_count` and `record_count` +- return `duplicate_path_count` in summaries + +### SQL complexity drift + +It is easy to accidentally reimplement the object manager in SQL. + +Mitigation: + +- keep phase 1 scoped to exact `{organization, project, path}` +- avoid organization-wide multi-scope fanout until the project-path query is stable + +## Concrete Implementation Order + +1. Add new OpenAPI paths and schemas to `apigen/openapi/metrics.openapi.yaml`. +2. Run `make gen`. +3. Add `StoragePathSummary` and `StoragePathChild` models. +4. Extend `internal/db/interface.go` with a storage metrics interface. +5. Implement SQLite queries first in `internal/db/sqlite/metrics.go`. +6. Mirror the behavior in `internal/db/postgres/metrics.go`. +7. Add metrics handlers in `internal/api/metrics`. +8. Add tests for handlers and DB parity. +9. Switch the frontend `FileSummary` page to use the new endpoints. + +## Immediate Recommendation + +Implement phase 1 only on this branch. + +That is the smallest meaningful delivery because it: + +- removes the frontend recursive `/index` workaround +- establishes the right metrics ownership boundary in Syfon +- preserves future room for duplicate-path diagnostics without coupling them to cleanup workflows diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index ac1d9f27..e595ede0 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -41,6 +41,7 @@ func RegisterInternalRoutes(router fiber.Router, om *core.ObjectManager) { router.Post(common.RouteInternalBulkCreate, handleInternalBulkCreateFiber(om)) router.Post(common.RouteInternalBulkDocs, handleInternalBulkDocumentsFiber(om)) router.Post(common.RouteInternalBulkDeleteHashes, handleInternalBulkDeleteFiber(om)) + router.Post(common.RouteInternalRepairProjectDiff, handleInternalProjectDiffAuditFiber(om)) router.Post(common.RouteInternalRepairCleanupAudit, handleInternalStorageCleanupAuditFiber(om)) router.Post(common.RouteInternalRepairCleanupApply, handleInternalStorageCleanupApplyFiber(om)) diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go index 3e73741d..543f7e58 100644 --- a/internal/api/internaldrs/repair_cleanup.go +++ b/internal/api/internaldrs/repair_cleanup.go @@ -41,6 +41,33 @@ func handleInternalStorageCleanupAuditFiber(om *core.ObjectManager) fiber.Handle } } +func handleInternalProjectDiffAuditFiber(om *core.ObjectManager) fiber.Handler { + svc := repair.NewProjectDiffService(om) + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req repair.ProjectDiffAuditRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + req.Organization = strings.TrimSpace(req.Organization) + req.Project = strings.TrimSpace(req.Project) + req.PathPrefix = strings.TrimSpace(req.PathPrefix) + if req.Organization == "" || req.Project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read"); err != nil { + return apiutil.HandleError(c, err) + } + report, err := svc.Audit(c.Context(), req) + if err != nil { + return apiutil.HandleError(c, err) + } + return c.JSON(report) + } +} + func handleInternalStorageCleanupApplyFiber(om *core.ObjectManager) fiber.Handler { svc := repair.NewStorageCleanupService(om) return func(c fiber.Ctx) error { diff --git a/internal/common/routes.go b/internal/common/routes.go index b4539251..bc26c2b7 100644 --- a/internal/common/routes.go +++ b/internal/common/routes.go @@ -16,6 +16,7 @@ const ( RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" RouteInternalRepairCleanupAudit = "/data/repair/storage-cleanup/audit" RouteInternalRepairCleanupApply = "/data/repair/storage-cleanup/apply" + RouteInternalRepairProjectDiff = "/data/repair/project-diff/audit" RouteInternalIndex = "/index" RouteInternalIndexDetail = "/index/{id}" diff --git a/internal/core/bucket_catalog.go b/internal/core/bucket_catalog.go index f511f3d6..b081acc8 100644 --- a/internal/core/bucket_catalog.go +++ b/internal/core/bucket_catalog.go @@ -58,6 +58,10 @@ func (m *ObjectManager) ListBucketScopes(ctx context.Context) ([]models.BucketSc } func (m *ObjectManager) ListVisibleBuckets(ctx context.Context) (map[string]VisibleBucket, error) { + return m.listVisibleBucketsCached(ctx) +} + +func (m *ObjectManager) listVisibleBucketsUncached(ctx context.Context) (map[string]VisibleBucket, error) { creds, err := m.db.ListS3Credentials(ctx) if err != nil { return nil, err diff --git a/internal/core/object_storage_inspect.go b/internal/core/object_storage_inspect.go index 3cdcd766..e1b95e2d 100644 --- a/internal/core/object_storage_inspect.go +++ b/internal/core/object_storage_inspect.go @@ -5,7 +5,9 @@ import ( "errors" "fmt" "path" + "sort" "strings" + "sync" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -28,15 +30,15 @@ type InspectStorageRequest struct { } type StorageObjectMetadata struct { - ObjectURL string - Provider string - Bucket string - Key string - Path string - SizeBytes int64 - MetaSHA256 string - ETag string - LastModTime time.Time + ObjectURL string + Provider string + Bucket string + Key string + Path string + SizeBytes int64 + MetaSHA256 string + ETag string + LastModTime time.Time } type StorageInspectErrorKind string @@ -55,6 +57,22 @@ type StorageInspectError struct { Message string } +var storageInspectCacheKey contextKey = "storageInspectCache" + +type storageInspectCredentialCacheEntry struct { + cred *models.S3Credential + err error +} + +type storageInspectRequestCache struct { + mu sync.Mutex + + credentials map[string]storageInspectCredentialCacheEntry + visible map[string]VisibleBucket + visibleErr error + visibleLoaded bool +} + func (e *StorageInspectError) Error() string { if e == nil { return "storage inspect failed" @@ -65,6 +83,20 @@ func (e *StorageInspectError) Error() string { return fmt.Sprintf("storage inspect failed: %s", e.Kind) } +func WithStorageInspectCache(ctx context.Context) context.Context { + if cache, _ := ctx.Value(storageInspectCacheKey).(*storageInspectRequestCache); cache != nil { + return ctx + } + return context.WithValue(ctx, storageInspectCacheKey, &storageInspectRequestCache{ + credentials: map[string]storageInspectCredentialCacheEntry{}, + }) +} + +func storageInspectCacheFromContext(ctx context.Context) *storageInspectRequestCache { + cache, _ := ctx.Value(storageInspectCacheKey).(*storageInspectRequestCache) + return cache +} + func (m *ObjectManager) SetS3ObjectInspector(fn func(context.Context, models.S3Credential, string, string) (*StorageObjectMetadata, error)) { if fn == nil { m.inspectS3Object = defaultS3ObjectInspector @@ -173,20 +205,38 @@ func (m *ObjectManager) credentialForBucket(ctx context.Context, bucket string) if bucket == "" { return nil, &StorageInspectError{Kind: StorageInspectInvalidInput, Message: "bucket is required"} } + if cache := storageInspectCacheFromContext(ctx); cache != nil { + if cred, err, ok := cache.getCredential(bucket); ok { + return cred, err + } + } if cred, err := m.db.GetS3Credential(ctx, bucket); err == nil && cred != nil { + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setCredential(bucket, cred, nil) + } return cred, nil } creds, err := m.db.ListS3Credentials(ctx) if err != nil { + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setCredential(bucket, nil, err) + } return nil, err } for _, cred := range creds { if strings.EqualFold(strings.TrimSpace(cred.Bucket), bucket) || strings.EqualFold(strings.TrimSpace(cred.CredentialID), bucket) { copy := cred + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setCredential(bucket, ©, nil) + } return ©, nil } } - return nil, &StorageInspectError{Kind: StorageInspectCredentialMissing, Message: fmt.Sprintf("no stored bucket credential found for bucket %q", bucket)} + err = &StorageInspectError{Kind: StorageInspectCredentialMissing, Message: fmt.Sprintf("no stored bucket credential found for bucket %q", bucket)} + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setCredential(bucket, nil, err) + } + return nil, err } func bucketVisibleToCaller(visible map[string]VisibleBucket, bucket string, credentialID string) bool { @@ -198,6 +248,78 @@ func bucketVisibleToCaller(visible map[string]VisibleBucket, bucket string, cred return false } +func (m *ObjectManager) listVisibleBucketsCached(ctx context.Context) (map[string]VisibleBucket, error) { + if cache := storageInspectCacheFromContext(ctx); cache != nil { + if visible, err, ok := cache.getVisible(); ok { + return visible, err + } + } + visible, err := m.listVisibleBucketsUncached(ctx) + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setVisible(visible, err) + } + return visible, err +} + +func (c *storageInspectRequestCache) getCredential(bucket string) (*models.S3Credential, error, bool) { + c.mu.Lock() + defer c.mu.Unlock() + entry, ok := c.credentials[strings.ToLower(strings.TrimSpace(bucket))] + if !ok { + return nil, nil, false + } + if entry.cred == nil { + return nil, entry.err, true + } + copy := *entry.cred + return ©, entry.err, true +} + +func (c *storageInspectRequestCache) setCredential(bucket string, cred *models.S3Credential, err error) { + c.mu.Lock() + defer c.mu.Unlock() + key := strings.ToLower(strings.TrimSpace(bucket)) + if cred == nil { + c.credentials[key] = storageInspectCredentialCacheEntry{err: err} + return + } + copy := *cred + c.credentials[key] = storageInspectCredentialCacheEntry{cred: ©, err: err} +} + +func (c *storageInspectRequestCache) getVisible() (map[string]VisibleBucket, error, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if !c.visibleLoaded { + return nil, nil, false + } + return cloneVisibleBuckets(c.visible), c.visibleErr, true +} + +func (c *storageInspectRequestCache) setVisible(visible map[string]VisibleBucket, err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.visible = cloneVisibleBuckets(visible) + c.visibleErr = err + c.visibleLoaded = true +} + +func cloneVisibleBuckets(in map[string]VisibleBucket) map[string]VisibleBucket { + if in == nil { + return nil + } + out := make(map[string]VisibleBucket, len(in)) + for key, bucket := range in { + programs := append([]string(nil), bucket.Programs...) + sort.Strings(programs) + out[key] = VisibleBucket{ + Credential: bucket.Credential, + Programs: programs, + } + } + return out +} + func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, bucket string, key string) (*StorageObjectMetadata, error) { loadOpts := []func(*awsconfig.LoadOptions) error{ awsconfig.WithRegion(cred.Region), diff --git a/internal/core/service_test.go b/internal/core/service_test.go index ab985d54..75402911 100644 --- a/internal/core/service_test.go +++ b/internal/core/service_test.go @@ -17,8 +17,10 @@ import ( type coreTestDB struct { *testutils.MockDatabase - aliases map[string]string - creds []models.S3Credential + aliases map[string]string + creds []models.S3Credential + getS3CredentialCalls int + listS3CredentialsCalls int } func (d *coreTestDB) ResolveObjectAlias(ctx context.Context, aliasID string) (string, error) { @@ -31,6 +33,7 @@ func (d *coreTestDB) ResolveObjectAlias(ctx context.Context, aliasID string) (st } func (d *coreTestDB) ListS3Credentials(ctx context.Context) ([]models.S3Credential, error) { + d.listS3CredentialsCalls++ if d.creds != nil { out := make([]models.S3Credential, len(d.creds)) copy(out, d.creds) @@ -42,6 +45,14 @@ func (d *coreTestDB) ListS3Credentials(ctx context.Context) ([]models.S3Credenti return d.MockDatabase.ListS3Credentials(ctx) } +func (d *coreTestDB) GetS3Credential(ctx context.Context, bucket string) (*models.S3Credential, error) { + d.getS3CredentialCalls++ + if d.MockDatabase == nil { + return nil, nil + } + return d.MockDatabase.GetS3Credential(ctx, bucket) +} + type capturingURLManager struct { signURLBucket string signURLAccessURL string @@ -979,4 +990,60 @@ func TestObjectManagerDeleteResolveAndSignDelegation(t *testing.T) { t.Fatalf("expected scoped storage url %q, got %q", want, um.signURLAccessURL) } }) + + t.Run("storage inspect cache reuses bucket credential and visibility lookups", func(t *testing.T) { + db := &coreTestDB{ + MockDatabase: &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "gdcdata": { + CredentialID: "gdcdata", + Bucket: "gdcdata", + Provider: "s3", + Region: "us-east-1", + AccessKey: "test-key", + SecretKey: "test-secret", + }, + }, + Objects: map[string]*drs.DrsObject{ + "obj-1": { + Id: "obj-1", + AccessMethods: &[]drs.AccessMethod{ + { + Type: drs.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{ + Url: "s3://gdcdata/existing/path-a", + }, + }, + }, + }, + }, + }, + } + om := NewObjectManager(db, &capturingURLManager{}) + om.SetS3ObjectInspector(func(ctx context.Context, cred models.S3Credential, bucket string, key string) (*StorageObjectMetadata, error) { + return &StorageObjectMetadata{ + ObjectURL: "s3://" + bucket + "/" + key, + Bucket: bucket, + Key: key, + }, nil + }) + + ctx := WithStorageInspectCache(context.Background()) + if _, err := om.InspectStorageObject(ctx, InspectStorageRequest{ObjectURL: "s3://gdcdata/probe/path-a"}); err != nil { + t.Fatalf("first InspectStorageObject failed: %v", err) + } + if _, err := om.InspectStorageObject(ctx, InspectStorageRequest{ObjectURL: "s3://gdcdata/probe/path-b"}); err != nil { + t.Fatalf("second InspectStorageObject failed: %v", err) + } + + if db.getS3CredentialCalls != 1 { + t.Fatalf("expected one GetS3Credential call, got %d", db.getS3CredentialCalls) + } + if db.listS3CredentialsCalls != 1 { + t.Fatalf("expected one ListS3Credentials call, got %d", db.listS3CredentialsCalls) + } + }) } diff --git a/internal/core/storage_cleanup.go b/internal/core/storage_cleanup.go index 47abce89..a0ae3e71 100644 --- a/internal/core/storage_cleanup.go +++ b/internal/core/storage_cleanup.go @@ -13,6 +13,14 @@ import ( ) func (m *ObjectManager) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + return m.listStorageCleanupRecords(ctx, organization, project, pathPrefix, false) +} + +func (m *ObjectManager) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + return m.listStorageCleanupRecords(ctx, organization, project, pathPrefix, true) +} + +func (m *ObjectManager) listStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string, duplicatesOnly bool) ([]models.StorageCleanupRecord, error) { store, ok := m.db.(db.StorageMetricsStore) if !ok { return nil, fmt.Errorf("storage cleanup listing not supported by database") @@ -34,5 +42,8 @@ func (m *ObjectManager) ListStorageCleanupRecords(ctx context.Context, organizat return nil, common.ErrUnauthorized } } + if duplicatesOnly { + return store.ListDuplicateStorageCleanupRecords(ctx, organization, project, pathPrefix) + } return store.ListStorageCleanupRecords(ctx, organization, project, pathPrefix) } diff --git a/internal/db/interface.go b/internal/db/interface.go index fe70dad7..1670bf74 100644 --- a/internal/db/interface.go +++ b/internal/db/interface.go @@ -69,6 +69,7 @@ type StorageMetricsStore interface { GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) + ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) } type TransferAttributionScopedStore interface { @@ -143,6 +144,7 @@ type MetricsStore interface { GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) + ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) } // LFSStore is the minimum storage surface needed by the LFS API. diff --git a/internal/db/postgres/metrics.go b/internal/db/postgres/metrics.go index 5e7c66e1..7d993d74 100644 --- a/internal/db/postgres/metrics.go +++ b/internal/db/postgres/metrics.go @@ -305,7 +305,7 @@ func (db *PostgresDB) ListStorageCleanupRecords(ctx context.Context, organizatio if err != nil { return nil, err } - _, prefixSegments, err := common.NormalizeBrowsePath(pathPrefix) + where, args, err := postgresStorageCleanupScopeArgs(resource, pathPrefix, 1, "bi.normalized_path") if err != nil { return nil, err } @@ -315,9 +315,9 @@ func (db *PostgresDB) ListStorageCleanupRecords(ctx context.Context, organizatio FROM drs_object_browse_index bi JOIN drs_object o ON o.id = bi.object_id LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE bi.resource = $1 + WHERE `+where+` ORDER BY bi.normalized_path ASC, bi.object_id ASC - `, resource) + `, args...) if err != nil { return nil, err } @@ -330,14 +330,63 @@ func (db *PostgresDB) ListStorageCleanupRecords(ctx context.Context, organizatio if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { return nil, err } - info, ok, err := common.BrowsePathInfoFromName(row.NormalizedPath) - if err != nil { - return nil, err + if lastDownload.Valid { + t := lastDownload.Time.UTC() + row.LastDownloadTime = &t } - if !ok || !common.HasBrowsePathPrefix(info.Segments, prefixSegments) { - continue + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + +func (db *PostgresDB) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + if err := db.flushObjectUsageEvents(ctx); err != nil { + return nil, err + } + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return nil, err + } + cteWhere, cteArgs, err := postgresStorageCleanupScopeArgs(resource, pathPrefix, 1, "normalized_path") + if err != nil { + return nil, err + } + mainWhere, _, err := postgresStorageCleanupScopeArgs(resource, pathPrefix, 1, "bi.normalized_path") + if err != nil { + return nil, err + } + rows, err := db.db.QueryContext(ctx, ` + WITH duplicate_paths AS ( + SELECT normalized_path + FROM drs_object_browse_index + WHERE `+cteWhere+` + GROUP BY normalized_path + HAVING COUNT(*) > 1 + ) + SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, + COALESCE(u.download_count, 0), u.last_download_time + FROM drs_object_browse_index bi + JOIN duplicate_paths dp ON dp.normalized_path = bi.normalized_path + JOIN drs_object o ON o.id = bi.object_id + LEFT JOIN object_usage u ON u.object_id = bi.object_id + WHERE `+mainWhere+` + ORDER BY bi.normalized_path ASC, bi.object_id ASC + `, cteArgs...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.StorageCleanupRecord, 0) + for rows.Next() { + var row models.StorageCleanupRecord + var lastDownload sql.NullTime + if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { + return nil, err } - row.NormalizedPath = info.Normalized if lastDownload.Valid { t := lastDownload.Time.UTC() row.LastDownloadTime = &t @@ -350,6 +399,21 @@ func (db *PostgresDB) ListStorageCleanupRecords(ctx context.Context, organizatio return out, nil } +func postgresStorageCleanupScopeArgs(resource, pathPrefix string, argStart int, normalizedPathColumn string) (string, []any, error) { + where := []string{fmt.Sprintf("resource = $%d", argStart)} + args := []any{resource} + normalizedPrefix, _, err := common.NormalizeBrowsePath(pathPrefix) + if err != nil { + return "", nil, err + } + if normalizedPrefix == "" { + return strings.Join(where, " AND "), args, nil + } + where = append(where, fmt.Sprintf("(%s = $%d OR %s LIKE $%d)", normalizedPathColumn, argStart+1, normalizedPathColumn, argStart+2)) + args = append(args, normalizedPrefix, normalizedPrefix+"/%") + return strings.Join(where, " AND "), args, nil +} + func (db *PostgresDB) ListFileUsageByObjectIDs(ctx context.Context, ids []string) ([]models.FileUsage, error) { if len(ids) == 0 { return []models.FileUsage{}, nil diff --git a/internal/db/postgres/metrics_sqlmock_test.go b/internal/db/postgres/metrics_sqlmock_test.go index 423f756c..15e5dd3e 100644 --- a/internal/db/postgres/metrics_sqlmock_test.go +++ b/internal/db/postgres/metrics_sqlmock_test.go @@ -151,3 +151,86 @@ func TestGetStoragePathSummary(t *testing.T) { t.Fatalf("unmet expectations: %v", err) } } + +func TestListStorageCleanupRecordsUsesSQLPrefixFilter(t *testing.T) { + pg, mock, rawDB := newMockPostgresDB(t) + defer rawDB.Close() + + now := time.Now().UTC() + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT DISTINCT e.object_id + FROM object_usage_event e + JOIN drs_object o ON o.id = e.object_id + `)).WillReturnRows(sqlmock.NewRows([]string{"object_id"})) + mock.ExpectCommit() + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, + COALESCE(u.download_count, 0), u.last_download_time + FROM drs_object_browse_index bi + JOIN drs_object o ON o.id = bi.object_id + LEFT JOIN object_usage u ON u.object_id = bi.object_id + WHERE resource = $1 AND (bi.normalized_path = $2 OR bi.normalized_path LIKE $3) + ORDER BY bi.normalized_path ASC, bi.object_id ASC + `)). + WithArgs("/organization/org/project/proj", "data", "data/%"). + WillReturnRows(sqlmock.NewRows([]string{"object_id", "normalized_path", "size", "updated_time", "download_count", "last_download_time"}). + AddRow("obj-1", "data/a.txt", int64(10), now, int64(2), now)) + + rows, err := pg.ListStorageCleanupRecords(context.Background(), "org", "proj", "data") + if err != nil { + t.Fatalf("ListStorageCleanupRecords returned error: %v", err) + } + if len(rows) != 1 || rows[0].NormalizedPath != "data/a.txt" { + t.Fatalf("unexpected rows: %+v", rows) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } +} + +func TestListDuplicateStorageCleanupRecordsFiltersToDuplicatePaths(t *testing.T) { + pg, mock, rawDB := newMockPostgresDB(t) + defer rawDB.Close() + + now := time.Now().UTC() + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT DISTINCT e.object_id + FROM object_usage_event e + JOIN drs_object o ON o.id = e.object_id + `)).WillReturnRows(sqlmock.NewRows([]string{"object_id"})) + mock.ExpectCommit() + mock.ExpectQuery(regexp.QuoteMeta(` + WITH duplicate_paths AS ( + SELECT normalized_path + FROM drs_object_browse_index + WHERE resource = $1 AND (normalized_path = $2 OR normalized_path LIKE $3) + GROUP BY normalized_path + HAVING COUNT(*) > 1 + ) + SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, + COALESCE(u.download_count, 0), u.last_download_time + FROM drs_object_browse_index bi + JOIN duplicate_paths dp ON dp.normalized_path = bi.normalized_path + JOIN drs_object o ON o.id = bi.object_id + LEFT JOIN object_usage u ON u.object_id = bi.object_id + WHERE resource = $1 AND (bi.normalized_path = $2 OR bi.normalized_path LIKE $3) + ORDER BY bi.normalized_path ASC, bi.object_id ASC + `)). + WithArgs("/organization/org/project/proj", "data", "data/%"). + WillReturnRows(sqlmock.NewRows([]string{"object_id", "normalized_path", "size", "updated_time", "download_count", "last_download_time"}). + AddRow("obj-1", "data/a.txt", int64(10), now, int64(2), now). + AddRow("obj-2", "data/a.txt", int64(11), now, int64(3), now)) + + rows, err := pg.ListDuplicateStorageCleanupRecords(context.Background(), "org", "proj", "data") + if err != nil { + t.Fatalf("ListDuplicateStorageCleanupRecords returned error: %v", err) + } + if len(rows) != 2 { + t.Fatalf("expected duplicate rows, got %+v", rows) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } +} diff --git a/internal/db/sqlite/metrics.go b/internal/db/sqlite/metrics.go index d1a97f56..beeeeb69 100644 --- a/internal/db/sqlite/metrics.go +++ b/internal/db/sqlite/metrics.go @@ -296,7 +296,7 @@ func (db *SqliteDB) ListStorageCleanupRecords(ctx context.Context, organization, if err != nil { return nil, err } - _, prefixSegments, err := common.NormalizeBrowsePath(pathPrefix) + where, args, err := sqliteStorageCleanupScopeArgs(resource, pathPrefix, "bi.normalized_path") if err != nil { return nil, err } @@ -306,9 +306,9 @@ func (db *SqliteDB) ListStorageCleanupRecords(ctx context.Context, organization, FROM drs_object_browse_index bi JOIN drs_object o ON o.id = bi.object_id LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE bi.resource = ? + WHERE `+where+` ORDER BY bi.normalized_path ASC, bi.object_id ASC - `, resource) + `, args...) if err != nil { return nil, err } @@ -321,14 +321,63 @@ func (db *SqliteDB) ListStorageCleanupRecords(ctx context.Context, organization, if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { return nil, err } - info, ok, err := common.BrowsePathInfoFromName(row.NormalizedPath) - if err != nil { - return nil, err + if lastDownload.Valid { + t := lastDownload.Time.UTC() + row.LastDownloadTime = &t } - if !ok || !common.HasBrowsePathPrefix(info.Segments, prefixSegments) { - continue + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + +func (db *SqliteDB) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + if err := db.flushObjectUsageEvents(ctx); err != nil { + return nil, err + } + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return nil, err + } + cteWhere, cteArgs, err := sqliteStorageCleanupScopeArgs(resource, pathPrefix, "normalized_path") + if err != nil { + return nil, err + } + mainWhere, _, err := sqliteStorageCleanupScopeArgs(resource, pathPrefix, "bi.normalized_path") + if err != nil { + return nil, err + } + rows, err := db.db.QueryContext(ctx, ` + WITH duplicate_paths AS ( + SELECT normalized_path + FROM drs_object_browse_index + WHERE `+cteWhere+` + GROUP BY normalized_path + HAVING COUNT(*) > 1 + ) + SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, + COALESCE(u.download_count, 0), u.last_download_time + FROM drs_object_browse_index bi + JOIN duplicate_paths dp ON dp.normalized_path = bi.normalized_path + JOIN drs_object o ON o.id = bi.object_id + LEFT JOIN object_usage u ON u.object_id = bi.object_id + WHERE `+mainWhere+` + ORDER BY bi.normalized_path ASC, bi.object_id ASC + `, cteArgs...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.StorageCleanupRecord, 0) + for rows.Next() { + var row models.StorageCleanupRecord + var lastDownload sql.NullTime + if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { + return nil, err } - row.NormalizedPath = info.Normalized if lastDownload.Valid { t := lastDownload.Time.UTC() row.LastDownloadTime = &t @@ -341,6 +390,21 @@ func (db *SqliteDB) ListStorageCleanupRecords(ctx context.Context, organization, return out, nil } +func sqliteStorageCleanupScopeArgs(resource, pathPrefix string, normalizedPathColumn string) (string, []any, error) { + where := []string{"resource = ?"} + args := []any{resource} + normalizedPrefix, _, err := common.NormalizeBrowsePath(pathPrefix) + if err != nil { + return "", nil, err + } + if normalizedPrefix == "" { + return strings.Join(where, " AND "), args, nil + } + where = append(where, fmt.Sprintf("(%s = ? OR %s LIKE ?)", normalizedPathColumn, normalizedPathColumn)) + args = append(args, normalizedPrefix, normalizedPrefix+"/%") + return strings.Join(where, " AND "), args, nil +} + func (db *SqliteDB) GetTransferAttributionSummaryByResources(ctx context.Context, filter models.TransferAttributionFilter, resources []string) (models.TransferAttributionSummary, error) { where, args := transferAttributionWhereByResources(filter, resources) var out models.TransferAttributionSummary diff --git a/internal/repair/format.go b/internal/repair/format.go index 8e6acfcb..8dfe1486 100644 --- a/internal/repair/format.go +++ b/internal/repair/format.go @@ -51,13 +51,18 @@ func WriteStorageCleanupAudit(w io.Writer, format string, report StorageCleanupR return err } for _, finding := range report.Findings { - if _, err := fmt.Fprintf(w, "%s [%s] %s action=%s\n", finding.NormalizedPath, finding.Kind, finding.Message, finding.RecommendedAction); err != nil { + if _, err := fmt.Fprintf(w, "%s [%s] %s action=%s scope=%s\n", finding.NormalizedPath, finding.Kind, finding.Message, finding.RecommendedAction, finding.CleanupScope); err != nil { return err } for _, record := range finding.Records { - if _, err := fmt.Fprintf(w, " - %s storage=%s downloads=%d\n", record.ObjectID, record.StorageStatus, record.DownloadCount); err != nil { + if _, err := fmt.Fprintf(w, " - %s storage=%s downloads=%d scope=%s\n", record.ObjectID, record.StorageStatus, record.DownloadCount, record.CleanupScope); err != nil { return err } + for _, probe := range record.AccessProbes { + if _, err := fmt.Fprintf(w, " * %s storage=%s bucket=%s error=%s message=%s\n", probe.URL, probe.StorageStatus, probe.Bucket, probe.ErrorKind, probe.StorageMessage); err != nil { + return err + } + } } } return nil diff --git a/internal/repair/project_diff_service.go b/internal/repair/project_diff_service.go new file mode 100644 index 00000000..2747277c --- /dev/null +++ b/internal/repair/project_diff_service.go @@ -0,0 +1,166 @@ +package repair + +import ( + "context" + "sort" + "strings" + "time" + + "github.com/calypr/syfon/internal/models" +) + +type projectDiffManager interface { + ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) + ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) +} + +type ProjectDiffService struct { + om projectDiffManager +} + +func NewProjectDiffService(om projectDiffManager) *ProjectDiffService { + return &ProjectDiffService{om: om} +} + +type projectDiffAggregate struct { + path string + objectIDs map[string]bool + recordCount int + sizeBytes int64 + downloadCount int64 + lastDownloadTime *time.Time +} + +func (s *ProjectDiffService) Audit(ctx context.Context, req ProjectDiffAuditRequest) (ProjectDiffReport, error) { + report := ProjectDiffReport{ + Organization: strings.TrimSpace(req.Organization), + Project: strings.TrimSpace(req.Project), + PathPrefix: strings.TrimSpace(req.PathPrefix), + Summary: ProjectDiffSummary{ + CountsByKind: map[FindingKind]int{}, + }, + } + + expected, hasExpected, err := normalizeExpectedPaths(req.ExpectedPaths) + if err != nil { + return ProjectDiffReport{}, err + } + + rows, err := s.loadRows(ctx, report.Organization, report.Project, report.PathPrefix, hasExpected) + if err != nil { + return ProjectDiffReport{}, err + } + + grouped := make(map[string]*projectDiffAggregate) + for _, row := range rows { + state := grouped[row.NormalizedPath] + if state == nil { + state = &projectDiffAggregate{ + path: row.NormalizedPath, + objectIDs: map[string]bool{}, + } + grouped[row.NormalizedPath] = state + } + state.recordCount++ + state.sizeBytes += row.Size + state.downloadCount += row.DownloadCount + state.objectIDs[row.ObjectID] = true + if row.LastDownloadTime != nil { + candidate := row.LastDownloadTime.UTC() + if state.lastDownloadTime == nil || candidate.After(*state.lastDownloadTime) { + state.lastDownloadTime = &candidate + } + } + } + + report.Summary.ScannedRecordCount = len(rows) + report.Summary.IndexedPathCount = len(grouped) + report.Summary.ExpectedPathCount = len(expected) + report.Summary.IncludesRepoManifest = hasExpected + + paths := make([]string, 0, len(grouped)) + for path := range grouped { + paths = append(paths, path) + } + sort.Strings(paths) + + for _, path := range paths { + agg := grouped[path] + if agg == nil { + continue + } + if hasExpected && expected[path] { + report.Summary.MatchedPathCount++ + } + if agg.recordCount > 1 { + report.Findings = append(report.Findings, buildProjectDiffFinding( + FindingDuplicateSyfonPaths, + SeverityWarn, + agg, + "Review duplicate records before deleting anything.", + )) + report.Summary.CountsByKind[FindingDuplicateSyfonPaths]++ + } + if hasExpected && !expected[path] { + report.Findings = append(report.Findings, buildProjectDiffFinding( + FindingSyfonMissingInRepo, + SeverityWarn, + agg, + "Prepare delete to verify storage before removing Syfon-only records.", + )) + report.Summary.CountsByKind[FindingSyfonMissingInRepo]++ + } + } + + if hasExpected { + missing := make([]string, 0) + for path := range expected { + if _, ok := grouped[path]; ok { + continue + } + missing = append(missing, path) + } + sort.Strings(missing) + for _, path := range missing { + report.Findings = append(report.Findings, ProjectDiffFinding{ + Kind: FindingRepoMissingInSyfon, + Severity: SeverityInfo, + NormalizedPath: path, + RecommendedAction: "Review missing Syfon records for this Git-tracked path.", + }) + report.Summary.CountsByKind[FindingRepoMissingInSyfon]++ + } + } + + report.Summary.TotalFindings = len(report.Findings) + if len(report.Summary.CountsByKind) == 0 { + report.Summary.CountsByKind = nil + } + return report, nil +} + +func (s *ProjectDiffService) loadRows(ctx context.Context, organization, project, pathPrefix string, hasExpected bool) ([]models.StorageCleanupRecord, error) { + if hasExpected { + return s.om.ListStorageCleanupRecords(ctx, organization, project, pathPrefix) + } + return s.om.ListDuplicateStorageCleanupRecords(ctx, organization, project, pathPrefix) +} + +func buildProjectDiffFinding(kind FindingKind, severity Severity, agg *projectDiffAggregate, recommendedAction string) ProjectDiffFinding { + objectIDs := make([]string, 0, len(agg.objectIDs)) + for objectID := range agg.objectIDs { + objectIDs = append(objectIDs, objectID) + } + sort.Strings(objectIDs) + return ProjectDiffFinding{ + Kind: kind, + Severity: severity, + NormalizedPath: agg.path, + ObjectIDs: objectIDs, + RecordCount: agg.recordCount, + SizeBytes: agg.sizeBytes, + DownloadCount: agg.downloadCount, + LastDownloadTime: agg.lastDownloadTime, + RecommendedAction: recommendedAction, + } +} diff --git a/internal/repair/project_diff_service_test.go b/internal/repair/project_diff_service_test.go new file mode 100644 index 00000000..5060d0d2 --- /dev/null +++ b/internal/repair/project_diff_service_test.go @@ -0,0 +1,90 @@ +package repair + +import ( + "context" + "testing" + "time" + + "github.com/calypr/syfon/internal/models" +) + +func TestProjectDiffAuditWithManifest(t *testing.T) { + lastDownload := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC) + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "dup-1", NormalizedPath: "data/a.tsv", Size: 10, DownloadCount: 2, LastDownloadTime: &lastDownload}, + {ObjectID: "dup-2", NormalizedPath: "data/a.tsv", Size: 10, DownloadCount: 3}, + {ObjectID: "syfon-only", NormalizedPath: "data/b.tsv", Size: 7, DownloadCount: 1}, + }, + } + + report, err := NewProjectDiffService(manager).Audit(context.Background(), ProjectDiffAuditRequest{ + Organization: "org", + Project: "proj", + PathPrefix: "data", + ExpectedPaths: []string{"data/a.tsv", "data/c.tsv"}, + }) + if err != nil { + t.Fatalf("Audit returned error: %v", err) + } + + if manager.listCalls != 1 { + t.Fatalf("expected full subtree listing once, got %d", manager.listCalls) + } + if manager.duplicateListCalls != 0 { + t.Fatalf("expected duplicate-only listing to be skipped, got %d", manager.duplicateListCalls) + } + if got := report.Summary.TotalFindings; got != 3 { + t.Fatalf("expected 3 findings, got %d", got) + } + if got := report.Summary.CountsByKind[FindingDuplicateSyfonPaths]; got != 1 { + t.Fatalf("expected 1 duplicate finding, got %d", got) + } + if got := report.Summary.CountsByKind[FindingSyfonMissingInRepo]; got != 1 { + t.Fatalf("expected 1 syfon-only finding, got %d", got) + } + if got := report.Summary.CountsByKind[FindingRepoMissingInSyfon]; got != 1 { + t.Fatalf("expected 1 repo-only finding, got %d", got) + } + if got := report.Summary.MatchedPathCount; got != 1 { + t.Fatalf("expected 1 matched path, got %d", got) + } + if got := report.Summary.IndexedPathCount; got != 2 { + t.Fatalf("expected 2 indexed paths, got %d", got) + } + if got := report.Summary.ScannedRecordCount; got != 3 { + t.Fatalf("expected 3 scanned records, got %d", got) + } +} + +func TestProjectDiffAuditWithoutManifestUsesDuplicateListing(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "dup-1", NormalizedPath: "data/a.tsv", Size: 10}, + {ObjectID: "dup-2", NormalizedPath: "data/a.tsv", Size: 10}, + {ObjectID: "single", NormalizedPath: "data/b.tsv", Size: 7}, + }, + } + + report, err := NewProjectDiffService(manager).Audit(context.Background(), ProjectDiffAuditRequest{ + Organization: "org", + Project: "proj", + PathPrefix: "data", + }) + if err != nil { + t.Fatalf("Audit returned error: %v", err) + } + + if manager.listCalls != 0 { + t.Fatalf("expected full subtree listing to be skipped, got %d", manager.listCalls) + } + if manager.duplicateListCalls != 1 { + t.Fatalf("expected duplicate-only listing once, got %d", manager.duplicateListCalls) + } + if got := report.Summary.TotalFindings; got != 1 { + t.Fatalf("expected 1 finding, got %d", got) + } + if got := report.Summary.IndexedPathCount; got != 1 { + t.Fatalf("expected 1 indexed path in duplicate mode, got %d", got) + } +} diff --git a/internal/repair/project_diff_types.go b/internal/repair/project_diff_types.go new file mode 100644 index 00000000..962848f4 --- /dev/null +++ b/internal/repair/project_diff_types.go @@ -0,0 +1,46 @@ +package repair + +import "time" + +const ( + FindingDuplicateSyfonPaths FindingKind = "duplicate_syfon_paths" + FindingSyfonMissingInRepo FindingKind = "syfon_missing_in_repo" + FindingRepoMissingInSyfon FindingKind = "repo_missing_in_syfon" +) + +type ProjectDiffAuditRequest struct { + Organization string `json:"organization"` + Project string `json:"project"` + PathPrefix string `json:"path_prefix,omitempty"` + ExpectedPaths []string `json:"expected_paths,omitempty"` +} + +type ProjectDiffFinding struct { + Kind FindingKind `json:"kind"` + Severity Severity `json:"severity"` + NormalizedPath string `json:"normalized_path"` + ObjectIDs []string `json:"object_ids,omitempty"` + RecordCount int `json:"record_count,omitempty"` + SizeBytes int64 `json:"size_bytes,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownloadTime *time.Time `json:"last_download_time,omitempty"` + RecommendedAction string `json:"recommended_action,omitempty"` +} + +type ProjectDiffSummary struct { + CountsByKind map[FindingKind]int `json:"counts_by_kind,omitempty"` + TotalFindings int `json:"total_findings"` + IndexedPathCount int `json:"indexed_path_count"` + ExpectedPathCount int `json:"expected_path_count"` + MatchedPathCount int `json:"matched_path_count"` + IncludesRepoManifest bool `json:"includes_repo_manifest"` + ScannedRecordCount int `json:"scanned_record_count"` +} + +type ProjectDiffReport struct { + Organization string `json:"organization"` + Project string `json:"project"` + PathPrefix string `json:"path_prefix,omitempty"` + Summary ProjectDiffSummary `json:"summary"` + Findings []ProjectDiffFinding `json:"findings,omitempty"` +} diff --git a/internal/repair/storage_cleanup_service.go b/internal/repair/storage_cleanup_service.go index c990f5f3..647e298a 100644 --- a/internal/repair/storage_cleanup_service.go +++ b/internal/repair/storage_cleanup_service.go @@ -2,19 +2,23 @@ package repair import ( "context" + "errors" "fmt" "sort" "strings" + "sync" "time" drsapi "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/core" "github.com/calypr/syfon/internal/models" + "github.com/google/uuid" ) type storageCleanupManager interface { ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) + ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) GetBulkObjects(ctx context.Context, ids []string, requiredMethod string) ([]models.InternalObject, error) InspectStorageObject(ctx context.Context, req core.InspectStorageRequest) (*core.StorageObjectMetadata, error) DeleteObjectWithOptions(ctx context.Context, id string, opts core.DeleteOptions) error @@ -29,10 +33,32 @@ func NewStorageCleanupService(om storageCleanupManager) *StorageCleanupService { } type storageCleanupPathState struct { - Path string - Records []StorageCleanupRecordAudit + Path string + Records []StorageCleanupRecordAudit + Analysis cleanupStructuralAnalysis } +type cleanupURLTemplate string + +const ( + cleanupURLTemplateUnknown cleanupURLTemplate = "unknown" + cleanupURLTemplateLegacyDID cleanupURLTemplate = "legacy_did_keyed" + cleanupURLTemplateProjectScope cleanupURLTemplate = "project_scoped" + cleanupURLTemplateChecksumRoot cleanupURLTemplate = "checksum_root" + cleanupURLTemplateOther cleanupURLTemplate = "other" +) + +type cleanupStructuralAnalysis struct { + ChecksumCount int + SizeCount int + StructuralReason string + LegacyURLTemplateDetected bool + StructurallyEquivalent bool + MixedContentCollision bool +} + +const defaultStorageCleanupProbeConcurrency = 8 + func (s *StorageCleanupService) Audit(ctx context.Context, req StorageCleanupAuditRequest) (StorageCleanupReport, error) { checkStorage := true if req.CheckStorage != nil { @@ -48,10 +74,23 @@ func (s *StorageCleanupService) Audit(ctx context.Context, req StorageCleanupAud if err != nil { return StorageCleanupReport{}, err } - rows, err := s.om.ListStorageCleanupRecords(ctx, report.Organization, report.Project, report.PathPrefix) + selectedPaths, hasSelectedPaths, err := normalizeExpectedPaths(req.SelectedPaths) if err != nil { return StorageCleanupReport{}, err } + rows, err := s.loadCleanupRows(ctx, report.Organization, report.Project, report.PathPrefix, hasExpected) + if err != nil { + return StorageCleanupReport{}, err + } + if hasSelectedPaths { + filtered := make([]models.StorageCleanupRecord, 0, len(rows)) + for _, row := range rows { + if selectedPaths[row.NormalizedPath] { + filtered = append(filtered, row) + } + } + rows = filtered + } report.Scanned = len(rows) ids := uniqueCleanupIDs(rows) @@ -82,26 +121,38 @@ func (s *StorageCleanupService) Audit(ctx context.Context, req StorageCleanupAud if obj, ok := objectByID[row.ObjectID]; ok && obj.AccessMethods != nil { record.CurrentAccessURLs = storageCleanupAccessMethodURLs(*obj.AccessMethods) } - if checkStorage { - record.StorageStatus, record.StorageMessage = s.inspectCleanupRecord(ctx, report.Organization, report.Project, record.CurrentAccessURLs) - } else { - record.StorageStatus = StorageProbeStatusUnknown - } + record.StorageStatus = StorageProbeStatusUnknown state.Records = append(state.Records, record) } + if checkStorage { + if err := s.populateProbeStates(ctx, report.Organization, report.Project, grouped); err != nil { + return StorageCleanupReport{}, err + } + } + paths := make([]string, 0, len(grouped)) for path := range grouped { paths = append(paths, path) } sort.Strings(paths) + report.ScannedPaths = len(paths) for _, path := range paths { state := grouped[path] sort.Slice(state.Records, func(i, j int) bool { return state.Records[i].ObjectID < state.Records[j].ObjectID }) + state.Analysis = analyzeCleanupStructuralDuplicates(state, objectByID) findings := classifyStorageCleanupPath(state, hasExpected, expected[state.Path]) + if len(findings) == 0 && len(state.Records) > 1 { + findings = append(findings, newDuplicateStorageProbeErrorFinding(state)) + } + if len(findings) == 0 { + report.UnclassifiedPaths++ + continue + } + report.ClassifiedPaths++ for _, finding := range findings { report.Summary[finding.Kind]++ report.Findings = append(report.Findings, finding) @@ -113,6 +164,13 @@ func (s *StorageCleanupService) Audit(ctx context.Context, req StorageCleanupAud return report, nil } +func (s *StorageCleanupService) loadCleanupRows(ctx context.Context, organization, project, pathPrefix string, hasExpected bool) ([]models.StorageCleanupRecord, error) { + if hasExpected { + return s.om.ListStorageCleanupRecords(ctx, organization, project, pathPrefix) + } + return s.om.ListDuplicateStorageCleanupRecords(ctx, organization, project, pathPrefix) +} + func (s *StorageCleanupService) Apply(ctx context.Context, req StorageCleanupApplyRequest) (StorageCleanupApplyResult, error) { checkStorage := req.CheckStorage report, err := s.Audit(ctx, StorageCleanupAuditRequest{ @@ -120,6 +178,7 @@ func (s *StorageCleanupService) Apply(ctx context.Context, req StorageCleanupApp Project: req.Project, PathPrefix: req.PathPrefix, ExpectedPaths: req.ExpectedPaths, + SelectedPaths: req.SelectedPaths, CheckStorage: checkStorage, }) if err != nil { @@ -165,42 +224,370 @@ func (s *StorageCleanupService) Apply(ctx context.Context, req StorageCleanupApp return result, nil } -func (s *StorageCleanupService) inspectCleanupRecord(ctx context.Context, organization, project string, urls []string) (StorageProbeStatus, string) { - if len(urls) == 0 { - return StorageProbeStatusUnknown, "record has no access URLs" +type cleanupProbeResult struct { + Status StorageProbeStatus + Message string + CleanupScope CleanupScope + HasBrokenAccess bool + Probes []StorageCleanupAccessProbe +} + +type storageCleanupProbeTask struct { + recordIdx int + urlIdx int + rawURL string +} + +type storageCleanupProbeResponse struct { + recordIdx int + urlIdx int + probe StorageCleanupAccessProbe +} + +type storageCleanupRecordSlot struct { + state *storageCleanupPathState + index int +} + +type storageCleanupAuditContext struct { + om storageCleanupManager + organization string + project string + + mu sync.Mutex + urlCache map[string]StorageCleanupAccessProbe + bucketErrCache map[string]StorageCleanupAccessProbe + bucketInFlight map[string]chan struct{} +} + +func (s *StorageCleanupService) populateProbeStates(ctx context.Context, organization, project string, grouped map[string]*storageCleanupPathState) error { + ctx = core.WithStorageInspectCache(ctx) + recordSlots := make([]storageCleanupRecordSlot, 0) + tasks := make([]storageCleanupProbeTask, 0) + for _, path := range sortedCleanupPaths(grouped) { + state := grouped[path] + for i := range state.Records { + recordIdx := len(recordSlots) + recordSlots = append(recordSlots, storageCleanupRecordSlot{state: state, index: i}) + urls := state.Records[i].CurrentAccessURLs + for urlIdx, raw := range urls { + raw = strings.TrimSpace(raw) + if raw == "" { + continue + } + tasks = append(tasks, storageCleanupProbeTask{recordIdx: recordIdx, urlIdx: urlIdx, rawURL: raw}) + } + } } - var sawMissing bool - var lastErr error - for _, raw := range urls { - if strings.TrimSpace(raw) == "" { - continue + if len(recordSlots) == 0 { + return nil + } + accumulators := make([][]StorageCleanupAccessProbe, len(recordSlots)) + for i, slot := range recordSlots { + accumulators[i] = make([]StorageCleanupAccessProbe, len(slot.state.Records[slot.index].CurrentAccessURLs)) + } + auditCtx := &storageCleanupAuditContext{ + om: s.om, + organization: organization, + project: project, + urlCache: map[string]StorageCleanupAccessProbe{}, + bucketErrCache: map[string]StorageCleanupAccessProbe{}, + bucketInFlight: map[string]chan struct{}{}, + } + if err := auditCtx.runProbes(ctx, tasks, accumulators); err != nil { + return err + } + for i, slot := range recordSlots { + record := &slot.state.Records[slot.index] + probeState := deriveCleanupProbeResult(accumulators[i]) + record.AccessProbes = probeState.Probes + record.StorageStatus = probeState.Status + record.StorageMessage = probeState.Message + record.CleanupScope = probeState.CleanupScope + } + return nil +} + +func (a *storageCleanupAuditContext) runProbes(ctx context.Context, tasks []storageCleanupProbeTask, accumulators [][]StorageCleanupAccessProbe) error { + if len(tasks) == 0 { + return nil + } + workerCount := defaultStorageCleanupProbeConcurrency + if workerCount > len(tasks) { + workerCount = len(tasks) + } + jobs := make(chan storageCleanupProbeTask) + results := make(chan storageCleanupProbeResponse, len(tasks)) + errCh := make(chan error, 1) + var wg sync.WaitGroup + + for i := 0; i < workerCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-ctx.Done(): + select { + case errCh <- ctx.Err(): + default: + } + return + case task, ok := <-jobs: + if !ok { + return + } + probe, err := a.probeURL(ctx, task.rawURL) + if err != nil { + select { + case errCh <- err: + default: + } + return + } + select { + case <-ctx.Done(): + select { + case errCh <- ctx.Err(): + default: + } + return + case results <- storageCleanupProbeResponse{recordIdx: task.recordIdx, urlIdx: task.urlIdx, probe: probe}: + } + } + } + }() + } + + go func() { + defer close(jobs) + for _, task := range tasks { + select { + case <-ctx.Done(): + return + case jobs <- task: + } } - _, err := s.om.InspectStorageObject(ctx, core.InspectStorageRequest{ - Organization: organization, - Project: project, - ObjectURL: raw, - }) - if err == nil { - return StorageProbeStatusLive, "" + }() + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + received := 0 + for received < len(tasks) { + select { + case err := <-errCh: + if err != nil { + <-done + return err + } + case <-ctx.Done(): + <-done + return ctx.Err() + case res := <-results: + accumulators[res.recordIdx][res.urlIdx] = res.probe + received++ } - if inspectErr, ok := err.(*core.StorageInspectError); ok && inspectErr.Kind == core.StorageInspectObjectNotFound { - sawMissing = true - lastErr = err - continue + } + <-done + return nil +} + +func (a *storageCleanupAuditContext) probeURL(ctx context.Context, raw string) (StorageCleanupAccessProbe, error) { + raw = strings.TrimSpace(raw) + probe := StorageCleanupAccessProbe{URL: raw} + if raw == "" { + return probe, nil + } + if bucket, _, ok := common.ParseS3URL(raw); ok { + probe.Bucket = bucket + } + + var ( + bucketLeader bool + releaseWait func() + ) + for { + a.mu.Lock() + if cached, ok := a.urlCache[raw]; ok { + a.mu.Unlock() + return cached, nil + } + if probe.Bucket != "" { + if cached, ok := a.bucketErrCache[probe.Bucket]; ok { + cached.URL = raw + cached.Bucket = probe.Bucket + a.urlCache[raw] = cached + a.mu.Unlock() + return cached, nil + } + if !bucketLeader { + if waiter, ok := a.bucketInFlight[probe.Bucket]; ok { + a.mu.Unlock() + select { + case <-ctx.Done(): + return StorageCleanupAccessProbe{}, ctx.Err() + case <-waiter: + continue + } + } + waiter := make(chan struct{}) + a.bucketInFlight[probe.Bucket] = waiter + bucketLeader = true + releaseWait = func() { + a.mu.Lock() + delete(a.bucketInFlight, probe.Bucket) + close(waiter) + a.mu.Unlock() + } + } } - lastErr = err + a.mu.Unlock() + break } - if sawMissing { - return StorageProbeStatusMissing, "storage object not found" + if releaseWait != nil { + defer releaseWait() } - if lastErr != nil { - return StorageProbeStatusError, lastErr.Error() + + if err := ctx.Err(); err != nil { + return StorageCleanupAccessProbe{}, err + } + + _, err := a.om.InspectStorageObject(ctx, core.InspectStorageRequest{ + Organization: a.organization, + Project: a.project, + ObjectURL: raw, + }) + if err == nil { + probe.StorageStatus = StorageProbeStatusLive + a.cacheProbe(raw, probe) + return probe, nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return StorageCleanupAccessProbe{}, err + } + if inspectErr, ok := err.(*core.StorageInspectError); ok { + probe.ErrorKind = string(inspectErr.Kind) + probe.StorageMessage = inspectErr.Error() + switch inspectErr.Kind { + case core.StorageInspectObjectNotFound: + probe.StorageStatus = StorageProbeStatusMissing + default: + probe.StorageStatus = StorageProbeStatusError + if probe.Bucket != "" && cacheableBucketInspectKind(inspectErr.Kind) { + a.cacheBucketError(probe.Bucket, probe) + } + } + a.cacheProbe(raw, probe) + return probe, nil } - return StorageProbeStatusUnknown, "" + probe.StorageStatus = StorageProbeStatusError + probe.StorageMessage = err.Error() + a.cacheProbe(raw, probe) + return probe, nil +} + +func (a *storageCleanupAuditContext) cacheProbe(raw string, probe StorageCleanupAccessProbe) { + a.mu.Lock() + defer a.mu.Unlock() + a.urlCache[raw] = probe +} + +func (a *storageCleanupAuditContext) cacheBucketError(bucket string, probe StorageCleanupAccessProbe) { + a.mu.Lock() + defer a.mu.Unlock() + a.bucketErrCache[bucket] = probe +} + +func cacheableBucketInspectKind(kind core.StorageInspectErrorKind) bool { + switch kind { + case core.StorageInspectCredentialMissing, core.StorageInspectPermissionDenied, core.StorageInspectUnsupported: + return true + default: + return false + } +} + +func deriveCleanupProbeResult(probes []StorageCleanupAccessProbe) cleanupProbeResult { + if len(probes) == 0 { + return cleanupProbeResult{ + Status: StorageProbeStatusUnknown, + Message: "record has no access URLs", + CleanupScope: CleanupScopeRecord, + } + } + + var ( + liveCount int + missingCount int + errorCount int + usableProbeSeen bool + lastMessage string + filtered = make([]StorageCleanupAccessProbe, 0, len(probes)) + ) + for _, probe := range probes { + if strings.TrimSpace(probe.URL) == "" { + continue + } + filtered = append(filtered, probe) + switch probe.StorageStatus { + case StorageProbeStatusLive: + liveCount++ + usableProbeSeen = true + case StorageProbeStatusMissing: + missingCount++ + usableProbeSeen = true + lastMessage = probe.StorageMessage + case StorageProbeStatusError: + errorCount++ + usableProbeSeen = true + lastMessage = probe.StorageMessage + } + } + if len(filtered) == 0 { + return cleanupProbeResult{ + Status: StorageProbeStatusUnknown, + Message: "record has no usable access URLs", + CleanupScope: CleanupScopeRecord, + } + } + + result := cleanupProbeResult{Probes: filtered} + switch { + case liveCount > 0: + result.Status = StorageProbeStatusLive + if errorCount > 0 { + result.CleanupScope = CleanupScopeAccessURL + result.Message = "record has one or more broken access URLs" + result.HasBrokenAccess = true + } + case missingCount > 0 && errorCount == 0: + result.Status = StorageProbeStatusMissing + result.CleanupScope = CleanupScopeRecord + result.Message = "storage object not found" + case errorCount > 0: + result.Status = StorageProbeStatusError + result.CleanupScope = CleanupScopeRecord + result.Message = lastMessage + result.HasBrokenAccess = true + case usableProbeSeen: + result.Status = StorageProbeStatusUnknown + result.CleanupScope = CleanupScopeRecord + result.Message = "record could not be classified from storage probes" + default: + result.Status = StorageProbeStatusUnknown + result.CleanupScope = CleanupScopeRecord + result.Message = "record has no usable access URLs" + } + return result } func classifyStorageCleanupPath(state *storageCleanupPathState, hasExpected, inExpectedSet bool) []StorageCleanupFinding { - liveCount, missingCount, errorCount := 0, 0, 0 + liveCount, missingCount, errorCount, unknownCount := 0, 0, 0, 0 + hasBrokenAccess := false for _, rec := range state.Records { switch rec.StorageStatus { case StorageProbeStatusLive: @@ -209,44 +596,55 @@ func classifyStorageCleanupPath(state *storageCleanupPathState, hasExpected, inE missingCount++ case StorageProbeStatusError: errorCount++ + case StorageProbeStatusUnknown: + unknownCount++ + } + if recordHasBrokenAccess(rec) { + hasBrokenAccess = true } } - findings := make([]StorageCleanupFinding, 0, 2) + findings := make([]StorageCleanupFinding, 0, 3) if len(state.Records) > 1 { switch { case liveCount > 1: - findings = append(findings, StorageCleanupFinding{ + findings = append(findings, withStructuralContext(StorageCleanupFinding{ Kind: FindingLiveDuplicateConflict, Severity: SeverityWarn, NormalizedPath: state.Path, Message: fmt.Sprintf("multiple records for %s still resolve in storage", state.Path), RecommendedAction: "manual_review", Records: cloneCleanupRecords(state.Records), - }) - case liveCount >= 1 && missingCount >= 1 && errorCount == 0: - findings = append(findings, StorageCleanupFinding{ + }, state.Analysis)) + case liveCount >= 1 && missingCount >= 1: + findings = append(findings, withStructuralContext(StorageCleanupFinding{ Kind: FindingStaleDuplicateRecord, Severity: SeverityWarn, NormalizedPath: state.Path, Message: fmt.Sprintf("duplicate path %s has stale records that no longer resolve", state.Path), RecommendedAction: "delete_stale_duplicates", Records: cloneCleanupRecords(state.Records), - }) - case errorCount > 0: - findings = append(findings, StorageCleanupFinding{ - Kind: FindingStorageProbeError, - Severity: SeverityWarn, - NormalizedPath: state.Path, - Message: fmt.Sprintf("duplicate path %s could not be fully verified in storage", state.Path), - RecommendedAction: "manual_review", - Records: cloneCleanupRecords(state.Records), - }) + }, state.Analysis)) + case errorCount > 0 && !hasBrokenAccess: + findings = append(findings, newDuplicateStorageProbeErrorFinding(state)) + case unknownCount > 0 || (errorCount > 0 && hasBrokenAccess) || (liveCount > 0 && unknownCount > 0) || (missingCount > 0 && unknownCount > 0): + findings = append(findings, newDuplicateStorageProbeErrorFinding(state)) } } + if hasBrokenAccess { + findings = append(findings, withStructuralContext(StorageCleanupFinding{ + Kind: FindingBrokenAccessURLError, + Severity: SeverityWarn, + NormalizedPath: state.Path, + Message: fmt.Sprintf("path %s includes one or more broken access URLs", state.Path), + RecommendedAction: "manual_review", + CleanupScope: cleanupScopeForFinding(state.Records), + Records: cloneCleanupRecords(filterCleanupRecordsWithBrokenAccess(state.Records)), + }, state.Analysis)) + } if hasExpected && !inExpectedSet { switch { case liveCount > 0: - findings = append(findings, StorageCleanupFinding{ + findings = append(findings, withStructuralContext(StorageCleanupFinding{ Kind: FindingRepoOrphanLiveObject, Severity: SeverityWarn, NormalizedPath: state.Path, @@ -254,9 +652,9 @@ func classifyStorageCleanupPath(state *storageCleanupPathState, hasExpected, inE RecommendedAction: "delete_repo_orphans", RepoDeleteCandidate: true, Records: cloneCleanupRecords(state.Records), - }) + }, state.Analysis)) case missingCount > 0 && errorCount == 0: - findings = append(findings, StorageCleanupFinding{ + findings = append(findings, withStructuralContext(StorageCleanupFinding{ Kind: FindingRepoOrphanStaleRecord, Severity: SeverityWarn, NormalizedPath: state.Path, @@ -264,17 +662,18 @@ func classifyStorageCleanupPath(state *storageCleanupPathState, hasExpected, inE RecommendedAction: "delete_repo_orphans", RepoDeleteCandidate: true, Records: cloneCleanupRecords(state.Records), - }) + }, state.Analysis)) case errorCount > 0: - findings = append(findings, StorageCleanupFinding{ + findings = append(findings, withStructuralContext(StorageCleanupFinding{ Kind: FindingStorageProbeError, Severity: SeverityWarn, NormalizedPath: state.Path, Message: fmt.Sprintf("path %s is absent from expected repo contents but storage verification failed", state.Path), RecommendedAction: "manual_review", RepoDeleteCandidate: true, + CleanupScope: cleanupScopeForFinding(state.Records), Records: cloneCleanupRecords(state.Records), - }) + }, state.Analysis)) } } return findings @@ -325,6 +724,9 @@ func buildStorageCleanupActions(report StorageCleanupReport, req StorageCleanupA if len(objectFilter) > 0 && !objectFilter[rec.ObjectID] { continue } + if rec.CleanupScope == CleanupScopeAccessURL { + continue + } actions[rec.ObjectID] = storageCleanupAction{Path: finding.NormalizedPath, Kind: finding.Kind} } case FindingRepoOrphanLiveObject, FindingRepoOrphanStaleRecord: @@ -336,6 +738,9 @@ func buildStorageCleanupActions(report StorageCleanupReport, req StorageCleanupA if len(objectFilter) > 0 && !objectFilter[rec.ObjectID] { continue } + if rec.CleanupScope == CleanupScopeAccessURL { + continue + } switch rec.StorageStatus { case StorageProbeStatusLive: actions[rec.ObjectID] = storageCleanupAction{Path: finding.NormalizedPath, Kind: finding.Kind, DeleteStorageData: true} @@ -384,6 +789,15 @@ func uniqueCleanupIDs(rows []models.StorageCleanupRecord) []string { return out } +func sortedCleanupPaths(grouped map[string]*storageCleanupPathState) []string { + paths := make([]string, 0, len(grouped)) + for path := range grouped { + paths = append(paths, path) + } + sort.Strings(paths) + return paths +} + func storageCleanupAccessMethodURLs(methods []drsapi.AccessMethod) []string { out := make([]string, 0, len(methods)) for _, method := range methods { @@ -398,12 +812,148 @@ func storageCleanupAccessMethodURLs(methods []drsapi.AccessMethod) []string { return out } +func analyzeCleanupStructuralDuplicates(state *storageCleanupPathState, objectByID map[string]models.InternalObject) cleanupStructuralAnalysis { + checksumSet := map[string]bool{} + sizeSet := map[int64]bool{} + templateSet := map[cleanupURLTemplate]bool{} + legacyDetected := false + for _, record := range state.Records { + sizeSet[record.Size] = true + if obj, ok := objectByID[record.ObjectID]; ok { + if checksum, ok := common.CanonicalSHA256(obj.Checksums); ok && checksum != "" { + checksumSet[strings.ToLower(strings.TrimSpace(checksum))] = true + for _, rawURL := range record.CurrentAccessURLs { + tpl := detectCleanupURLTemplate(rawURL, checksum) + templateSet[tpl] = true + if tpl == cleanupURLTemplateLegacyDID { + legacyDetected = true + } + } + continue + } + } + for _, rawURL := range record.CurrentAccessURLs { + tpl := detectCleanupURLTemplate(rawURL, "") + templateSet[tpl] = true + if tpl == cleanupURLTemplateLegacyDID { + legacyDetected = true + } + } + } + analysis := cleanupStructuralAnalysis{ + ChecksumCount: len(checksumSet), + SizeCount: len(sizeSet), + LegacyURLTemplateDetected: legacyDetected, + } + sameChecksum := len(checksumSet) == 1 && len(state.Records) > 1 + sameSize := len(sizeSet) == 1 && len(state.Records) > 1 + multipleTemplates := len(templateSet) > 1 + switch { + case sameChecksum && sameSize && legacyDetected && multipleTemplates: + analysis.StructurallyEquivalent = true + analysis.StructuralReason = "same_checksum_same_size_legacy_url_mismatch" + case sameChecksum && sameSize && multipleTemplates: + analysis.StructurallyEquivalent = true + analysis.StructuralReason = "same_checksum_same_size_url_template_mismatch" + case sameChecksum && sameSize: + analysis.StructurallyEquivalent = true + analysis.StructuralReason = "same_checksum_same_size" + case len(checksumSet) > 1 || len(sizeSet) > 1: + analysis.MixedContentCollision = true + analysis.StructuralReason = "mixed_checksum_or_size_duplicates" + case len(checksumSet) == 0: + analysis.StructuralReason = "duplicate_path_no_checksum_evidence" + default: + analysis.StructuralReason = "duplicate_path_structural_mismatch" + } + return analysis +} + +func detectCleanupURLTemplate(rawURL string, checksum string) cleanupURLTemplate { + _, key, ok := common.ParseS3URL(strings.TrimSpace(rawURL)) + if !ok { + return cleanupURLTemplateUnknown + } + segments := make([]string, 0) + for _, segment := range strings.Split(strings.Trim(key, "/"), "/") { + segment = strings.TrimSpace(segment) + if segment != "" { + segments = append(segments, segment) + } + } + if len(segments) == 0 { + return cleanupURLTemplateUnknown + } + last := strings.TrimSpace(segments[len(segments)-1]) + if checksum != "" && !strings.EqualFold(last, checksum) { + return cleanupURLTemplateOther + } + if len(segments) >= 2 { + if _, err := uuid.Parse(strings.TrimSpace(segments[0])); err == nil { + return cleanupURLTemplateLegacyDID + } + return cleanupURLTemplateProjectScope + } + if checksum != "" && strings.EqualFold(segments[0], checksum) { + return cleanupURLTemplateChecksumRoot + } + return cleanupURLTemplateOther +} + +func withStructuralContext(finding StorageCleanupFinding, analysis cleanupStructuralAnalysis) StorageCleanupFinding { + finding.StructuralReason = analysis.StructuralReason + finding.ChecksumCount = analysis.ChecksumCount + finding.SizeCount = analysis.SizeCount + finding.LegacyURLTemplateDetected = analysis.LegacyURLTemplateDetected + return finding +} + +func newDuplicateStorageProbeErrorFinding(state *storageCleanupPathState) StorageCleanupFinding { + return withStructuralContext(StorageCleanupFinding{ + Kind: FindingStorageProbeError, + Severity: SeverityWarn, + NormalizedPath: state.Path, + Message: fmt.Sprintf("duplicate path %s could not be fully classified after storage verification", state.Path), + RecommendedAction: "manual_review", + CleanupScope: cleanupScopeForFinding(state.Records), + Records: cloneCleanupRecords(state.Records), + }, state.Analysis) +} + func cloneCleanupRecords(in []StorageCleanupRecordAudit) []StorageCleanupRecordAudit { out := make([]StorageCleanupRecordAudit, len(in)) copy(out, in) return out } +func filterCleanupRecordsWithBrokenAccess(in []StorageCleanupRecordAudit) []StorageCleanupRecordAudit { + out := make([]StorageCleanupRecordAudit, 0, len(in)) + for _, record := range in { + if recordHasBrokenAccess(record) { + out = append(out, record) + } + } + return out +} + +func recordHasBrokenAccess(record StorageCleanupRecordAudit) bool { + for _, probe := range record.AccessProbes { + if probe.StorageStatus == StorageProbeStatusError { + return true + } + } + return false +} + +func cleanupScopeForFinding(records []StorageCleanupRecordAudit) CleanupScope { + for _, record := range records { + if record.CleanupScope == CleanupScopeAccessURL { + return CleanupScopeAccessURL + } + } + return CleanupScopeRecord +} + func utcTimeValue(v time.Time) *time.Time { if v.IsZero() { return nil diff --git a/internal/repair/storage_cleanup_service_test.go b/internal/repair/storage_cleanup_service_test.go index bc28b68e..8d3789b5 100644 --- a/internal/repair/storage_cleanup_service_test.go +++ b/internal/repair/storage_cleanup_service_test.go @@ -5,6 +5,7 @@ import ( "testing" drsapi "github.com/calypr/syfon/apigen/server/drs" + "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/core" "github.com/calypr/syfon/internal/models" ) @@ -16,19 +17,30 @@ func TestStorageCleanupAuditStaleDuplicateRecord(t *testing.T) { {ObjectID: "obj-missing", NormalizedPath: "data/file.tsv", Size: 10}, }, objects: map[string]models.InternalObject{ - "obj-live": cleanupObject("obj-live", "s3://bucket/live/file.tsv"), - "obj-missing": cleanupObject("obj-missing", "s3://bucket/missing/file.tsv"), + "obj-live": cleanupObjectWithChecksum( + "obj-live", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "s3://bucket/project/path/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ), + "obj-missing": cleanupObjectWithChecksum( + "obj-missing", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "s3://bucket/11111111-1111-1111-1111-111111111111/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ), + }, + inspectByURL: map[string]error{ + "s3://bucket/project/path/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": nil, + "s3://bucket/11111111-1111-1111-1111-111111111111/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, }, - } - manager.inspectByObject = map[string]error{ - "obj-live": nil, - "obj-missing": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, } svc := NewStorageCleanupService(manager) report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ Organization: "org", Project: "proj", + ExpectedPaths: []string{ + "data/file.tsv", + }, }) if err != nil { t.Fatalf("Audit failed: %v", err) @@ -39,6 +51,12 @@ func TestStorageCleanupAuditStaleDuplicateRecord(t *testing.T) { if report.Findings[0].Kind != FindingStaleDuplicateRecord { t.Fatalf("expected stale duplicate finding, got %s", report.Findings[0].Kind) } + if report.Findings[0].StructuralReason != "same_checksum_same_size_legacy_url_mismatch" { + t.Fatalf("unexpected structural reason: %q", report.Findings[0].StructuralReason) + } + if !report.Findings[0].LegacyURLTemplateDetected { + t.Fatal("expected legacy url template detection") + } } func TestStorageCleanupAuditLiveDuplicateConflict(t *testing.T) { @@ -62,6 +80,199 @@ func TestStorageCleanupAuditLiveDuplicateConflict(t *testing.T) { } } +func TestStorageCleanupAuditDuplicateWithoutAccessMethodsEmitsProbeError(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-1", NormalizedPath: "data/file.tsv", Size: 10}, + {ObjectID: "obj-2", NormalizedPath: "data/file.tsv", Size: 11}, + }, + objects: map[string]models.InternalObject{ + "obj-1": cleanupObject("obj-1"), + "obj-2": cleanupObject("obj-2"), + }, + } + + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if report.Scanned != 2 || report.ScannedPaths != 1 || report.ClassifiedPaths != 1 || report.UnclassifiedPaths != 0 { + t.Fatalf("unexpected counters: %+v", report) + } + if len(report.Findings) != 1 || report.Findings[0].Kind != FindingStorageProbeError { + t.Fatalf("expected one storage probe error, got %+v", report.Findings) + } + for _, record := range report.Findings[0].Records { + if record.StorageStatus != StorageProbeStatusUnknown { + t.Fatalf("expected unknown status, got %s", record.StorageStatus) + } + if record.StorageMessage != "record has no access URLs" { + t.Fatalf("unexpected storage message: %q", record.StorageMessage) + } + } +} + +func TestStorageCleanupAuditDuplicateWithOneLiveAndOneUnknownEmitsProbeError(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-live", NormalizedPath: "data/file.tsv", Size: 10}, + {ObjectID: "obj-unknown", NormalizedPath: "data/file.tsv", Size: 11}, + }, + objects: map[string]models.InternalObject{ + "obj-live": cleanupObject("obj-live", "s3://bucket/live.tsv"), + "obj-unknown": cleanupObject("obj-unknown"), + }, + inspectByURL: map[string]error{ + "s3://bucket/live.tsv": nil, + }, + } + + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if len(report.Findings) != 1 || report.Findings[0].Kind != FindingStorageProbeError { + t.Fatalf("expected one storage probe error, got %+v", report.Findings) + } +} + +func TestStorageCleanupAuditDuplicateWithEmptyAccessURLsEmitsProbeError(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-1", NormalizedPath: "data/file.tsv", Size: 10}, + {ObjectID: "obj-2", NormalizedPath: "data/file.tsv", Size: 11}, + }, + objects: map[string]models.InternalObject{ + "obj-1": cleanupObject("obj-1", ""), + "obj-2": cleanupObject("obj-2", " "), + }, + } + + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if len(report.Findings) != 1 || report.Findings[0].Kind != FindingStorageProbeError { + t.Fatalf("expected one storage probe error, got %+v", report.Findings) + } + for _, record := range report.Findings[0].Records { + if record.StorageStatus != StorageProbeStatusUnknown { + t.Fatalf("expected unknown status, got %s", record.StorageStatus) + } + if record.StorageMessage != "record has no access URLs" { + t.Fatalf("unexpected storage message: %q", record.StorageMessage) + } + } +} + +func TestStorageCleanupAuditBrokenAccessURLDuplicateDoesNotAbort(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-live", NormalizedPath: "data/file.tsv", Size: 10}, + {ObjectID: "obj-broken", NormalizedPath: "data/file.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "obj-live": cleanupObjectWithChecksum( + "obj-live", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "s3://bucket/project/path/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + ), + "obj-broken": cleanupObjectWithChecksum( + "obj-broken", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "s3://old-bucket/22222222-2222-2222-2222-222222222222/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + ), + }, + inspectByURL: map[string]error{ + "s3://bucket/project/path/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": nil, + "s3://old-bucket/22222222-2222-2222-2222-222222222222/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": &core.StorageInspectError{ + Kind: core.StorageInspectCredentialMissing, + Message: "no stored bucket credential found for bucket \"old-bucket\"", + }, + }, + } + + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + ExpectedPaths: []string{"data/file.tsv"}, + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if len(report.Findings) != 2 { + t.Fatalf("expected two findings, got %+v", report.Findings) + } + if report.Findings[0].Kind != FindingStorageProbeError || report.Findings[1].Kind != FindingBrokenAccessURLError { + t.Fatalf("unexpected finding kinds: %+v", report.Findings) + } + finding := report.Findings[1] + if finding.Kind != FindingBrokenAccessURLError { + t.Fatalf("expected broken access url finding, got %s", finding.Kind) + } + if len(finding.Records) != 1 || finding.Records[0].ObjectID != "obj-broken" { + t.Fatalf("expected only broken record in finding, got %+v", finding.Records) + } + if finding.CleanupScope != CleanupScopeRecord { + t.Fatalf("expected record cleanup scope, got %s", finding.CleanupScope) + } + if finding.StructuralReason != "same_checksum_same_size_legacy_url_mismatch" { + t.Fatalf("unexpected structural reason: %q", finding.StructuralReason) + } +} + +func TestStorageCleanupAuditMixedMissingAndBrokenDuplicateEmitsBothFindings(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-live", NormalizedPath: "data/file.tsv", Size: 10}, + {ObjectID: "obj-missing", NormalizedPath: "data/file.tsv", Size: 10}, + {ObjectID: "obj-broken", NormalizedPath: "data/file.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "obj-live": cleanupObject("obj-live", "s3://bucket/live/file.tsv"), + "obj-missing": cleanupObject("obj-missing", "s3://bucket/missing/file.tsv"), + "obj-broken": cleanupObject("obj-broken", "s3://old-bucket/file.tsv"), + }, + inspectByURL: map[string]error{ + "s3://bucket/live/file.tsv": nil, + "s3://bucket/missing/file.tsv": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, + "s3://old-bucket/file.tsv": &core.StorageInspectError{ + Kind: core.StorageInspectCredentialMissing, + Message: "no stored bucket credential found for bucket \"old-bucket\"", + }, + }, + } + + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + ExpectedPaths: []string{"data/file.tsv"}, + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if len(report.Findings) != 2 { + t.Fatalf("expected two findings, got %+v", report.Findings) + } + if report.Findings[0].Kind != FindingStaleDuplicateRecord || report.Findings[1].Kind != FindingBrokenAccessURLError { + t.Fatalf("unexpected finding kinds: %+v", report.Findings) + } +} + func TestStorageCleanupAuditRepoOrphanStatuses(t *testing.T) { manager := &fakeCleanupManager{ rows: []models.StorageCleanupRecord{ @@ -72,9 +283,9 @@ func TestStorageCleanupAuditRepoOrphanStatuses(t *testing.T) { "obj-live": cleanupObject("obj-live", "s3://bucket/data/live.tsv"), "obj-stale": cleanupObject("obj-stale", "s3://bucket/data/stale.tsv"), }, - inspectByObject: map[string]error{ - "obj-live": nil, - "obj-stale": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, + inspectByURL: map[string]error{ + "s3://bucket/data/live.tsv": nil, + "s3://bucket/data/stale.tsv": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, }, } svc := NewStorageCleanupService(manager) @@ -94,6 +305,54 @@ func TestStorageCleanupAuditRepoOrphanStatuses(t *testing.T) { } } +func TestStorageCleanupAuditPartialBrokenAccessURLIsAuditOnly(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-partial", NormalizedPath: "data/file.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "obj-partial": cleanupObject("obj-partial", + "s3://broken-bucket/file.tsv", + "s3://bucket/live/file.tsv", + ), + }, + inspectByURL: map[string]error{ + "s3://broken-bucket/file.tsv": &core.StorageInspectError{ + Kind: core.StorageInspectCredentialMissing, + Message: "no stored bucket credential found for bucket \"broken-bucket\"", + }, + "s3://bucket/live/file.tsv": nil, + }, + } + + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + ExpectedPaths: []string{"data/file.tsv"}, + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if len(report.Findings) != 1 { + t.Fatalf("expected one finding, got %+v", report.Findings) + } + finding := report.Findings[0] + if finding.Kind != FindingBrokenAccessURLError { + t.Fatalf("expected broken access url finding, got %s", finding.Kind) + } + if finding.CleanupScope != CleanupScopeAccessURL { + t.Fatalf("expected access_url scope, got %s", finding.CleanupScope) + } + record := finding.Records[0] + if record.StorageStatus != StorageProbeStatusLive { + t.Fatalf("expected live derived status, got %s", record.StorageStatus) + } + if record.CleanupScope != CleanupScopeAccessURL { + t.Fatalf("expected record access_url scope, got %s", record.CleanupScope) + } +} + func TestStorageCleanupApplyDeletesExpectedRecords(t *testing.T) { manager := &fakeCleanupManager{ rows: []models.StorageCleanupRecord{ @@ -108,11 +367,11 @@ func TestStorageCleanupApplyDeletesExpectedRecords(t *testing.T) { "orphan-live": cleanupObject("orphan-live", "s3://bucket/data/orphan.tsv"), "orphan-stale": cleanupObject("orphan-stale", "s3://bucket/data/stale.tsv"), }, - inspectByObject: map[string]error{ - "dup-live": nil, - "dup-missing": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, - "orphan-live": nil, - "orphan-stale": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, + inspectByURL: map[string]error{ + "s3://bucket/dup/live.tsv": nil, + "s3://bucket/dup/missing.tsv": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, + "s3://bucket/data/orphan.tsv": nil, + "s3://bucket/data/stale.tsv": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, }, } svc := NewStorageCleanupService(manager) @@ -140,18 +399,195 @@ func TestStorageCleanupApplyDeletesExpectedRecords(t *testing.T) { } } +func TestStorageCleanupApplySkipsPartialBrokenAccessRecord(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-partial", NormalizedPath: "data/file.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "obj-partial": cleanupObject("obj-partial", + "s3://broken-bucket/file.tsv", + "s3://bucket/live/file.tsv", + ), + }, + inspectByURL: map[string]error{ + "s3://broken-bucket/file.tsv": &core.StorageInspectError{ + Kind: core.StorageInspectCredentialMissing, + Message: "no stored bucket credential found for bucket \"broken-bucket\"", + }, + "s3://bucket/live/file.tsv": nil, + }, + } + + svc := NewStorageCleanupService(manager) + result, err := svc.Apply(context.Background(), StorageCleanupApplyRequest{ + Organization: "org", + Project: "proj", + ExpectedPaths: []string{"data/file.tsv"}, + DeleteStaleDuplicates: true, + DeleteRepoOrphans: true, + }) + if err != nil { + t.Fatalf("Apply failed: %v", err) + } + if len(result.DeletedRecordIDs) != 0 { + t.Fatalf("expected no deleted records, got %+v", result.DeletedRecordIDs) + } +} + +func TestStorageCleanupAuditUsesBulkObjectFetchOnce(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-1", NormalizedPath: "data/a.tsv", Size: 10}, + {ObjectID: "obj-1", NormalizedPath: "data/a.tsv", Size: 10}, + {ObjectID: "obj-2", NormalizedPath: "data/b.tsv", Size: 11}, + }, + objects: map[string]models.InternalObject{ + "obj-1": cleanupObject("obj-1", "s3://bucket/a.tsv"), + "obj-2": cleanupObject("obj-2", "s3://bucket/b.tsv"), + }, + } + + svc := NewStorageCleanupService(manager) + if _, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + }); err != nil { + t.Fatalf("Audit failed: %v", err) + } + if manager.bulkCalls != 1 { + t.Fatalf("expected one bulk object fetch, got %d", manager.bulkCalls) + } +} + +func TestStorageCleanupAuditUsesDuplicateListingWithoutExpectedPaths(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "dup-1", NormalizedPath: "data/a.tsv", Size: 10}, + {ObjectID: "dup-2", NormalizedPath: "data/a.tsv", Size: 10}, + {ObjectID: "single", NormalizedPath: "data/b.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "dup-1": cleanupObject("dup-1", "s3://bucket/data/a.tsv"), + "dup-2": cleanupObject("dup-2", "s3://bucket/data/a-copy.tsv"), + "single": cleanupObject("single", "s3://bucket/data/b.tsv"), + }, + } + + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if report.Scanned != 2 { + t.Fatalf("expected duplicate-only scan count 2, got %d", report.Scanned) + } + if report.ScannedPaths != 1 { + t.Fatalf("expected one scanned path, got %d", report.ScannedPaths) + } + if manager.duplicateListCalls != 1 || manager.listCalls != 0 { + t.Fatalf("expected duplicate listing only, got list=%d duplicate=%d", manager.listCalls, manager.duplicateListCalls) + } +} + +func TestStorageCleanupAuditUsesFullListingForOrphanAudit(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-1", NormalizedPath: "data/a.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "obj-1": cleanupObject("obj-1", "s3://bucket/data/a.tsv"), + }, + } + + svc := NewStorageCleanupService(manager) + if _, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + ExpectedPaths: []string{"data/other.tsv"}, + }); err != nil { + t.Fatalf("Audit failed: %v", err) + } + if manager.listCalls != 1 || manager.duplicateListCalls != 0 { + t.Fatalf("expected full listing only, got list=%d duplicate=%d", manager.listCalls, manager.duplicateListCalls) + } +} + +func TestStorageCleanupAuditCachesBrokenBucketFailures(t *testing.T) { + manager := &fakeCleanupManager{ + rows: []models.StorageCleanupRecord{ + {ObjectID: "obj-1", NormalizedPath: "data/a.tsv", Size: 10}, + {ObjectID: "obj-2", NormalizedPath: "data/a.tsv", Size: 10}, + }, + objects: map[string]models.InternalObject{ + "obj-1": cleanupObject("obj-1", "s3://old-bucket/data/a.tsv"), + "obj-2": cleanupObject("obj-2", "s3://old-bucket/data/b.tsv"), + }, + inspectByURL: map[string]error{ + "s3://old-bucket/data/a.tsv": &core.StorageInspectError{ + Kind: core.StorageInspectCredentialMissing, + Message: "no stored bucket credential found for bucket \"old-bucket\"", + }, + "s3://old-bucket/data/b.tsv": &core.StorageInspectError{ + Kind: core.StorageInspectCredentialMissing, + Message: "no stored bucket credential found for bucket \"old-bucket\"", + }, + }, + } + + svc := NewStorageCleanupService(manager) + report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ + Organization: "org", + Project: "proj", + }) + if err != nil { + t.Fatalf("Audit failed: %v", err) + } + if len(report.Findings) == 0 { + t.Fatalf("expected findings for broken duplicate path") + } + if manager.inspectCalls != 1 { + t.Fatalf("expected one inspect call after broken bucket cache, got %d", manager.inspectCalls) + } +} + type fakeCleanupManager struct { - rows []models.StorageCleanupRecord - objects map[string]models.InternalObject - inspectByObject map[string]error - deleteOpts map[string]core.DeleteOptions + rows []models.StorageCleanupRecord + objects map[string]models.InternalObject + inspectByURL map[string]error + deleteOpts map[string]core.DeleteOptions + bulkCalls int + listCalls int + duplicateListCalls int + inspectCalls int } func (f *fakeCleanupManager) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - return append([]models.StorageCleanupRecord(nil), f.rows...), nil + f.listCalls++ + return append([]models.StorageCleanupRecord(nil), filterCleanupRowsByPrefix(f.rows, pathPrefix)...), nil +} + +func (f *fakeCleanupManager) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + f.duplicateListCalls++ + rows := filterCleanupRowsByPrefix(f.rows, pathPrefix) + counts := make(map[string]int, len(rows)) + for _, row := range rows { + counts[row.NormalizedPath]++ + } + out := make([]models.StorageCleanupRecord, 0, len(rows)) + for _, row := range rows { + if counts[row.NormalizedPath] > 1 { + out = append(out, row) + } + } + return out, nil } func (f *fakeCleanupManager) GetBulkObjects(ctx context.Context, ids []string, requiredMethod string) ([]models.InternalObject, error) { + f.bulkCalls++ out := make([]models.InternalObject, 0, len(ids)) for _, id := range ids { if obj, ok := f.objects[id]; ok { @@ -162,14 +598,10 @@ func (f *fakeCleanupManager) GetBulkObjects(ctx context.Context, ids []string, r } func (f *fakeCleanupManager) InspectStorageObject(ctx context.Context, req core.InspectStorageRequest) (*core.StorageObjectMetadata, error) { - for id, obj := range f.objects { - for _, method := range derefServerMethods(obj.AccessMethods) { - if method.AccessUrl != nil && method.AccessUrl.Url == req.ObjectURL { - if err, ok := f.inspectByObject[id]; ok { - return nil, err - } - return &core.StorageObjectMetadata{ObjectURL: req.ObjectURL}, nil - } + f.inspectCalls++ + if err, ok := f.inspectByURL[req.ObjectURL]; ok { + if err != nil { + return nil, err } } return &core.StorageObjectMetadata{ObjectURL: req.ObjectURL}, nil @@ -183,14 +615,17 @@ func (f *fakeCleanupManager) DeleteObjectWithOptions(ctx context.Context, id str return nil } -func cleanupObject(id, url string) models.InternalObject { - methods := []drsapi.AccessMethod{{ - Type: drsapi.AccessMethodTypeS3, - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: url}, - }} +func cleanupObject(id string, urls ...string) models.InternalObject { + methods := make([]drsapi.AccessMethod, 0, len(urls)) + for _, url := range urls { + methods = append(methods, drsapi.AccessMethod{ + Type: drsapi.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: url}, + }) + } return models.InternalObject{ DrsObject: drsapi.DrsObject{ Id: id, @@ -199,9 +634,30 @@ func cleanupObject(id, url string) models.InternalObject { } } -func derefServerMethods(methods *[]drsapi.AccessMethod) []drsapi.AccessMethod { - if methods == nil { +func cleanupObjectWithChecksum(id string, checksum string, urls ...string) models.InternalObject { + obj := cleanupObject(id, urls...) + obj.Checksums = []drsapi.Checksum{{ + Type: "sha256", + Checksum: checksum, + }} + return obj +} + +func filterCleanupRowsByPrefix(rows []models.StorageCleanupRecord, pathPrefix string) []models.StorageCleanupRecord { + normalizedPrefix, prefixSegments, err := common.NormalizeBrowsePath(pathPrefix) + if err != nil { return nil } - return *methods + if normalizedPrefix == "" { + return append([]models.StorageCleanupRecord(nil), rows...) + } + out := make([]models.StorageCleanupRecord, 0, len(rows)) + for _, row := range rows { + info, ok, err := common.BrowsePathInfoFromName(row.NormalizedPath) + if err != nil || !ok || !common.HasBrowsePathPrefix(info.Segments, prefixSegments) { + continue + } + out = append(out, row) + } + return out } diff --git a/internal/repair/storage_cleanup_types.go b/internal/repair/storage_cleanup_types.go index 16151850..8a897911 100644 --- a/internal/repair/storage_cleanup_types.go +++ b/internal/repair/storage_cleanup_types.go @@ -14,39 +14,65 @@ const ( const ( FindingStaleDuplicateRecord FindingKind = "stale_duplicate_record" FindingLiveDuplicateConflict FindingKind = "live_duplicate_conflict" + FindingBrokenAccessURLError FindingKind = "broken_access_url_error" FindingRepoOrphanLiveObject FindingKind = "repo_orphan_live_object" FindingRepoOrphanStaleRecord FindingKind = "repo_orphan_stale_record" ) +type CleanupScope string + +const ( + CleanupScopeRecord CleanupScope = "record" + CleanupScopeAccessURL CleanupScope = "access_url" +) + +type StorageCleanupAccessProbe struct { + URL string `json:"url"` + Bucket string `json:"bucket,omitempty"` + StorageStatus StorageProbeStatus `json:"storage_status"` + StorageMessage string `json:"storage_message,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` +} + type StorageCleanupRecordAudit struct { - ObjectID string `json:"object_id"` - NormalizedPath string `json:"normalized_path"` - Size int64 `json:"size"` - UpdatedTime *time.Time `json:"updated_time,omitempty"` - DownloadCount int64 `json:"download_count"` - LastDownloadTime *time.Time `json:"last_download_time,omitempty"` - CurrentAccessURLs []string `json:"current_access_urls,omitempty"` - StorageStatus StorageProbeStatus `json:"storage_status"` - StorageMessage string `json:"storage_message,omitempty"` + ObjectID string `json:"object_id"` + NormalizedPath string `json:"normalized_path"` + Size int64 `json:"size"` + UpdatedTime *time.Time `json:"updated_time,omitempty"` + DownloadCount int64 `json:"download_count"` + LastDownloadTime *time.Time `json:"last_download_time,omitempty"` + CurrentAccessURLs []string `json:"current_access_urls,omitempty"` + AccessProbes []StorageCleanupAccessProbe `json:"access_probes,omitempty"` + CleanupScope CleanupScope `json:"cleanup_scope,omitempty"` + StorageStatus StorageProbeStatus `json:"storage_status"` + StorageMessage string `json:"storage_message,omitempty"` } type StorageCleanupFinding struct { - Kind FindingKind `json:"kind"` - Severity Severity `json:"severity"` - NormalizedPath string `json:"normalized_path"` - Message string `json:"message"` - RecommendedAction string `json:"recommended_action,omitempty"` - RepoDeleteCandidate bool `json:"repo_delete_candidate"` - Records []StorageCleanupRecordAudit `json:"records,omitempty"` + Kind FindingKind `json:"kind"` + Severity Severity `json:"severity"` + NormalizedPath string `json:"normalized_path"` + Message string `json:"message"` + RecommendedAction string `json:"recommended_action,omitempty"` + RepoDeleteCandidate bool `json:"repo_delete_candidate"` + CleanupScope CleanupScope `json:"cleanup_scope,omitempty"` + StructuralReason string `json:"structural_reason,omitempty"` + ChecksumCount int `json:"checksum_count,omitempty"` + SizeCount int `json:"size_count,omitempty"` + LegacyURLTemplateDetected bool `json:"legacy_url_template_detected,omitempty"` + Records []StorageCleanupRecordAudit `json:"records,omitempty"` } type StorageCleanupReport struct { - Organization string `json:"organization"` - Project string `json:"project"` - PathPrefix string `json:"path_prefix,omitempty"` - Scanned int `json:"scanned"` - Summary map[FindingKind]int `json:"summary,omitempty"` - Findings []StorageCleanupFinding `json:"findings,omitempty"` + Organization string `json:"organization"` + Project string `json:"project"` + PathPrefix string `json:"path_prefix,omitempty"` + Scanned int `json:"scanned"` + ScannedPaths int `json:"scanned_paths,omitempty"` + ClassifiedPaths int `json:"classified_paths,omitempty"` + UnclassifiedPaths int `json:"unclassified_paths,omitempty"` + Summary map[FindingKind]int `json:"summary,omitempty"` + Findings []StorageCleanupFinding `json:"findings,omitempty"` } type StorageCleanupAuditRequest struct { @@ -54,6 +80,7 @@ type StorageCleanupAuditRequest struct { Project string `json:"project"` PathPrefix string `json:"path_prefix,omitempty"` ExpectedPaths []string `json:"expected_paths,omitempty"` + SelectedPaths []string `json:"selected_paths,omitempty"` CheckStorage *bool `json:"check_storage,omitempty"` } diff --git a/internal/testutils/mocks.go b/internal/testutils/mocks.go index a2d4d26c..5e681dc8 100644 --- a/internal/testutils/mocks.go +++ b/internal/testutils/mocks.go @@ -206,6 +206,24 @@ func (m *MockDatabase) ListStorageCleanupRecords(ctx context.Context, organizati return rows, nil } +func (m *MockDatabase) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { + rows, err := m.ListStorageCleanupRecords(ctx, organization, project, pathPrefix) + if err != nil { + return nil, err + } + counts := make(map[string]int, len(rows)) + for _, row := range rows { + counts[row.NormalizedPath]++ + } + out := make([]models.StorageCleanupRecord, 0, len(rows)) + for _, row := range rows { + if counts[row.NormalizedPath] > 1 { + out = append(out, row) + } + } + return out, nil +} + func (m *MockDatabase) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { normalizedPath, pathSegments, err := common.NormalizeBrowsePath(path) if err != nil { From 369fd83fb2984a450b16f08d21be80f2fb9eef0d Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 26 Jun 2026 09:58:48 -0700 Subject: [PATCH 03/24] fix issues related db allowing multiple credentials for the same bucket name in the same storage url --- client/services/repair_service.go | 30 ++ client/services/repair_service_test.go | 64 +++++ client/services/types.go | 8 + cmd/repair/main.go | 78 +++++- cmd/repair/main_test.go | 20 ++ internal/api/internaldrs/index.go | 19 +- internal/api/internaldrs/index_test.go | 37 +++ internal/api/internaldrs/repair_cleanup.go | 257 ++++++++++++++++++ internal/common/routes.go | 2 + internal/core/converters.go | 26 +- internal/core/converters_test.go | 17 ++ internal/db/postgres/credentials.go | 27 ++ .../db/postgres/credentials_sqlmock_test.go | 35 +++ internal/db/postgres/postgres.go | 17 ++ internal/db/postgres/postgres_schema_test.go | 17 ++ internal/db/sqlite/credentials.go | 27 ++ internal/db/sqlite/sqlite.go | 22 ++ internal/db/sqlite/sqlite_test.go | 104 ++++++- internal/repair/types.go | 4 + 19 files changed, 791 insertions(+), 20 deletions(-) create mode 100644 client/services/repair_service_test.go diff --git a/client/services/repair_service.go b/client/services/repair_service.go index 6e5bcd9d..57d64e91 100644 --- a/client/services/repair_service.go +++ b/client/services/repair_service.go @@ -71,3 +71,33 @@ func (s *RepairService) StorageCleanupApply(ctx context.Context, opts StorageCle } return out, nil } + +func (s *RepairService) ScopeAudit(ctx context.Context, opts ScopeRepairOptions) (repair.Report, error) { + req := repair.Options{ + Organization: opts.Organization, + Project: opts.ProjectID, + CheckStorage: opts.CheckStorage, + Limit: opts.Limit, + PageSize: opts.PageSize, + } + var out repair.Report + if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairScopeAudit, req, &out); err != nil { + return repair.Report{}, err + } + return out, nil +} + +func (s *RepairService) ScopeApply(ctx context.Context, opts ScopeRepairOptions) (repair.ApplyResult, error) { + req := repair.Options{ + Organization: opts.Organization, + Project: opts.ProjectID, + CheckStorage: opts.CheckStorage, + Limit: opts.Limit, + PageSize: opts.PageSize, + } + var out repair.ApplyResult + if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairScopeApply, req, &out); err != nil { + return repair.ApplyResult{}, err + } + return out, nil +} diff --git a/client/services/repair_service_test.go b/client/services/repair_service_test.go new file mode 100644 index 00000000..229757a1 --- /dev/null +++ b/client/services/repair_service_test.go @@ -0,0 +1,64 @@ +package services + +import ( + "context" + "testing" + + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/repair" +) + +func TestRepairServiceScopeAuditUsesScopeRoute(t *testing.T) { + requester := &fakeRequester{responseJSON: []byte(`{"organization":"org","project":"proj","scanned":1}`)} + service := NewRepairService(requester) + + report, err := service.ScopeAudit(context.Background(), ScopeRepairOptions{ + Organization: "org", + ProjectID: "proj", + CheckStorage: true, + Limit: 10, + PageSize: 5, + }) + if err != nil { + t.Fatalf("ScopeAudit returned error: %v", err) + } + if requester.path != common.RouteInternalRepairScopeAudit { + t.Fatalf("expected scope audit path %q, got %q", common.RouteInternalRepairScopeAudit, requester.path) + } + body, ok := requester.body.(repair.Options) + if !ok { + t.Fatalf("expected repair.Options body, got %T", requester.body) + } + if body.Organization != "org" || body.Project != "proj" || body.Limit != 10 || body.PageSize != 5 || !body.CheckStorage { + t.Fatalf("unexpected scope audit body: %+v", body) + } + if report.Organization != "org" || report.Project != "proj" || report.Scanned != 1 { + t.Fatalf("unexpected scope audit report: %+v", report) + } +} + +func TestRepairServiceScopeApplyUsesScopeRoute(t *testing.T) { + requester := &fakeRequester{responseJSON: []byte(`{"report":{"organization":"org","project":"proj","scanned":1},"mutated":1,"auto_fixable":1}`)} + service := NewRepairService(requester) + + result, err := service.ScopeApply(context.Background(), ScopeRepairOptions{ + Organization: "org", + ProjectID: "proj", + }) + if err != nil { + t.Fatalf("ScopeApply returned error: %v", err) + } + if requester.path != common.RouteInternalRepairScopeApply { + t.Fatalf("expected scope apply path %q, got %q", common.RouteInternalRepairScopeApply, requester.path) + } + body, ok := requester.body.(repair.Options) + if !ok { + t.Fatalf("expected repair.Options body, got %T", requester.body) + } + if body.Organization != "org" || body.Project != "proj" { + t.Fatalf("unexpected scope apply body: %+v", body) + } + if result.Mutated != 1 || result.AutoFixable != 1 || result.Report.Organization != "org" { + t.Fatalf("unexpected scope apply result: %+v", result) + } +} diff --git a/client/services/types.go b/client/services/types.go index 5e5cb81f..4f99d5fc 100644 --- a/client/services/types.go +++ b/client/services/types.go @@ -110,3 +110,11 @@ type StorageCleanupApplyOptions struct { SelectedFindingKinds []string CheckStorage *bool } + +type ScopeRepairOptions struct { + Organization string + ProjectID string + CheckStorage bool + Limit int + PageSize int +} diff --git a/cmd/repair/main.go b/cmd/repair/main.go index 2da77ad7..10d1cbcb 100644 --- a/cmd/repair/main.go +++ b/cmd/repair/main.go @@ -18,6 +18,8 @@ import ( type auditRunner func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error type applyRunner func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error +type scopeAuditRunner func(cmd *cobra.Command, opts repairsvc.Options) error +type scopeApplyRunner func(cmd *cobra.Command, opts repairsvc.Options) error type commandOptions struct { Organization string @@ -35,9 +37,9 @@ type commandOptions struct { SelectedFindingKinds []string } -var Cmd = newCommand(runAudit, runApply) +var Cmd = newCommand(runAudit, runApply, runScopeAudit, runScopeApply) -func newCommand(audit auditRunner, apply applyRunner) *cobra.Command { +func newCommand(audit auditRunner, apply applyRunner, scopeAudit scopeAuditRunner, scopeApply scopeApplyRunner) *cobra.Command { var opts commandOptions cmd := &cobra.Command{ Use: "repair", @@ -79,6 +81,34 @@ func newCommand(audit auditRunner, apply applyRunner) *cobra.Command { return apply(cmd, req) }, }) + cmd.AddCommand(&cobra.Command{ + Use: "scope-audit", + Short: "Audit records missing canonical controlled_access for a project scope", + PreRunE: func(cmd *cobra.Command, args []string) error { + req := normalizedScopeOptions(opts) + if req.Organization == "" || req.Project == "" { + return fmt.Errorf("repair scope-audit requires --organization and --project") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + return scopeAudit(cmd, normalizedScopeOptions(opts)) + }, + }) + cmd.AddCommand(&cobra.Command{ + Use: "scope-apply", + Short: "Backfill canonical controlled_access for a project scope", + PreRunE: func(cmd *cobra.Command, args []string) error { + req := normalizedScopeOptions(opts) + if req.Organization == "" || req.Project == "" { + return fmt.Errorf("repair scope-apply requires --organization and --project") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + return scopeApply(cmd, normalizedScopeOptions(opts)) + }, + }) return cmd } @@ -138,6 +168,14 @@ func normalizedApplyOptions(opts commandOptions) (repairsvc.StorageCleanupApplyR }, nil } +func normalizedScopeOptions(opts commandOptions) repairsvc.Options { + return repairsvc.Options{ + Organization: strings.TrimSpace(opts.Organization), + Project: strings.TrimSpace(opts.Project), + CheckStorage: opts.CheckStorage, + } +} + func runAudit(cmd *cobra.Command, req repairsvc.StorageCleanupAuditRequest) error { client, err := buildClient(cmd) if err != nil { @@ -162,6 +200,42 @@ func runApply(cmd *cobra.Command, req repairsvc.StorageCleanupApplyRequest) erro return repairsvc.WriteStorageCleanupApply(cmd.OutOrStdout(), normalizedFormat(cmd), result) } +func runScopeAudit(cmd *cobra.Command, req repairsvc.Options) error { + client, err := buildClient(cmd) + if err != nil { + return err + } + report, err := client.Repair().ScopeAudit(cmd.Context(), services.ScopeRepairOptions{ + Organization: req.Organization, + ProjectID: req.Project, + CheckStorage: req.CheckStorage, + Limit: req.Limit, + PageSize: req.PageSize, + }) + if err != nil { + return err + } + return repairsvc.WriteAudit(cmd.OutOrStdout(), normalizedFormat(cmd), report) +} + +func runScopeApply(cmd *cobra.Command, req repairsvc.Options) error { + client, err := buildClient(cmd) + if err != nil { + return err + } + result, err := client.Repair().ScopeApply(cmd.Context(), services.ScopeRepairOptions{ + Organization: req.Organization, + ProjectID: req.Project, + CheckStorage: req.CheckStorage, + Limit: req.Limit, + PageSize: req.PageSize, + }) + if err != nil { + return err + } + return repairsvc.WriteApply(cmd.OutOrStdout(), normalizedFormat(cmd), result) +} + func buildClient(cmd *cobra.Command) (*syclient.Client, error) { client, err := cliauth.NewServerClient(cmd) if err != nil { diff --git a/cmd/repair/main_test.go b/cmd/repair/main_test.go index 19b99872..7f6a45ca 100644 --- a/cmd/repair/main_test.go +++ b/cmd/repair/main_test.go @@ -11,6 +11,8 @@ func TestApplyRequiresScope(t *testing.T) { cmd := newCommand( func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { return nil }, func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, ) cmd.SetArgs([]string{"apply", "--delete-stale-duplicates"}) err := cmd.Execute() @@ -30,6 +32,8 @@ func TestAuditCollectsCheckStorageFlag(t *testing.T) { return nil }, func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, ) cmd.SetArgs([]string{"audit", "--check-storage"}) if err := cmd.Execute(); err != nil { @@ -44,6 +48,8 @@ func TestApplyRequiresDeleteMode(t *testing.T) { cmd := newCommand( func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { return nil }, func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, ) cmd.SetArgs([]string{"apply", "--organization", "org", "--project", "proj"}) err := cmd.Execute() @@ -51,3 +57,17 @@ func TestApplyRequiresDeleteMode(t *testing.T) { t.Fatalf("expected delete mode validation error, got %v", err) } } + +func TestScopeApplyRequiresScope(t *testing.T) { + cmd := newCommand( + func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, + func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, + ) + cmd.SetArgs([]string{"scope-apply"}) + err := cmd.Execute() + if err == nil || err.Error() != "repair scope-apply requires --organization and --project" { + t.Fatalf("expected scope apply validation error, got %v", err) + } +} diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index e595ede0..59dffcca 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -44,6 +44,8 @@ func RegisterInternalRoutes(router fiber.Router, om *core.ObjectManager) { router.Post(common.RouteInternalRepairProjectDiff, handleInternalProjectDiffAuditFiber(om)) router.Post(common.RouteInternalRepairCleanupAudit, handleInternalStorageCleanupAuditFiber(om)) router.Post(common.RouteInternalRepairCleanupApply, handleInternalStorageCleanupApplyFiber(om)) + router.Post(common.RouteInternalRepairScopeAudit, handleInternalScopeRepairAuditFiber(om)) + router.Post(common.RouteInternalRepairScopeApply, handleInternalScopeRepairApplyFiber(om)) registerInternalTransferRoutes(router, om) } @@ -371,23 +373,30 @@ func handleInternalBulkDeleteFiber(om *core.ObjectManager) fiber.Handler { func handleInternalUpdateFiber(c fiber.Ctx, om *core.ObjectManager) error { id := c.Params("id") - var req models.InternalObject - if err := c.Bind().JSON(&req); err != nil { - return c.Status(fiber.StatusBadRequest).SendString("Invalid request body") + var req internalapi.InternalRecord + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return c.Status(fiber.StatusBadRequest).SendString("Invalid request body: " + err.Error()) + } + if strings.TrimSpace(req.Did) == "" { + req.Did = id + } + update, err := core.InternalRecordToInternalObject(req, time.Now().UTC()) + if err != nil { + return c.Status(fiber.StatusBadRequest).SendString("Invalid request body: " + err.Error()) } existing, err := om.GetObject(c.Context(), id, "update") if err != nil { return apiutil.HandleError(c, err) } - merged, err := core.MergeInternalObjectUpdate(*existing, req, id, time.Now().UTC()) + merged, err := core.MergeInternalObjectUpdate(*existing, update, id, time.Now().UTC()) if err != nil { return apiutil.HandleError(c, err) } if err := om.ReplaceObjects(c.Context(), []models.InternalObject{merged}); err != nil { return apiutil.HandleError(c, err) } - return c.JSON(merged) + return c.JSON(core.InternalObjectToInternalRecordResponse(merged)) } func parseScopeQueryParts(organization, program, project string) (string, string, bool, error) { diff --git a/internal/api/internaldrs/index_test.go b/internal/api/internaldrs/index_test.go index ad0616d1..789f5c80 100644 --- a/internal/api/internaldrs/index_test.go +++ b/internal/api/internaldrs/index_test.go @@ -623,6 +623,43 @@ func TestHandleInternalBulkCreate_PersistsControlledAccess(t *testing.T) { } } +func TestHandleInternalBulkCreate_OrganizationProjectAddsCanonicalControlledAccess(t *testing.T) { + mockDB := &testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}} + reqBody := `{"records":[{"did":"obj-bulk-2","organization":"test","project":"p2","size":7,"access_methods":[{"type":"s3","access_url":{"url":"s3://bucket/path/obj-bulk-2"}}]}]}` + req := httptest.NewRequest(http.MethodPost, "/index/bulk", strings.NewReader(reqBody)) + req.Header.Set("Content-Type", "application/json") + om := core.NewObjectManager(mockDB, &testutils.MockUrlManager{}) + rr := doInternalDRSTestRequest(req, om) + + if rr.Code != http.StatusCreated { + t.Fatalf("expected 201, got %d body=%s", rr.Code, rr.Body.String()) + } + if got := mockDB.ObjectAuthz["obj-bulk-2"]; len(got["test"]) != 1 || got["test"][0] != "p2" { + t.Fatalf("expected canonical project authz, got %v", got) + } +} + +func TestHandleInternalUpdate_OrganizationProjectAddsCanonicalControlledAccess(t *testing.T) { + now := time.Now().UTC() + mockDB := &testutils.MockDatabase{ + Objects: map[string]*drs.DrsObject{ + "obj-update": {Id: "obj-update", CreatedTime: now, UpdatedTime: &now}, + }, + } + reqBody := `{"did":"obj-update","organization":"test","project":"p3"}` + req := httptest.NewRequest(http.MethodPut, "/index/obj-update", strings.NewReader(reqBody)) + req.Header.Set("Content-Type", "application/json") + om := core.NewObjectManager(mockDB, &testutils.MockUrlManager{}) + rr := doInternalDRSTestRequest(req, om) + + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + if got := mockDB.ObjectAuthz["obj-update"]; len(got["test"]) != 1 || got["test"][0] != "p3" { + t.Fatalf("expected canonical project authz after update, got %v", got) + } +} + func TestHandleInternalBulkCreate_AllowsCreateAccessForAnyControlledAccessScope(t *testing.T) { ctx := indexTestAuthContext(context.Background(), "gen3", true, map[string]map[string]bool{ "/programs/test/projects/p1": {"create": true}, diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go index 543f7e58..60361c4d 100644 --- a/internal/api/internaldrs/repair_cleanup.go +++ b/internal/api/internaldrs/repair_cleanup.go @@ -2,14 +2,22 @@ package internaldrs import ( "context" + "fmt" "strings" + "time" + "github.com/calypr/syfon/apigen/client/bucketapi" + clientdrs "github.com/calypr/syfon/apigen/client/drs" + clientinternalapi "github.com/calypr/syfon/apigen/client/internalapi" + serverdrs "github.com/calypr/syfon/apigen/server/drs" + serverinternalapi "github.com/calypr/syfon/apigen/server/internalapi" sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/api/apiutil" apimiddleware "github.com/calypr/syfon/internal/api/middleware" "github.com/calypr/syfon/internal/authz" intcommon "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/models" "github.com/calypr/syfon/internal/repair" "github.com/gofiber/fiber/v3" ) @@ -108,3 +116,252 @@ func authorizeStorageCleanupScope(ctx context.Context, organization, project str } return intcommon.ErrUnauthorized } + +func handleInternalScopeRepairAuditFiber(om *core.ObjectManager) fiber.Handler { + svc := repair.NewService(scopeRepairIndexAdapter{om: om}, scopeRepairBucketsAdapter{om: om}, nil) + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req repair.Options + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + req.Organization = strings.TrimSpace(req.Organization) + req.Project = strings.TrimSpace(req.Project) + req.CheckStorage = false + if req.Organization == "" || req.Project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read"); err != nil { + return apiutil.HandleError(c, err) + } + report, err := svc.Audit(c.Context(), req) + if err != nil { + return apiutil.HandleError(c, err) + } + return c.JSON(report) + } +} + +func handleInternalScopeRepairApplyFiber(om *core.ObjectManager) fiber.Handler { + svc := repair.NewService(scopeRepairIndexAdapter{om: om}, scopeRepairBucketsAdapter{om: om}, nil) + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req repair.Options + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + req.Organization = strings.TrimSpace(req.Organization) + req.Project = strings.TrimSpace(req.Project) + req.CheckStorage = false + if req.Organization == "" || req.Project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read", "update"); err != nil { + return apiutil.HandleError(c, err) + } + result, err := svc.Apply(c.Context(), req) + if err != nil { + return apiutil.HandleError(c, err) + } + return c.JSON(result) + } +} + +type scopeRepairIndexAdapter struct { + om *core.ObjectManager +} + +func (a scopeRepairIndexAdapter) List(ctx context.Context, opts repair.ListRecordsOptions) (clientinternalapi.ListRecordsResponse, error) { + ids, err := a.om.ListObjectIDsPageByScope(ctx, "", "", "read", strings.TrimSpace(opts.Start), opts.Limit, 0) + if err != nil { + return clientinternalapi.ListRecordsResponse{}, err + } + objects, err := a.om.GetBulkObjects(ctx, ids, "read") + if err != nil { + return clientinternalapi.ListRecordsResponse{}, err + } + records := make([]clientinternalapi.InternalRecord, 0, len(objects)) + for _, obj := range objects { + records = append(records, clientRecordFromServer(core.InternalObjectToInternalRecord(obj))) + } + return clientinternalapi.ListRecordsResponse{Records: &records}, nil +} + +func (a scopeRepairIndexAdapter) Update(ctx context.Context, did string, rec clientinternalapi.InternalRecord) (clientinternalapi.InternalRecordResponse, error) { + update, err := core.InternalRecordToInternalObject(serverRecordFromClient(rec), time.Now().UTC()) + if err != nil { + return clientinternalapi.InternalRecordResponse{}, err + } + existing, err := a.om.GetObject(ctx, did, "update") + if err != nil { + return clientinternalapi.InternalRecordResponse{}, err + } + merged, err := core.MergeInternalObjectUpdate(*existing, update, did, time.Now().UTC()) + if err != nil { + return clientinternalapi.InternalRecordResponse{}, err + } + if err := a.om.ReplaceObjects(ctx, []models.InternalObject{merged}); err != nil { + return clientinternalapi.InternalRecordResponse{}, err + } + return clientRecordResponseFromServer(core.InternalObjectToInternalRecordResponse(merged)), nil +} + +type scopeRepairBucketsAdapter struct { + om *core.ObjectManager +} + +func (a scopeRepairBucketsAdapter) List(ctx context.Context) (bucketapi.BucketsResponse, error) { + creds, err := a.om.ListS3Credentials(ctx) + if err != nil { + return bucketapi.BucketsResponse{}, err + } + out := bucketapi.BucketsResponse{S3BUCKETS: map[string]bucketapi.BucketMetadata{}} + for _, cred := range creds { + out.S3BUCKETS[strings.TrimSpace(cred.Bucket)] = bucketapi.BucketMetadata{} + } + return out, nil +} + +func (a scopeRepairBucketsAdapter) ListScopes(ctx context.Context, bucket string) ([]bucketapi.BucketScopeResponse, error) { + scopes, err := a.om.ListBucketScopes(ctx) + if err != nil { + return nil, err + } + out := make([]bucketapi.BucketScopeResponse, 0) + for _, scope := range scopes { + if strings.TrimSpace(scope.Bucket) != strings.TrimSpace(bucket) { + continue + } + path := "" + if strings.TrimSpace(scope.PathPrefix) != "" { + path = fmt.Sprintf("s3://%s/%s", scope.Bucket, strings.Trim(strings.TrimSpace(scope.PathPrefix), "/")) + } else { + path = fmt.Sprintf("s3://%s", scope.Bucket) + } + out = append(out, bucketapi.BucketScopeResponse{ + Organization: scope.Organization, + ProjectId: scope.ProjectID, + Path: &path, + }) + } + return out, nil +} + +func clientRecordFromServer(rec serverinternalapi.InternalRecord) clientinternalapi.InternalRecord { + out := clientinternalapi.InternalRecord{ + Did: rec.Did, + AccessMethods: clientAccessMethodsFromServer(rec.AccessMethods), + ControlledAccess: rec.ControlledAccess, + Size: rec.Size, + CreatedTime: rec.CreatedTime, + Description: rec.Description, + FileName: rec.FileName, + Name: rec.Name, + Version: rec.Version, + UpdatedTime: rec.UpdatedTime, + Hashes: (*clientinternalapi.HashInfo)(rec.Hashes), + Organization: rec.Organization, + Project: rec.Project, + } + return out +} + +func serverRecordFromClient(rec clientinternalapi.InternalRecord) serverinternalapi.InternalRecord { + out := serverinternalapi.InternalRecord{ + Did: rec.Did, + AccessMethods: serverAccessMethodsFromClient(rec.AccessMethods), + ControlledAccess: rec.ControlledAccess, + Size: rec.Size, + CreatedTime: rec.CreatedTime, + Description: rec.Description, + FileName: rec.FileName, + Name: rec.Name, + Version: rec.Version, + UpdatedTime: rec.UpdatedTime, + Hashes: (*serverinternalapi.HashInfo)(rec.Hashes), + Organization: rec.Organization, + Project: rec.Project, + } + return out +} + +func clientRecordResponseFromServer(rec serverinternalapi.InternalRecordResponse) clientinternalapi.InternalRecordResponse { + out := clientinternalapi.InternalRecordResponse{ + Did: rec.Did, + AccessMethods: clientAccessMethodsFromServer(rec.AccessMethods), + ControlledAccess: rec.ControlledAccess, + Size: rec.Size, + CreatedTime: rec.CreatedTime, + Description: rec.Description, + FileName: rec.FileName, + Name: rec.Name, + Version: rec.Version, + UpdatedTime: rec.UpdatedTime, + Hashes: (*clientinternalapi.HashInfo)(rec.Hashes), + Organization: rec.Organization, + Project: rec.Project, + } + return out +} + +func clientAccessMethodsFromServer(in *[]serverdrs.AccessMethod) *[]clientdrs.AccessMethod { + if in == nil { + return nil + } + out := make([]clientdrs.AccessMethod, 0, len(*in)) + for _, method := range *in { + var accessURL *struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + } + if method.AccessUrl != nil { + accessURL = &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{ + Headers: method.AccessUrl.Headers, + Url: method.AccessUrl.Url, + } + } + out = append(out, clientdrs.AccessMethod{ + AccessId: method.AccessId, + AccessUrl: accessURL, + Region: method.Region, + Type: clientdrs.AccessMethodType(method.Type), + }) + } + return &out +} + +func serverAccessMethodsFromClient(in *[]clientdrs.AccessMethod) *[]serverdrs.AccessMethod { + if in == nil { + return nil + } + out := make([]serverdrs.AccessMethod, 0, len(*in)) + for _, method := range *in { + var accessURL *struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + } + if method.AccessUrl != nil { + accessURL = &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{ + Headers: method.AccessUrl.Headers, + Url: method.AccessUrl.Url, + } + } + out = append(out, serverdrs.AccessMethod{ + AccessId: method.AccessId, + AccessUrl: accessURL, + Region: method.Region, + Type: serverdrs.AccessMethodType(method.Type), + }) + } + return &out +} diff --git a/internal/common/routes.go b/internal/common/routes.go index bc26c2b7..a7c94788 100644 --- a/internal/common/routes.go +++ b/internal/common/routes.go @@ -17,6 +17,8 @@ const ( RouteInternalRepairCleanupAudit = "/data/repair/storage-cleanup/audit" RouteInternalRepairCleanupApply = "/data/repair/storage-cleanup/apply" RouteInternalRepairProjectDiff = "/data/repair/project-diff/audit" + RouteInternalRepairScopeAudit = "/data/repair/project-scope/audit" + RouteInternalRepairScopeApply = "/data/repair/project-scope/apply" RouteInternalIndex = "/index" RouteInternalIndexDetail = "/index/{id}" diff --git a/internal/core/converters.go b/internal/core/converters.go index 9c1c7302..ce4db2fa 100644 --- a/internal/core/converters.go +++ b/internal/core/converters.go @@ -14,6 +14,27 @@ import ( "github.com/calypr/syfon/internal/models" ) +func EnforceCanonicalProjectScope(obj models.InternalObject, organization, project string) (models.InternalObject, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if project != "" && organization == "" { + return models.InternalObject{}, fmt.Errorf("organization is required when project is set") + } + if organization == "" || project == "" { + return obj, nil + } + + resource, err := syfoncommon.ResourcePath(organization, project) + if err != nil { + return models.InternalObject{}, err + } + controlled := append(ObjectAccessResources(&obj), resource) + controlled = syfoncommon.NormalizeAccessResources(controlled) + obj.ControlledAccess = &controlled + obj.Authorizations = syfoncommon.ControlledAccessToAuthzMap(controlled) + return obj, nil +} + // LFSCandidateToDRS converts an LFS-specific candidate to a DRS-generic one. func LFSCandidateToDRS(in lfsapi.DrsObjectCandidate) drs.DrsObjectCandidate { aliases := append([]string(nil), common.DerefStringSlice(in.Aliases)...) @@ -263,13 +284,14 @@ func InternalRecordToInternalObject(r internalapi.InternalRecord, now time.Time) methods := append([]drs.AccessMethod(nil), (*r.AccessMethods)...) obj.AccessMethods = &methods } - return models.InternalObject{ + internalObj := models.InternalObject{ DrsObject: obj, Authorizations: authzMap, Properties: map[string]interface{}{ "file_name": common.StringVal(r.FileName), }, - }, nil + } + return EnforceCanonicalProjectScope(internalObj, common.StringVal(r.Organization), common.StringVal(r.Project)) } func parseInternalRecordTime(raw *string, fallback time.Time) time.Time { diff --git a/internal/core/converters_test.go b/internal/core/converters_test.go index 5e107eb2..f87c5fcf 100644 --- a/internal/core/converters_test.go +++ b/internal/core/converters_test.go @@ -7,6 +7,7 @@ import ( "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/apigen/server/lfsapi" + "github.com/calypr/syfon/internal/models" ) func TestConverters(t *testing.T) { @@ -95,6 +96,22 @@ func TestConverters(t *testing.T) { t.Fatalf("expected oid-derived id alias, got %+v", got.Aliases) } }) + + t.Run("enforce canonical project scope appends exact controlled access", func(t *testing.T) { + obj, err := EnforceCanonicalProjectScope(models.InternalObject{ + DrsObject: drs.DrsObject{ + Id: "obj-1", + }, + Authorizations: map[string][]string{"other": {"proj"}}, + }, "org", "proj") + if err != nil { + t.Fatalf("EnforceCanonicalProjectScope returned error: %v", err) + } + got := ObjectAccessResources(&obj) + if len(got) != 2 || got[0] != "/organization/other/project/proj" || got[1] != "/organization/org/project/proj" { + t.Fatalf("unexpected controlled access: %+v", got) + } + }) } func strPtr(s string) *string { return &s } diff --git a/internal/db/postgres/credentials.go b/internal/db/postgres/credentials.go index 36ce4461..dddf02b7 100644 --- a/internal/db/postgres/credentials.go +++ b/internal/db/postgres/credentials.go @@ -91,6 +91,10 @@ func (db *PostgresDB) SaveS3Credential(ctx context.Context, cred *models.S3Crede common.AuditS3CredentialAccess(ctx, "write", bucket, wrapped) return wrapped } + if err := db.ensureUniquePhysicalBucket(ctx, stored.CredentialID, stored.Bucket); err != nil { + common.AuditS3CredentialAccess(ctx, "write", stored.Bucket, err) + return err + } _, err = db.db.ExecContext(ctx, ` INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) @@ -113,6 +117,29 @@ func (db *PostgresDB) SaveS3Credential(ctx context.Context, cred *models.S3Crede return nil } +func (db *PostgresDB) ensureUniquePhysicalBucket(ctx context.Context, credentialID, bucket string) error { + credentialID = strings.TrimSpace(credentialID) + bucket = strings.TrimSpace(bucket) + if bucket == "" { + return nil + } + + var existingCredentialID string + err := db.db.QueryRowContext(ctx, ` + SELECT credential_id + FROM s3_credential + WHERE bucket = $1 AND credential_id <> $2 + LIMIT 1 + `, bucket, credentialID).Scan(&existingCredentialID) + if err == nil { + return fmt.Errorf("physical bucket %q is already configured under credential %q; reuse that credential and add a bucket scope instead", bucket, existingCredentialID) + } + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return fmt.Errorf("failed to validate physical bucket uniqueness: %w", err) +} + func (db *PostgresDB) DeleteS3Credential(ctx context.Context, credentialID string) error { resolvedID, err := db.resolveCredentialID(ctx, credentialID) if err != nil { diff --git a/internal/db/postgres/credentials_sqlmock_test.go b/internal/db/postgres/credentials_sqlmock_test.go index 005dbe96..11ee9654 100644 --- a/internal/db/postgres/credentials_sqlmock_test.go +++ b/internal/db/postgres/credentials_sqlmock_test.go @@ -104,6 +104,14 @@ func TestSaveS3Credential(t *testing.T) { defer rawDB.Close() derivedID := common.DeriveCredentialID("b1", "", "us-east-1", "https://s3.example", "ak") + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT credential_id + FROM s3_credential + WHERE bucket = $1 AND credential_id <> $2 + LIMIT 1 + `)). + WithArgs("b1", derivedID). + WillReturnError(sql.ErrNoRows) mock.ExpectExec(regexp.QuoteMeta(` INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) VALUES ($1, $2, $3, $4, $5, $6, $7) @@ -130,6 +138,33 @@ func TestSaveS3Credential(t *testing.T) { } } +func TestSaveS3CredentialRejectsDuplicatePhysicalBucket(t *testing.T) { + t.Setenv(crypto.CredentialMasterKeyEnv, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") + pg, mock, rawDB := newMockPostgresDB(t) + defer rawDB.Close() + + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT credential_id + FROM s3_credential + WHERE bucket = $1 AND credential_id <> $2 + LIMIT 1 + `)). + WithArgs("shared-bucket", "org-b/default"). + WillReturnRows(sqlmock.NewRows([]string{"credential_id"}).AddRow("org-a/default")) + + err := pg.SaveS3Credential(context.Background(), &models.S3Credential{ + CredentialID: "org-b/default", + Bucket: "shared-bucket", + Provider: "s3", + Region: "us-east-1", + AccessKey: "ak", + SecretKey: "sk", + }) + if err == nil || err.Error() != `physical bucket "shared-bucket" is already configured under credential "org-a/default"; reuse that credential and add a bucket scope instead` { + t.Fatalf("expected duplicate physical bucket error, got %v", err) + } +} + func TestDeleteS3Credential(t *testing.T) { t.Run("success", func(t *testing.T) { pg, mock, rawDB := newMockPostgresDB(t) diff --git a/internal/db/postgres/postgres.go b/internal/db/postgres/postgres.go index c60a80d8..d53becf9 100644 --- a/internal/db/postgres/postgres.go +++ b/internal/db/postgres/postgres.go @@ -138,6 +138,23 @@ func (db *PostgresDB) ensureS3CredentialSchema() error { `ALTER TABLE s3_credential DROP CONSTRAINT IF EXISTS s3_credential_pkey`, `ALTER TABLE s3_credential ADD PRIMARY KEY (credential_id)`, `CREATE INDEX IF NOT EXISTS idx_s3_credential_bucket ON s3_credential(bucket)`, + `CREATE OR REPLACE FUNCTION enforce_s3_credential_unique_bucket() RETURNS trigger AS $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM s3_credential + WHERE bucket = NEW.bucket AND credential_id <> NEW.credential_id + ) THEN + RAISE EXCEPTION 'physical bucket "%" is already configured under another credential', NEW.bucket; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql`, + `DROP TRIGGER IF EXISTS s3_credential_unique_bucket_trigger ON s3_credential`, + `CREATE TRIGGER s3_credential_unique_bucket_trigger + BEFORE INSERT OR UPDATE OF bucket, credential_id ON s3_credential + FOR EACH ROW + EXECUTE FUNCTION enforce_s3_credential_unique_bucket()`, } for _, q := range queries { if _, err := db.db.Exec(q); err != nil { diff --git a/internal/db/postgres/postgres_schema_test.go b/internal/db/postgres/postgres_schema_test.go index 5469a874..0fa3bcf5 100644 --- a/internal/db/postgres/postgres_schema_test.go +++ b/internal/db/postgres/postgres_schema_test.go @@ -102,6 +102,23 @@ func TestSchemaEnsurers(t *testing.T) { `ALTER TABLE s3_credential DROP CONSTRAINT IF EXISTS s3_credential_pkey`, `ALTER TABLE s3_credential ADD PRIMARY KEY (credential_id)`, `CREATE INDEX IF NOT EXISTS idx_s3_credential_bucket ON s3_credential(bucket)`, + `CREATE OR REPLACE FUNCTION enforce_s3_credential_unique_bucket() RETURNS trigger AS $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM s3_credential + WHERE bucket = NEW.bucket AND credential_id <> NEW.credential_id + ) THEN + RAISE EXCEPTION 'physical bucket "%" is already configured under another credential', NEW.bucket; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql`, + `DROP TRIGGER IF EXISTS s3_credential_unique_bucket_trigger ON s3_credential`, + `CREATE TRIGGER s3_credential_unique_bucket_trigger + BEFORE INSERT OR UPDATE OF bucket, credential_id ON s3_credential + FOR EACH ROW + EXECUTE FUNCTION enforce_s3_credential_unique_bucket()`, } { mock.ExpectExec(regexp.QuoteMeta(query)).WillReturnResult(sqlmock.NewResult(0, 0)) } diff --git a/internal/db/sqlite/credentials.go b/internal/db/sqlite/credentials.go index 457e8060..ce550bcb 100644 --- a/internal/db/sqlite/credentials.go +++ b/internal/db/sqlite/credentials.go @@ -91,6 +91,10 @@ func (db *SqliteDB) SaveS3Credential(ctx context.Context, cred *models.S3Credent common.AuditS3CredentialAccess(ctx, "write", bucket, wrapped) return wrapped } + if err := db.ensureUniquePhysicalBucket(ctx, stored.CredentialID, stored.Bucket); err != nil { + common.AuditS3CredentialAccess(ctx, "write", stored.Bucket, err) + return err + } // SQLite UPSERT syntax: INSERT INTO ... ON CONFLICT (...) DO UPDATE SET ... _, err = db.db.ExecContext(ctx, ` @@ -114,6 +118,29 @@ func (db *SqliteDB) SaveS3Credential(ctx context.Context, cred *models.S3Credent return nil } +func (db *SqliteDB) ensureUniquePhysicalBucket(ctx context.Context, credentialID, bucket string) error { + credentialID = strings.TrimSpace(credentialID) + bucket = strings.TrimSpace(bucket) + if bucket == "" { + return nil + } + + var existingCredentialID string + err := db.db.QueryRowContext(ctx, ` + SELECT credential_id + FROM s3_credential + WHERE bucket = ? AND credential_id <> ? + LIMIT 1 + `, bucket, credentialID).Scan(&existingCredentialID) + if err == nil { + return fmt.Errorf("physical bucket %q is already configured under credential %q; reuse that credential and add a bucket scope instead", bucket, existingCredentialID) + } + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return fmt.Errorf("failed to validate physical bucket uniqueness: %w", err) +} + func (db *SqliteDB) DeleteS3Credential(ctx context.Context, credentialID string) error { resolvedID, err := db.resolveCredentialID(ctx, credentialID) if err != nil { diff --git a/internal/db/sqlite/sqlite.go b/internal/db/sqlite/sqlite.go index 87422192..0d7fe807 100644 --- a/internal/db/sqlite/sqlite.go +++ b/internal/db/sqlite/sqlite.go @@ -116,6 +116,28 @@ func (db *SqliteDB) initSchema() error { secret_key TEXT, endpoint TEXT )`, + `CREATE TRIGGER IF NOT EXISTS s3_credential_unique_bucket_insert + BEFORE INSERT ON s3_credential + FOR EACH ROW + WHEN EXISTS ( + SELECT 1 + FROM s3_credential + WHERE bucket = NEW.bucket AND credential_id <> NEW.credential_id + ) + BEGIN + SELECT RAISE(ABORT, 'physical bucket is already configured under another credential'); + END`, + `CREATE TRIGGER IF NOT EXISTS s3_credential_unique_bucket_update + BEFORE UPDATE OF bucket, credential_id ON s3_credential + FOR EACH ROW + WHEN EXISTS ( + SELECT 1 + FROM s3_credential + WHERE bucket = NEW.bucket AND credential_id <> NEW.credential_id + ) + BEGIN + SELECT RAISE(ABORT, 'physical bucket is already configured under another credential'); + END`, `CREATE TABLE IF NOT EXISTS bucket_scope ( organization TEXT NOT NULL, project_id TEXT NOT NULL, diff --git a/internal/db/sqlite/sqlite_test.go b/internal/db/sqlite/sqlite_test.go index e771a948..407bfc67 100644 --- a/internal/db/sqlite/sqlite_test.go +++ b/internal/db/sqlite/sqlite_test.go @@ -504,7 +504,7 @@ func TestSqliteDB_S3Credentials(t *testing.T) { } } -func TestSqliteDB_S3CredentialsCredentialIDAllowsSharedPhysicalBucket(t *testing.T) { +func TestSqliteDB_SaveS3CredentialRejectsDuplicatePhysicalBucket(t *testing.T) { t.Setenv(crypto.CredentialMasterKeyEnv, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") ctx := context.Background() db, err := NewSqliteDB(":memory:") @@ -512,26 +512,108 @@ func TestSqliteDB_S3CredentialsCredentialIDAllowsSharedPhysicalBucket(t *testing t.Fatalf("failed to create db: %v", err) } + first := models.S3Credential{ + CredentialID: "org-a/default", + Bucket: "shared-bucket", + Region: "us-east-1", + AccessKey: "key-a", + SecretKey: "secret-a", + } + if err := db.SaveS3Credential(ctx, &first); err != nil { + t.Fatalf("SaveS3Credential(first) failed: %v", err) + } + + second := models.S3Credential{ + CredentialID: "org-b/default", + Bucket: "shared-bucket", + Region: "us-east-1", + AccessKey: "key-b", + SecretKey: "secret-b", + } + err = db.SaveS3Credential(ctx, &second) + if err == nil || !strings.Contains(err.Error(), `physical bucket "shared-bucket" is already configured under credential "org-a/default"`) { + t.Fatalf("expected duplicate physical bucket error, got %v", err) + } +} + +func TestSqliteDB_GetS3CredentialRejectsAmbiguousLegacyPhysicalBucket(t *testing.T) { + t.Setenv(crypto.CredentialMasterKeyEnv, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") + ctx := context.Background() + db, err := NewSqliteDB(":memory:") + if err != nil { + t.Fatalf("failed to create db: %v", err) + } + if _, err := db.db.ExecContext(ctx, `DROP TRIGGER IF EXISTS s3_credential_unique_bucket_insert`); err != nil { + t.Fatalf("drop insert trigger: %v", err) + } + if _, err := db.db.ExecContext(ctx, `DROP TRIGGER IF EXISTS s3_credential_unique_bucket_update`); err != nil { + t.Fatalf("drop update trigger: %v", err) + } + for _, cred := range []models.S3Credential{ {CredentialID: "org-a/default", Bucket: "shared-bucket", Region: "us-east-1", AccessKey: "key-a", SecretKey: "secret-a"}, {CredentialID: "org-b/default", Bucket: "shared-bucket", Region: "us-east-1", AccessKey: "key-b", SecretKey: "secret-b"}, } { - cred := cred - if err := db.SaveS3Credential(ctx, &cred); err != nil { - t.Fatalf("SaveS3Credential(%s) failed: %v", cred.CredentialID, err) + stored, err := crypto.PrepareS3CredentialForStorage(&cred) + if err != nil { + t.Fatalf("PrepareS3CredentialForStorage(%s) failed: %v", cred.CredentialID, err) + } + if _, err := db.db.ExecContext(ctx, ` + INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, stored.CredentialID, stored.Bucket, "s3", stored.Region, stored.AccessKey, stored.SecretKey, stored.Endpoint); err != nil { + t.Fatalf("legacy insert(%s) failed: %v", cred.CredentialID, err) } } - got, err := db.GetS3Credential(ctx, "org-b/default") - if err != nil { - t.Fatalf("GetS3Credential by credential_id failed: %v", err) + if _, err := db.GetS3Credential(ctx, "shared-bucket"); err == nil || !strings.Contains(err.Error(), "multiple credentials") { + t.Fatalf("expected ambiguous physical bucket lookup error, got %v", err) } - if got.Bucket != "shared-bucket" || got.AccessKey != "key-b" { - t.Fatalf("unexpected credential: %+v", got) +} + +func TestSqliteDB_DirectInsertRejectsDuplicatePhysicalBucket(t *testing.T) { + t.Setenv(crypto.CredentialMasterKeyEnv, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") + ctx := context.Background() + db, err := NewSqliteDB(":memory:") + if err != nil { + t.Fatalf("failed to create db: %v", err) } - if _, err := db.GetS3Credential(ctx, "shared-bucket"); err == nil || !strings.Contains(err.Error(), "multiple credentials") { - t.Fatalf("expected ambiguous physical bucket lookup error, got %v", err) + first, err := crypto.PrepareS3CredentialForStorage(&models.S3Credential{ + CredentialID: "org-a/default", + Bucket: "shared-bucket", + Provider: "s3", + Region: "us-east-1", + AccessKey: "key-a", + SecretKey: "secret-a", + }) + if err != nil { + t.Fatalf("PrepareS3CredentialForStorage(first) failed: %v", err) + } + if _, err := db.db.ExecContext(ctx, ` + INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, first.CredentialID, first.Bucket, "s3", first.Region, first.AccessKey, first.SecretKey, first.Endpoint); err != nil { + t.Fatalf("raw first insert failed: %v", err) + } + + second, err := crypto.PrepareS3CredentialForStorage(&models.S3Credential{ + CredentialID: "org-b/default", + Bucket: "shared-bucket", + Provider: "s3", + Region: "us-east-1", + AccessKey: "key-b", + SecretKey: "secret-b", + }) + if err != nil { + t.Fatalf("PrepareS3CredentialForStorage(second) failed: %v", err) + } + _, err = db.db.ExecContext(ctx, ` + INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, second.CredentialID, second.Bucket, "s3", second.Region, second.AccessKey, second.SecretKey, second.Endpoint) + if err == nil || !strings.Contains(err.Error(), "physical bucket is already configured under another credential") { + t.Fatalf("expected trigger rejection, got %v", err) } } diff --git a/internal/repair/types.go b/internal/repair/types.go index b23324d1..dff0a3b1 100644 --- a/internal/repair/types.go +++ b/internal/repair/types.go @@ -90,6 +90,10 @@ type ApplyResult struct { AutoFixable int `json:"auto_fixable"` } +func (r Report) FindingCount() int { + return len(r.Objects) +} + type Service struct { index IndexAPI buckets BucketsAPI From 93de956934198f6f70ec36da1938c9a7b4238493 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 26 Jun 2026 11:54:12 -0700 Subject: [PATCH 04/24] fix apigen --- apigen/client/internalapi/internal.gen.go | 22 +++++++++++----------- apigen/server/internalapi/internal.gen.go | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index 516ca0e2..1a8ee773 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -204,6 +204,15 @@ type ListRecordsResponse struct { Records *[]InternalRecord `json:"records,omitempty"` } +// StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. +type StorageCleanupAccessProbe struct { + Bucket *string `json:"bucket,omitempty"` + ErrorKind *string `json:"error_kind,omitempty"` + StorageMessage *string `json:"storage_message,omitempty"` + StorageStatus *string `json:"storage_status,omitempty"` + Url *string `json:"url,omitempty"` +} + // StorageCleanupApplyRequest defines model for StorageCleanupApplyRequest. type StorageCleanupApplyRequest struct { CheckStorage *bool `json:"check_storage,omitempty"` @@ -240,8 +249,8 @@ type StorageCleanupAuditRequest struct { // StorageCleanupFinding defines model for StorageCleanupFinding. type StorageCleanupFinding struct { - CleanupScope *string `json:"cleanup_scope,omitempty"` ChecksumCount *int `json:"checksum_count,omitempty"` + CleanupScope *string `json:"cleanup_scope,omitempty"` Kind *string `json:"kind,omitempty"` LegacyUrlTemplateDetected *bool `json:"legacy_url_template_detected,omitempty"` Message *string `json:"message,omitempty"` @@ -249,18 +258,9 @@ type StorageCleanupFinding struct { RecommendedAction *string `json:"recommended_action,omitempty"` Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` + Severity *string `json:"severity,omitempty"` SizeCount *int `json:"size_count,omitempty"` StructuralReason *string `json:"structural_reason,omitempty"` - Severity *string `json:"severity,omitempty"` -} - -// StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. -type StorageCleanupAccessProbe struct { - Bucket *string `json:"bucket,omitempty"` - ErrorKind *string `json:"error_kind,omitempty"` - StorageMessage *string `json:"storage_message,omitempty"` - StorageStatus *string `json:"storage_status,omitempty"` - Url *string `json:"url,omitempty"` } // StorageCleanupRecordAudit defines model for StorageCleanupRecordAudit. diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 4f3df1ce..2c8fcb86 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -202,6 +202,15 @@ type ListRecordsResponse struct { Records *[]InternalRecord `json:"records,omitempty"` } +// StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. +type StorageCleanupAccessProbe struct { + Bucket *string `json:"bucket,omitempty"` + ErrorKind *string `json:"error_kind,omitempty"` + StorageMessage *string `json:"storage_message,omitempty"` + StorageStatus *string `json:"storage_status,omitempty"` + Url *string `json:"url,omitempty"` +} + // StorageCleanupApplyRequest defines model for StorageCleanupApplyRequest. type StorageCleanupApplyRequest struct { CheckStorage *bool `json:"check_storage,omitempty"` @@ -238,8 +247,8 @@ type StorageCleanupAuditRequest struct { // StorageCleanupFinding defines model for StorageCleanupFinding. type StorageCleanupFinding struct { - CleanupScope *string `json:"cleanup_scope,omitempty"` ChecksumCount *int `json:"checksum_count,omitempty"` + CleanupScope *string `json:"cleanup_scope,omitempty"` Kind *string `json:"kind,omitempty"` LegacyUrlTemplateDetected *bool `json:"legacy_url_template_detected,omitempty"` Message *string `json:"message,omitempty"` @@ -247,18 +256,9 @@ type StorageCleanupFinding struct { RecommendedAction *string `json:"recommended_action,omitempty"` Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` + Severity *string `json:"severity,omitempty"` SizeCount *int `json:"size_count,omitempty"` StructuralReason *string `json:"structural_reason,omitempty"` - Severity *string `json:"severity,omitempty"` -} - -// StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. -type StorageCleanupAccessProbe struct { - Bucket *string `json:"bucket,omitempty"` - ErrorKind *string `json:"error_kind,omitempty"` - StorageMessage *string `json:"storage_message,omitempty"` - StorageStatus *string `json:"storage_status,omitempty"` - Url *string `json:"url,omitempty"` } // StorageCleanupRecordAudit defines model for StorageCleanupRecordAudit. From 6ad7fecd8fb022262f0908f24cb6b5047ffbbcd4 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 29 Jun 2026 09:34:13 -0700 Subject: [PATCH 05/24] scrub file_name from syfon --- apigen/client/internalapi/internal.gen.go | 33 +- apigen/client/metricsapi/metrics.gen.go | 462 ------------------ apigen/openapi/internal.openapi.yaml | 24 +- apigen/openapi/metrics.openapi.yaml | 73 --- apigen/server/internalapi/internal.gen.go | 39 +- apigen/server/metricsapi/metrics.gen.go | 406 --------------- client/services/data_service.go | 8 +- client/services/data_service_test.go | 6 +- client/services/drs_service.go | 6 +- client/services/drs_service_test.go | 4 +- .../services/index_data_drs_helpers_test.go | 2 +- client/services/index_service.go | 17 +- client/services/index_service_test.go | 20 +- client/services/metrics_service.go | 40 -- client/services/services_test.go | 34 +- client/services/types.go | 2 +- cmd/cli_basic_commands_test.go | 10 +- cmd/copyproject/main.go | 4 +- cmd/download/main.go | 7 +- cmd/list/main.go | 102 +--- cmd/list/main_test.go | 59 +-- cmd/projectcopy/helpers.go | 81 +-- cmd/projectcopy/helpers_test.go | 21 +- internal/api/internaldrs/index.go | 41 -- .../api/internaldrs/index_additional_test.go | 10 +- internal/api/internaldrs/index_test.go | 19 +- internal/api/internaldrs/repair_cleanup.go | 3 - internal/api/internaldrs/transfer.go | 18 +- .../api/internaldrs/upload_routes_test.go | 16 +- internal/api/metrics/storage_summary.go | 140 ------ internal/api/metrics/storage_summary_test.go | 131 ----- internal/core/converters.go | 20 +- internal/db/postgres/metrics.go | 2 +- internal/db/postgres/objects.go | 39 +- internal/db/sqlite/metrics.go | 2 +- internal/db/sqlite/objects.go | 39 +- internal/models/object.go | 15 +- internal/models/object_test.go | 6 +- internal/models/storage.go | 2 +- internal/storagemetrics/storage_metrics.go | 2 +- .../storagemetrics/storage_metrics_test.go | 2 +- 41 files changed, 240 insertions(+), 1727 deletions(-) delete mode 100644 internal/api/metrics/storage_summary.go delete mode 100644 internal/api/metrics/storage_summary_test.go diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index 1a8ee773..0dca3c92 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -60,12 +60,6 @@ type DeleteByQueryResponse struct { // HashInfo Hash map, e.g. {"sha256":"..."} type HashInfo map[string]string -// IndexDirectory defines model for IndexDirectory. -type IndexDirectory struct { - Name string `json:"name"` - Path string `json:"path"` -} - // InternalMultipartCompleteRequest defines model for InternalMultipartCompleteRequest. type InternalMultipartCompleteRequest struct { Bucket *string `json:"bucket,omitempty"` @@ -82,8 +76,8 @@ type InternalMultipartInitOutput struct { // InternalMultipartInitRequest defines model for InternalMultipartInitRequest. type InternalMultipartInitRequest struct { - FileName *string `json:"file_name,omitempty"` Guid *string `json:"guid,omitempty"` + Key *string `json:"key,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` } @@ -114,7 +108,6 @@ type InternalRecord struct { CreatedTime *string `json:"created_time,omitempty"` Description *string `json:"description,omitempty"` Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` @@ -135,7 +128,6 @@ type InternalRecordResponse struct { CreatedTime *string `json:"created_time,omitempty"` Description *string `json:"description,omitempty"` Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` @@ -173,7 +165,7 @@ type InternalUploadBlankRequest struct { type InternalUploadBulkItem struct { ExpiresIn *int32 `json:"expires_in,omitempty"` FileId string `json:"file_id"` - FileName *string `json:"file_name,omitempty"` + Key *string `json:"key,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` } @@ -190,18 +182,17 @@ type InternalUploadBulkRequest struct { // InternalUploadBulkResult defines model for InternalUploadBulkResult. type InternalUploadBulkResult struct { - Bucket *string `json:"bucket,omitempty"` - Error *string `json:"error,omitempty"` - FileId string `json:"file_id"` - FileName *string `json:"file_name,omitempty"` - Status int32 `json:"status"` - Url *string `json:"url,omitempty"` + Bucket *string `json:"bucket,omitempty"` + Error *string `json:"error,omitempty"` + FileId string `json:"file_id"` + Key *string `json:"key,omitempty"` + Status int32 `json:"status"` + Url *string `json:"url,omitempty"` } // ListRecordsResponse defines model for ListRecordsResponse. type ListRecordsResponse struct { - Directories *[]IndexDirectory `json:"directories,omitempty"` - Records *[]InternalRecord `json:"records,omitempty"` + Records *[]InternalRecord `json:"records,omitempty"` } // StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. @@ -322,7 +313,7 @@ type InternalDownloadPartParams struct { type InternalUploadURLParams struct { Organization *string `form:"organization,omitempty" json:"organization,omitempty"` Project *string `form:"project,omitempty" json:"project,omitempty"` - FileName *string `form:"file_name,omitempty" json:"file_name,omitempty"` + Key *string `form:"key,omitempty" json:"key,omitempty"` ExpiresIn *int32 `form:"expires_in,omitempty" json:"expires_in,omitempty"` } @@ -1546,9 +1537,9 @@ func NewInternalUploadURLRequest(server string, fileId string, params *InternalU } - if params.FileName != nil { + if params.Key != nil { - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "file_name", runtime.ParamLocationQuery, *params.FileName); err != nil { + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "key", runtime.ParamLocationQuery, *params.Key); err != nil { return nil, err } else if parsed, err := url.ParseQuery(queryFrag); err != nil { return nil, err diff --git a/apigen/client/metricsapi/metrics.gen.go b/apigen/client/metricsapi/metrics.gen.go index e09245ef..863b0183 100644 --- a/apigen/client/metricsapi/metrics.gen.go +++ b/apigen/client/metricsapi/metrics.gen.go @@ -31,12 +31,6 @@ const ( Unmatched ProviderTransferReconciliationStatus = "unmatched" ) -// Defines values for StoragePathChildType. -const ( - Directory StoragePathChildType = "directory" - File StoragePathChildType = "file" -) - // Defines values for TransferBreakdownResponseGroupBy. const ( TransferBreakdownResponseGroupByObject TransferBreakdownResponseGroupBy = "object" @@ -45,20 +39,6 @@ const ( TransferBreakdownResponseGroupByUser TransferBreakdownResponseGroupBy = "user" ) -// Defines values for ListStorageChildrenParamsSortBy. -const ( - Bytes ListStorageChildrenParamsSortBy = "bytes" - Name ListStorageChildrenParamsSortBy = "name" - Records ListStorageChildrenParamsSortBy = "records" - UpdatedTime ListStorageChildrenParamsSortBy = "updated_time" -) - -// Defines values for ListStorageChildrenParamsSortOrder. -const ( - Asc ListStorageChildrenParamsSortOrder = "asc" - Desc ListStorageChildrenParamsSortOrder = "desc" -) - // Defines values for GetTransferBreakdownParamsGroupBy. const ( GetTransferBreakdownParamsGroupByObject GetTransferBreakdownParamsGroupBy = "object" @@ -142,45 +122,6 @@ type ProviderTransferEventsRequest struct { // ProviderTransferReconciliationStatus defines model for ProviderTransferReconciliationStatus. type ProviderTransferReconciliationStatus string -// StoragePathChild defines model for StoragePathChild. -type StoragePathChild struct { - DownloadCount *int64 `json:"download_count,omitempty"` - FileCount *int64 `json:"file_count,omitempty"` - LastDownloadTime *time.Time `json:"last_download_time"` - LatestUpdateTime *time.Time `json:"latest_update_time"` - Name *string `json:"name,omitempty"` - Path *string `json:"path,omitempty"` - RecordCount *int64 `json:"record_count,omitempty"` - TotalBytes *int64 `json:"total_bytes,omitempty"` - Type *StoragePathChildType `json:"type,omitempty"` -} - -// StoragePathChildType defines model for StoragePathChild.Type. -type StoragePathChildType string - -// StoragePathChildrenResponse defines model for StoragePathChildrenResponse. -type StoragePathChildrenResponse struct { - Items *[]StoragePathChild `json:"items,omitempty"` - Organization *string `json:"organization,omitempty"` - Path *string `json:"path,omitempty"` - Project *string `json:"project,omitempty"` -} - -// StoragePathSummary defines model for StoragePathSummary. -type StoragePathSummary struct { - DirectChildCount *int64 `json:"direct_child_count,omitempty"` - DownloadCount *int64 `json:"download_count,omitempty"` - DuplicatePathCount *int64 `json:"duplicate_path_count,omitempty"` - FileCount *int64 `json:"file_count,omitempty"` - LastDownloadTime *time.Time `json:"last_download_time"` - LatestUpdateTime *time.Time `json:"latest_update_time"` - Organization *string `json:"organization,omitempty"` - Path *string `json:"path,omitempty"` - Project *string `json:"project,omitempty"` - RecordCount *int64 `json:"record_count,omitempty"` - TotalBytes *int64 `json:"total_bytes,omitempty"` -} - // TransferAttributionBreakdown defines model for TransferAttributionBreakdown. type TransferAttributionBreakdown struct { ActorEmail *string `json:"actor_email,omitempty"` @@ -249,9 +190,6 @@ type From = time.Time // Organization defines model for Organization. type Organization = string -// Path defines model for Path. -type Path = string - // Project defines model for Project. type Project = string @@ -264,12 +202,6 @@ type ReconciliationStatus = ProviderTransferReconciliationStatus // SHA256 defines model for SHA256. type SHA256 = string -// StorageOrganization defines model for StorageOrganization. -type StorageOrganization = string - -// StorageProject defines model for StorageProject. -type StorageProject = string - // To defines model for To. type To = time.Time @@ -307,40 +239,6 @@ type RecordProviderTransferEventsParams struct { Project *Project `form:"project,omitempty" json:"project,omitempty"` } -// ListStorageChildrenParams defines parameters for ListStorageChildren. -type ListStorageChildrenParams struct { - // Organization Organization scope for the storage subtree query. - Organization StorageOrganization `form:"organization" json:"organization"` - - // Project Project scope for the storage subtree query. - Project StorageProject `form:"project" json:"project"` - - // Path Optional storage subtree path. Omit for the project root. - Path *Path `form:"path,omitempty" json:"path,omitempty"` - Limit *int `form:"limit,omitempty" json:"limit,omitempty"` - Offset *int `form:"offset,omitempty" json:"offset,omitempty"` - SortBy *ListStorageChildrenParamsSortBy `form:"sort_by,omitempty" json:"sort_by,omitempty"` - SortOrder *ListStorageChildrenParamsSortOrder `form:"sort_order,omitempty" json:"sort_order,omitempty"` -} - -// ListStorageChildrenParamsSortBy defines parameters for ListStorageChildren. -type ListStorageChildrenParamsSortBy string - -// ListStorageChildrenParamsSortOrder defines parameters for ListStorageChildren. -type ListStorageChildrenParamsSortOrder string - -// GetStorageSummaryParams defines parameters for GetStorageSummary. -type GetStorageSummaryParams struct { - // Organization Organization scope for the storage subtree query. - Organization StorageOrganization `form:"organization" json:"organization"` - - // Project Project scope for the storage subtree query. - Project StorageProject `form:"project" json:"project"` - - // Path Optional storage subtree path. Omit for the project root. - Path *Path `form:"path,omitempty" json:"path,omitempty"` -} - // GetMetricsSummaryParams defines parameters for GetMetricsSummary. type GetMetricsSummaryParams struct { InactiveDays *int `form:"inactive_days,omitempty" json:"inactive_days,omitempty"` @@ -487,12 +385,6 @@ type ClientInterface interface { RecordProviderTransferEvents(ctx context.Context, params *RecordProviderTransferEventsParams, body RecordProviderTransferEventsJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // ListStorageChildren request - ListStorageChildren(ctx context.Context, params *ListStorageChildrenParams, reqEditors ...RequestEditorFn) (*http.Response, error) - - // GetStorageSummary request - GetStorageSummary(ctx context.Context, params *GetStorageSummaryParams, reqEditors ...RequestEditorFn) (*http.Response, error) - // GetMetricsSummary request GetMetricsSummary(ctx context.Context, params *GetMetricsSummaryParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -551,30 +443,6 @@ func (c *Client) RecordProviderTransferEvents(ctx context.Context, params *Recor return c.Client.Do(req) } -func (c *Client) ListStorageChildren(ctx context.Context, params *ListStorageChildrenParams, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewListStorageChildrenRequest(c.Server, params) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) GetStorageSummary(ctx context.Context, params *GetStorageSummaryParams, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetStorageSummaryRequest(c.Server, params) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - func (c *Client) GetMetricsSummary(ctx context.Context, params *GetMetricsSummaryParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetMetricsSummaryRequest(c.Server, params) if err != nil { @@ -874,216 +742,6 @@ func NewRecordProviderTransferEventsRequestWithBody(server string, params *Recor return req, nil } -// NewListStorageChildrenRequest generates requests for ListStorageChildren -func NewListStorageChildrenRequest(server string, params *ListStorageChildrenParams) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/index/v1/metrics/storage/children") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - if params != nil { - queryValues := queryURL.Query() - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "organization", runtime.ParamLocationQuery, params.Organization); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "project", runtime.ParamLocationQuery, params.Project); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - if params.Path != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "path", runtime.ParamLocationQuery, *params.Path); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - if params.Limit != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "limit", runtime.ParamLocationQuery, *params.Limit); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - if params.Offset != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "offset", runtime.ParamLocationQuery, *params.Offset); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - if params.SortBy != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "sort_by", runtime.ParamLocationQuery, *params.SortBy); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - if params.SortOrder != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "sort_order", runtime.ParamLocationQuery, *params.SortOrder); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - queryURL.RawQuery = queryValues.Encode() - } - - req, err := http.NewRequest("GET", queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - -// NewGetStorageSummaryRequest generates requests for GetStorageSummary -func NewGetStorageSummaryRequest(server string, params *GetStorageSummaryParams) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/index/v1/metrics/storage/summary") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - if params != nil { - queryValues := queryURL.Query() - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "organization", runtime.ParamLocationQuery, params.Organization); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "project", runtime.ParamLocationQuery, params.Project); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - if params.Path != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "path", runtime.ParamLocationQuery, *params.Path); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - queryURL.RawQuery = queryValues.Encode() - } - - req, err := http.NewRequest("GET", queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - // NewGetMetricsSummaryRequest generates requests for GetMetricsSummary func NewGetMetricsSummaryRequest(server string, params *GetMetricsSummaryParams) (*http.Request, error) { var err error @@ -1653,12 +1311,6 @@ type ClientWithResponsesInterface interface { RecordProviderTransferEventsWithResponse(ctx context.Context, params *RecordProviderTransferEventsParams, body RecordProviderTransferEventsJSONRequestBody, reqEditors ...RequestEditorFn) (*RecordProviderTransferEventsResponse, error) - // ListStorageChildrenWithResponse request - ListStorageChildrenWithResponse(ctx context.Context, params *ListStorageChildrenParams, reqEditors ...RequestEditorFn) (*ListStorageChildrenResponse, error) - - // GetStorageSummaryWithResponse request - GetStorageSummaryWithResponse(ctx context.Context, params *GetStorageSummaryParams, reqEditors ...RequestEditorFn) (*GetStorageSummaryResponse, error) - // GetMetricsSummaryWithResponse request GetMetricsSummaryWithResponse(ctx context.Context, params *GetMetricsSummaryParams, reqEditors ...RequestEditorFn) (*GetMetricsSummaryResponse, error) @@ -1735,50 +1387,6 @@ func (r RecordProviderTransferEventsResponse) StatusCode() int { return 0 } -type ListStorageChildrenResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *StoragePathChildrenResponse -} - -// Status returns HTTPResponse.Status -func (r ListStorageChildrenResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r ListStorageChildrenResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -type GetStorageSummaryResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *StoragePathSummary -} - -// Status returns HTTPResponse.Status -func (r GetStorageSummaryResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r GetStorageSummaryResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - type GetMetricsSummaryResponse struct { Body []byte HTTPResponse *http.Response @@ -1880,24 +1488,6 @@ func (c *ClientWithResponses) RecordProviderTransferEventsWithResponse(ctx conte return ParseRecordProviderTransferEventsResponse(rsp) } -// ListStorageChildrenWithResponse request returning *ListStorageChildrenResponse -func (c *ClientWithResponses) ListStorageChildrenWithResponse(ctx context.Context, params *ListStorageChildrenParams, reqEditors ...RequestEditorFn) (*ListStorageChildrenResponse, error) { - rsp, err := c.ListStorageChildren(ctx, params, reqEditors...) - if err != nil { - return nil, err - } - return ParseListStorageChildrenResponse(rsp) -} - -// GetStorageSummaryWithResponse request returning *GetStorageSummaryResponse -func (c *ClientWithResponses) GetStorageSummaryWithResponse(ctx context.Context, params *GetStorageSummaryParams, reqEditors ...RequestEditorFn) (*GetStorageSummaryResponse, error) { - rsp, err := c.GetStorageSummary(ctx, params, reqEditors...) - if err != nil { - return nil, err - } - return ParseGetStorageSummaryResponse(rsp) -} - // GetMetricsSummaryWithResponse request returning *GetMetricsSummaryResponse func (c *ClientWithResponses) GetMetricsSummaryWithResponse(ctx context.Context, params *GetMetricsSummaryParams, reqEditors ...RequestEditorFn) (*GetMetricsSummaryResponse, error) { rsp, err := c.GetMetricsSummary(ctx, params, reqEditors...) @@ -2003,58 +1593,6 @@ func ParseRecordProviderTransferEventsResponse(rsp *http.Response) (*RecordProvi return response, nil } -// ParseListStorageChildrenResponse parses an HTTP response from a ListStorageChildrenWithResponse call -func ParseListStorageChildrenResponse(rsp *http.Response) (*ListStorageChildrenResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &ListStorageChildrenResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest StoragePathChildrenResponse - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - } - - return response, nil -} - -// ParseGetStorageSummaryResponse parses an HTTP response from a GetStorageSummaryWithResponse call -func ParseGetStorageSummaryResponse(rsp *http.Response) (*GetStorageSummaryResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &GetStorageSummaryResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest StoragePathSummary - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - } - - return response, nil -} - // ParseGetMetricsSummaryResponse parses an HTTP response from a GetMetricsSummaryWithResponse call func ParseGetMetricsSummaryResponse(rsp *http.Response) (*GetMetricsSummaryResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/openapi/internal.openapi.yaml b/apigen/openapi/internal.openapi.yaml index 4a996744..bc6b2e87 100644 --- a/apigen/openapi/internal.openapi.yaml +++ b/apigen/openapi/internal.openapi.yaml @@ -417,7 +417,7 @@ paths: schema: type: string - in: query - name: file_name + name: key required: false schema: type: string @@ -722,8 +722,6 @@ components: type: array items: $ref: './openapi.yaml#/components/schemas/AccessMethod' - file_name: - type: string organization: type: string project: @@ -754,24 +752,10 @@ components: ListRecordsResponse: type: object properties: - directories: - type: array - items: - $ref: '#/components/schemas/IndexDirectory' records: type: array items: $ref: '#/components/schemas/InternalRecord' - IndexDirectory: - type: object - required: - - name - - path - properties: - name: - type: string - path: - type: string BulkCreateRequest: type: object required: @@ -1053,7 +1037,7 @@ components: properties: file_id: type: string - file_name: + key: type: string organization: type: string @@ -1079,7 +1063,7 @@ components: type: string bucket: type: string - file_name: + key: type: string url: type: string @@ -1093,7 +1077,7 @@ components: properties: guid: type: string - file_name: + key: type: string organization: type: string diff --git a/apigen/openapi/metrics.openapi.yaml b/apigen/openapi/metrics.openapi.yaml index 73297541..088b32bd 100644 --- a/apigen/openapi/metrics.openapi.yaml +++ b/apigen/openapi/metrics.openapi.yaml @@ -107,79 +107,6 @@ paths: description: Forbidden '500': description: Internal server error - /index/v1/metrics/storage/summary: - get: - tags: - - metrics - operationId: getStorageSummary - parameters: - - $ref: '#/components/parameters/StorageOrganization' - - $ref: '#/components/parameters/StorageProject' - - $ref: '#/components/parameters/Path' - responses: - '200': - description: Aggregate storage metrics for one path subtree - content: - application/json: - schema: - $ref: '#/components/schemas/StoragePathSummary' - '400': - description: Invalid query params - '401': - description: Authentication required - '403': - description: Forbidden - '500': - description: Internal server error - /index/v1/metrics/storage/children: - get: - tags: - - metrics - operationId: listStorageChildren - parameters: - - $ref: '#/components/parameters/StorageOrganization' - - $ref: '#/components/parameters/StorageProject' - - $ref: '#/components/parameters/Path' - - in: query - name: limit - schema: - type: integer - minimum: 1 - maximum: 1000 - default: 200 - - in: query - name: offset - schema: - type: integer - minimum: 0 - default: 0 - - in: query - name: sort_by - schema: - type: string - enum: [bytes, name, updated_time, records] - default: name - - in: query - name: sort_order - schema: - type: string - enum: [asc, desc] - default: asc - responses: - '200': - description: Direct child storage metrics for one path subtree - content: - application/json: - schema: - $ref: '#/components/schemas/StoragePathChildrenResponse' - '400': - description: Invalid query params - '401': - description: Authentication required - '403': - description: Forbidden - '500': - description: Internal server error /index/v1/metrics/provider-transfer-events: post: tags: diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 2c8fcb86..7e4bf918 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -58,12 +58,6 @@ type DeleteByQueryResponse struct { // HashInfo Hash map, e.g. {"sha256":"..."} type HashInfo map[string]string -// IndexDirectory defines model for IndexDirectory. -type IndexDirectory struct { - Name string `json:"name"` - Path string `json:"path"` -} - // InternalMultipartCompleteRequest defines model for InternalMultipartCompleteRequest. type InternalMultipartCompleteRequest struct { Bucket *string `json:"bucket,omitempty"` @@ -80,8 +74,8 @@ type InternalMultipartInitOutput struct { // InternalMultipartInitRequest defines model for InternalMultipartInitRequest. type InternalMultipartInitRequest struct { - FileName *string `json:"file_name,omitempty"` Guid *string `json:"guid,omitempty"` + Key *string `json:"key,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` } @@ -112,7 +106,6 @@ type InternalRecord struct { CreatedTime *string `json:"created_time,omitempty"` Description *string `json:"description,omitempty"` Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` @@ -133,7 +126,6 @@ type InternalRecordResponse struct { CreatedTime *string `json:"created_time,omitempty"` Description *string `json:"description,omitempty"` Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` @@ -171,7 +163,7 @@ type InternalUploadBlankRequest struct { type InternalUploadBulkItem struct { ExpiresIn *int32 `json:"expires_in,omitempty"` FileId string `json:"file_id"` - FileName *string `json:"file_name,omitempty"` + Key *string `json:"key,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` } @@ -188,18 +180,17 @@ type InternalUploadBulkRequest struct { // InternalUploadBulkResult defines model for InternalUploadBulkResult. type InternalUploadBulkResult struct { - Bucket *string `json:"bucket,omitempty"` - Error *string `json:"error,omitempty"` - FileId string `json:"file_id"` - FileName *string `json:"file_name,omitempty"` - Status int32 `json:"status"` - Url *string `json:"url,omitempty"` + Bucket *string `json:"bucket,omitempty"` + Error *string `json:"error,omitempty"` + FileId string `json:"file_id"` + Key *string `json:"key,omitempty"` + Status int32 `json:"status"` + Url *string `json:"url,omitempty"` } // ListRecordsResponse defines model for ListRecordsResponse. type ListRecordsResponse struct { - Directories *[]IndexDirectory `json:"directories,omitempty"` - Records *[]InternalRecord `json:"records,omitempty"` + Records *[]InternalRecord `json:"records,omitempty"` } // StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. @@ -320,7 +311,7 @@ type InternalDownloadPartParams struct { type InternalUploadURLParams struct { Organization *string `form:"organization,omitempty" json:"organization,omitempty"` Project *string `form:"project,omitempty" json:"project,omitempty"` - FileName *string `form:"file_name,omitempty" json:"file_name,omitempty"` + Key *string `form:"key,omitempty" json:"key,omitempty"` ExpiresIn *int32 `form:"expires_in,omitempty" json:"expires_in,omitempty"` } @@ -689,15 +680,15 @@ func (siw *ServerInterfaceWrapper) InternalUploadURL(c fiber.Ctx) error { params.Project = &value } - // ------------- Optional query parameter "file_name" ------------- - if paramValue := c.Query("file_name"); paramValue != "" { + // ------------- Optional query parameter "key" ------------- + if paramValue := c.Query("key"); paramValue != "" { var value string - err = runtime.BindStyledParameterWithOptions("form", "file_name", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + err = runtime.BindStyledParameterWithOptions("form", "key", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter file_name: %w", err).Error()) + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter key: %w", err).Error()) } - params.FileName = &value + params.Key = &value } // ------------- Optional query parameter "expires_in" ------------- diff --git a/apigen/server/metricsapi/metrics.gen.go b/apigen/server/metricsapi/metrics.gen.go index 87a4de98..ab20b076 100644 --- a/apigen/server/metricsapi/metrics.gen.go +++ b/apigen/server/metricsapi/metrics.gen.go @@ -26,12 +26,6 @@ const ( Unmatched ProviderTransferReconciliationStatus = "unmatched" ) -// Defines values for StoragePathChildType. -const ( - Directory StoragePathChildType = "directory" - File StoragePathChildType = "file" -) - // Defines values for TransferBreakdownResponseGroupBy. const ( TransferBreakdownResponseGroupByObject TransferBreakdownResponseGroupBy = "object" @@ -40,20 +34,6 @@ const ( TransferBreakdownResponseGroupByUser TransferBreakdownResponseGroupBy = "user" ) -// Defines values for ListStorageChildrenParamsSortBy. -const ( - Bytes ListStorageChildrenParamsSortBy = "bytes" - Name ListStorageChildrenParamsSortBy = "name" - Records ListStorageChildrenParamsSortBy = "records" - UpdatedTime ListStorageChildrenParamsSortBy = "updated_time" -) - -// Defines values for ListStorageChildrenParamsSortOrder. -const ( - Asc ListStorageChildrenParamsSortOrder = "asc" - Desc ListStorageChildrenParamsSortOrder = "desc" -) - // Defines values for GetTransferBreakdownParamsGroupBy. const ( GetTransferBreakdownParamsGroupByObject GetTransferBreakdownParamsGroupBy = "object" @@ -137,45 +117,6 @@ type ProviderTransferEventsRequest struct { // ProviderTransferReconciliationStatus defines model for ProviderTransferReconciliationStatus. type ProviderTransferReconciliationStatus string -// StoragePathChild defines model for StoragePathChild. -type StoragePathChild struct { - DownloadCount *int64 `json:"download_count,omitempty"` - FileCount *int64 `json:"file_count,omitempty"` - LastDownloadTime *time.Time `json:"last_download_time"` - LatestUpdateTime *time.Time `json:"latest_update_time"` - Name *string `json:"name,omitempty"` - Path *string `json:"path,omitempty"` - RecordCount *int64 `json:"record_count,omitempty"` - TotalBytes *int64 `json:"total_bytes,omitempty"` - Type *StoragePathChildType `json:"type,omitempty"` -} - -// StoragePathChildType defines model for StoragePathChild.Type. -type StoragePathChildType string - -// StoragePathChildrenResponse defines model for StoragePathChildrenResponse. -type StoragePathChildrenResponse struct { - Items *[]StoragePathChild `json:"items,omitempty"` - Organization *string `json:"organization,omitempty"` - Path *string `json:"path,omitempty"` - Project *string `json:"project,omitempty"` -} - -// StoragePathSummary defines model for StoragePathSummary. -type StoragePathSummary struct { - DirectChildCount *int64 `json:"direct_child_count,omitempty"` - DownloadCount *int64 `json:"download_count,omitempty"` - DuplicatePathCount *int64 `json:"duplicate_path_count,omitempty"` - FileCount *int64 `json:"file_count,omitempty"` - LastDownloadTime *time.Time `json:"last_download_time"` - LatestUpdateTime *time.Time `json:"latest_update_time"` - Organization *string `json:"organization,omitempty"` - Path *string `json:"path,omitempty"` - Project *string `json:"project,omitempty"` - RecordCount *int64 `json:"record_count,omitempty"` - TotalBytes *int64 `json:"total_bytes,omitempty"` -} - // TransferAttributionBreakdown defines model for TransferAttributionBreakdown. type TransferAttributionBreakdown struct { ActorEmail *string `json:"actor_email,omitempty"` @@ -244,9 +185,6 @@ type From = time.Time // Organization defines model for Organization. type Organization = string -// Path defines model for Path. -type Path = string - // Project defines model for Project. type Project = string @@ -259,12 +197,6 @@ type ReconciliationStatus = ProviderTransferReconciliationStatus // SHA256 defines model for SHA256. type SHA256 = string -// StorageOrganization defines model for StorageOrganization. -type StorageOrganization = string - -// StorageProject defines model for StorageProject. -type StorageProject = string - // To defines model for To. type To = time.Time @@ -302,40 +234,6 @@ type RecordProviderTransferEventsParams struct { Project *Project `form:"project,omitempty" json:"project,omitempty"` } -// ListStorageChildrenParams defines parameters for ListStorageChildren. -type ListStorageChildrenParams struct { - // Organization Organization scope for the storage subtree query. - Organization StorageOrganization `form:"organization" json:"organization"` - - // Project Project scope for the storage subtree query. - Project StorageProject `form:"project" json:"project"` - - // Path Optional storage subtree path. Omit for the project root. - Path *Path `form:"path,omitempty" json:"path,omitempty"` - Limit *int `form:"limit,omitempty" json:"limit,omitempty"` - Offset *int `form:"offset,omitempty" json:"offset,omitempty"` - SortBy *ListStorageChildrenParamsSortBy `form:"sort_by,omitempty" json:"sort_by,omitempty"` - SortOrder *ListStorageChildrenParamsSortOrder `form:"sort_order,omitempty" json:"sort_order,omitempty"` -} - -// ListStorageChildrenParamsSortBy defines parameters for ListStorageChildren. -type ListStorageChildrenParamsSortBy string - -// ListStorageChildrenParamsSortOrder defines parameters for ListStorageChildren. -type ListStorageChildrenParamsSortOrder string - -// GetStorageSummaryParams defines parameters for GetStorageSummary. -type GetStorageSummaryParams struct { - // Organization Organization scope for the storage subtree query. - Organization StorageOrganization `form:"organization" json:"organization"` - - // Project Project scope for the storage subtree query. - Project StorageProject `form:"project" json:"project"` - - // Path Optional storage subtree path. Omit for the project root. - Path *Path `form:"path,omitempty" json:"path,omitempty"` -} - // GetMetricsSummaryParams defines parameters for GetMetricsSummary. type GetMetricsSummaryParams struct { InactiveDays *int `form:"inactive_days,omitempty" json:"inactive_days,omitempty"` @@ -410,12 +308,6 @@ type ServerInterface interface { // (POST /index/v1/metrics/provider-transfer-events) RecordProviderTransferEvents(c fiber.Ctx, params RecordProviderTransferEventsParams) error - // (GET /index/v1/metrics/storage/children) - ListStorageChildren(c fiber.Ctx, params ListStorageChildrenParams) error - - // (GET /index/v1/metrics/storage/summary) - GetStorageSummary(c fiber.Ctx, params GetStorageSummaryParams) error - // (GET /index/v1/metrics/summary) GetMetricsSummary(c fiber.Ctx, params GetMetricsSummaryParams) error // Group transfer attribution events @@ -567,142 +459,6 @@ func (siw *ServerInterfaceWrapper) RecordProviderTransferEvents(c fiber.Ctx) err return siw.Handler.RecordProviderTransferEvents(c, params) } -// ListStorageChildren operation middleware -func (siw *ServerInterfaceWrapper) ListStorageChildren(c fiber.Ctx) error { - var err error - var params ListStorageChildrenParams - - // ------------- Required query parameter "organization" ------------- - if paramValue := c.Query("organization"); paramValue != "" { - - var value StorageOrganization - err = runtime.BindStyledParameterWithOptions("form", "organization", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter organization: %w", err).Error()) - } - params.Organization = value - - } else { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument organization is required, but not found").Error()) - } - // ------------- Required query parameter "project" ------------- - if paramValue := c.Query("project"); paramValue != "" { - - var value StorageProject - err = runtime.BindStyledParameterWithOptions("form", "project", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter project: %w", err).Error()) - } - params.Project = value - - } else { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument project is required, but not found").Error()) - } - // ------------- Optional query parameter "path" ------------- - if paramValue := c.Query("path"); paramValue != "" { - - var value Path - err = runtime.BindStyledParameterWithOptions("form", "path", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter path: %w", err).Error()) - } - params.Path = &value - - } - // ------------- Optional query parameter "limit" ------------- - if paramValue := c.Query("limit"); paramValue != "" { - - var value int - err = runtime.BindStyledParameterWithOptions("form", "limit", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter limit: %w", err).Error()) - } - params.Limit = &value - - } - // ------------- Optional query parameter "offset" ------------- - if paramValue := c.Query("offset"); paramValue != "" { - - var value int - err = runtime.BindStyledParameterWithOptions("form", "offset", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter offset: %w", err).Error()) - } - params.Offset = &value - - } - // ------------- Optional query parameter "sort_by" ------------- - if paramValue := c.Query("sort_by"); paramValue != "" { - - var value ListStorageChildrenParamsSortBy - err = runtime.BindStyledParameterWithOptions("form", "sort_by", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter sort_by: %w", err).Error()) - } - params.SortBy = &value - - } - // ------------- Optional query parameter "sort_order" ------------- - if paramValue := c.Query("sort_order"); paramValue != "" { - - var value ListStorageChildrenParamsSortOrder - err = runtime.BindStyledParameterWithOptions("form", "sort_order", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter sort_order: %w", err).Error()) - } - params.SortOrder = &value - - } - - return siw.Handler.ListStorageChildren(c, params) -} - -// GetStorageSummary operation middleware -func (siw *ServerInterfaceWrapper) GetStorageSummary(c fiber.Ctx) error { - var err error - var params GetStorageSummaryParams - - // ------------- Required query parameter "organization" ------------- - if paramValue := c.Query("organization"); paramValue != "" { - - var value StorageOrganization - err = runtime.BindStyledParameterWithOptions("form", "organization", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter organization: %w", err).Error()) - } - params.Organization = value - - } else { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument organization is required, but not found").Error()) - } - // ------------- Required query parameter "project" ------------- - if paramValue := c.Query("project"); paramValue != "" { - - var value StorageProject - err = runtime.BindStyledParameterWithOptions("form", "project", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter project: %w", err).Error()) - } - params.Project = value - - } else { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument project is required, but not found").Error()) - } - // ------------- Optional query parameter "path" ------------- - if paramValue := c.Query("path"); paramValue != "" { - - var value Path - err = runtime.BindStyledParameterWithOptions("form", "path", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter path: %w", err).Error()) - } - params.Path = &value - - } - - return siw.Handler.GetStorageSummary(c, params) -} - // GetMetricsSummary operation middleware func (siw *ServerInterfaceWrapper) GetMetricsSummary(c fiber.Ctx) error { var err error @@ -1043,10 +799,6 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Post(options.BaseURL+"/index/v1/metrics/provider-transfer-events", wrapper.RecordProviderTransferEvents) - router.Get(options.BaseURL+"/index/v1/metrics/storage/children", wrapper.ListStorageChildren) - - router.Get(options.BaseURL+"/index/v1/metrics/storage/summary", wrapper.GetStorageSummary) - router.Get(options.BaseURL+"/index/v1/metrics/summary", wrapper.GetMetricsSummary) router.Get(options.BaseURL+"/index/v1/metrics/transfers/breakdown", wrapper.GetTransferBreakdown) @@ -1212,104 +964,6 @@ func (response RecordProviderTransferEvents500Response) VisitRecordProviderTrans return nil } -type ListStorageChildrenRequestObject struct { - Params ListStorageChildrenParams -} - -type ListStorageChildrenResponseObject interface { - VisitListStorageChildrenResponse(ctx fiber.Ctx) error -} - -type ListStorageChildren200JSONResponse StoragePathChildrenResponse - -func (response ListStorageChildren200JSONResponse) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { - ctx.Response().Header.Set("Content-Type", "application/json") - ctx.Status(200) - - return ctx.JSON(&response) -} - -type ListStorageChildren400Response struct { -} - -func (response ListStorageChildren400Response) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { - ctx.Status(400) - return nil -} - -type ListStorageChildren401Response struct { -} - -func (response ListStorageChildren401Response) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { - ctx.Status(401) - return nil -} - -type ListStorageChildren403Response struct { -} - -func (response ListStorageChildren403Response) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { - ctx.Status(403) - return nil -} - -type ListStorageChildren500Response struct { -} - -func (response ListStorageChildren500Response) VisitListStorageChildrenResponse(ctx fiber.Ctx) error { - ctx.Status(500) - return nil -} - -type GetStorageSummaryRequestObject struct { - Params GetStorageSummaryParams -} - -type GetStorageSummaryResponseObject interface { - VisitGetStorageSummaryResponse(ctx fiber.Ctx) error -} - -type GetStorageSummary200JSONResponse StoragePathSummary - -func (response GetStorageSummary200JSONResponse) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { - ctx.Response().Header.Set("Content-Type", "application/json") - ctx.Status(200) - - return ctx.JSON(&response) -} - -type GetStorageSummary400Response struct { -} - -func (response GetStorageSummary400Response) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { - ctx.Status(400) - return nil -} - -type GetStorageSummary401Response struct { -} - -func (response GetStorageSummary401Response) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { - ctx.Status(401) - return nil -} - -type GetStorageSummary403Response struct { -} - -func (response GetStorageSummary403Response) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { - ctx.Status(403) - return nil -} - -type GetStorageSummary500Response struct { -} - -func (response GetStorageSummary500Response) VisitGetStorageSummaryResponse(ctx fiber.Ctx) error { - ctx.Status(500) - return nil -} - type GetMetricsSummaryRequestObject struct { Params GetMetricsSummaryParams } @@ -1469,12 +1123,6 @@ type StrictServerInterface interface { // (POST /index/v1/metrics/provider-transfer-events) RecordProviderTransferEvents(ctx context.Context, request RecordProviderTransferEventsRequestObject) (RecordProviderTransferEventsResponseObject, error) - // (GET /index/v1/metrics/storage/children) - ListStorageChildren(ctx context.Context, request ListStorageChildrenRequestObject) (ListStorageChildrenResponseObject, error) - - // (GET /index/v1/metrics/storage/summary) - GetStorageSummary(ctx context.Context, request GetStorageSummaryRequestObject) (GetStorageSummaryResponseObject, error) - // (GET /index/v1/metrics/summary) GetMetricsSummary(ctx context.Context, request GetMetricsSummaryRequestObject) (GetMetricsSummaryResponseObject, error) // Group transfer attribution events @@ -1586,60 +1234,6 @@ func (sh *strictHandler) RecordProviderTransferEvents(ctx fiber.Ctx, params Reco return nil } -// ListStorageChildren operation middleware -func (sh *strictHandler) ListStorageChildren(ctx fiber.Ctx, params ListStorageChildrenParams) error { - var request ListStorageChildrenRequestObject - - request.Params = params - - handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { - return sh.ssi.ListStorageChildren(ctx.Context(), request.(ListStorageChildrenRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "ListStorageChildren") - } - - response, err := handler(ctx, request) - - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } else if validResponse, ok := response.(ListStorageChildrenResponseObject); ok { - if err := validResponse.VisitListStorageChildrenResponse(ctx); err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } - } else if response != nil { - return fmt.Errorf("unexpected response type: %T", response) - } - return nil -} - -// GetStorageSummary operation middleware -func (sh *strictHandler) GetStorageSummary(ctx fiber.Ctx, params GetStorageSummaryParams) error { - var request GetStorageSummaryRequestObject - - request.Params = params - - handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { - return sh.ssi.GetStorageSummary(ctx.Context(), request.(GetStorageSummaryRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "GetStorageSummary") - } - - response, err := handler(ctx, request) - - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } else if validResponse, ok := response.(GetStorageSummaryResponseObject); ok { - if err := validResponse.VisitGetStorageSummaryResponse(ctx); err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } - } else if response != nil { - return fmt.Errorf("unexpected response type: %T", response) - } - return nil -} - // GetMetricsSummary operation middleware func (sh *strictHandler) GetMetricsSummary(ctx fiber.Ctx, params GetMetricsSummaryParams) error { var request GetMetricsSummaryRequestObject diff --git a/client/services/data_service.go b/client/services/data_service.go index a6b66405..63ada687 100644 --- a/client/services/data_service.go +++ b/client/services/data_service.go @@ -50,8 +50,8 @@ func (d *DataService) UploadBlank(ctx context.Context, req internalapi.InternalU func (d *DataService) UploadURL(ctx context.Context, req UploadURLRequest) (internalapi.InternalSignedURL, error) { params := &internalapi.InternalUploadURLParams{} - if req.FileName != "" { - params.FileName = &req.FileName + if req.Key != "" { + params.Key = &req.Key } if req.ExpiresIn > 0 { expires := int32(req.ExpiresIn) @@ -254,7 +254,7 @@ func (d *DataService) ResolveUploadURL(ctx context.Context, guid, filename strin organization, project := uploadScopeFromMetadata(metadata) resp, err := d.UploadURL(ctx, UploadURLRequest{ FileID: guid, - FileName: filename, + Key: filename, Organization: organization, Project: project, }) @@ -338,7 +338,7 @@ func (d *DataService) InitMultipartUploadWithMetadata(ctx context.Context, guid, organization, project := uploadScopeFromMetadata(metadata) req := internalapi.InternalMultipartInitRequest{ Guid: &guid, - FileName: &filename, + Key: &filename, Organization: nil, Project: nil, } diff --git a/client/services/data_service_test.go b/client/services/data_service_test.go index 21707903..ccbce326 100644 --- a/client/services/data_service_test.go +++ b/client/services/data_service_test.go @@ -124,10 +124,10 @@ func TestDataServiceOperationsAndTransferHelpers(t *testing.T) { t.Fatalf("unexpected upload blank request: %+v", lastUploadBlank) } - if _, err := service.UploadURL(ctx, UploadURLRequest{FileID: "file-1", FileName: "name.txt", ExpiresIn: 60, Organization: "org-a", Project: "proj-a"}); err != nil { + if _, err := service.UploadURL(ctx, UploadURLRequest{FileID: "file-1", Key: "name.txt", ExpiresIn: 60, Organization: "org-a", Project: "proj-a"}); err != nil { t.Fatalf("UploadURL returned error: %v", err) } - if uploadURLQuery.Get("organization") != "org-a" || uploadURLQuery.Get("project") != "proj-a" || uploadURLQuery.Get("file_name") != "name.txt" || uploadURLQuery.Get("expires_in") != "60" { + if uploadURLQuery.Get("organization") != "org-a" || uploadURLQuery.Get("project") != "proj-a" || uploadURLQuery.Get("key") != "name.txt" || uploadURLQuery.Get("expires_in") != "60" { t.Fatalf("unexpected upload URL query values: %v", uploadURLQuery) } @@ -170,7 +170,7 @@ func TestDataServiceOperationsAndTransferHelpers(t *testing.T) { if err != nil || uploadID != "upload-id" || guid != "multipart-guid" { t.Fatalf("InitMultipartUpload returned uploadID=%q guid=%q err=%v", uploadID, guid, err) } - if lastMultipartInit.Guid == nil || *lastMultipartInit.Guid != "guid-a" || lastMultipartInit.FileName == nil || *lastMultipartInit.FileName != "name.txt" { + if lastMultipartInit.Guid == nil || *lastMultipartInit.Guid != "guid-a" || lastMultipartInit.Key == nil || *lastMultipartInit.Key != "name.txt" { t.Fatalf("unexpected multipart init request: %+v", lastMultipartInit) } if lastMultipartInit.Organization != nil || lastMultipartInit.Project != nil { diff --git a/client/services/drs_service.go b/client/services/drs_service.go index 71cc2d2e..ae9966d1 100644 --- a/client/services/drs_service.go +++ b/client/services/drs_service.go @@ -307,10 +307,10 @@ func internalRecordToDRSObject(rec *internalapi.InternalRecord) drsapi.DrsObject SelfUri: "drs://" + rec.Did, Size: size, } - if rec.FileName != nil { - base := path.Base(strings.TrimSpace(*rec.FileName)) + if rec.Name != nil { + base := path.Base(strings.TrimSpace(*rec.Name)) if base == "." || base == "/" || base == "" { - base = strings.TrimSpace(*rec.FileName) + base = strings.TrimSpace(*rec.Name) } obj.Name = &base } diff --git a/client/services/drs_service_test.go b/client/services/drs_service_test.go index be88ef32..69aa2c34 100644 --- a/client/services/drs_service_test.go +++ b/client/services/drs_service_test.go @@ -89,7 +89,7 @@ func TestDRSServiceResolveAndList(t *testing.T) { recordHashes := internalapi.HashInfo{"sha256": "abc123"} recordAuthz := map[string][]string{"org1": {"p1"}} record := testRecordForURL("did-record", "https://storage.example/record.bin", recordAuthz) - record.FileName = &recordName + record.Name = &recordName record.Size = &recordSize record.Hashes = &recordHashes records := []internalapi.InternalRecord{record} @@ -282,7 +282,7 @@ func TestDRSServiceDeleteRecordsByHash(t *testing.T) { testRecordForURL("obj-2", recordURLs[0], map[string][]string{"org1": {}}), } for i := range records { - records[i].FileName = &recordName + records[i].Name = &recordName records[i].Size = &recordSize records[i].Hashes = &recordHashes } diff --git a/client/services/index_data_drs_helpers_test.go b/client/services/index_data_drs_helpers_test.go index 46a87f87..b332f3c2 100644 --- a/client/services/index_data_drs_helpers_test.go +++ b/client/services/index_data_drs_helpers_test.go @@ -94,7 +94,7 @@ func toRecordResponse(rec internalapi.InternalRecord) internalapi.InternalRecord AccessMethods: rec.AccessMethods, ControlledAccess: rec.ControlledAccess, Description: rec.Description, - FileName: rec.FileName, + Name: rec.Name, Hashes: rec.Hashes, Size: rec.Size, Version: rec.Version, diff --git a/client/services/index_service.go b/client/services/index_service.go index 3b696911..5d66e139 100644 --- a/client/services/index_service.go +++ b/client/services/index_service.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + "path" "strings" "github.com/calypr/syfon/apigen/client/drs" @@ -216,7 +217,11 @@ func (s *IndexService) SHA256Validity(ctx context.Context, values []string) (map return s.BulkSHA256Validity(ctx, internalapi.BulkSHA256ValidityRequest{Sha256: &values}) } -func (s *IndexService) Upsert(ctx context.Context, did, objectURL, fileName string, size int64, sha256sum string, authorizations map[string][]string) error { +func (s *IndexService) Upsert(ctx context.Context, did, objectURL, recordPath string, size int64, sha256sum string, authorizations map[string][]string) error { + recordName := strings.TrimSpace(recordPath) + if recordName != "" { + recordName = path.Base(strings.Trim(recordName, "/")) + } existing, err := s.Get(ctx, did) if err == nil { req := internalapi.InternalRecord{ @@ -224,7 +229,7 @@ func (s *IndexService) Upsert(ctx context.Context, did, objectURL, fileName stri AccessMethods: existing.AccessMethods, ControlledAccess: existing.ControlledAccess, Description: existing.Description, - FileName: existing.FileName, + Name: existing.Name, Hashes: existing.Hashes, Size: existing.Size, Version: existing.Version, @@ -242,8 +247,8 @@ func (s *IndexService) Upsert(ctx context.Context, did, objectURL, fileName stri controlled := syfoncommon.AuthzMapToControlledAccess(authorizations) req.ControlledAccess = &controlled } - if fileName != "" { - req.FileName = &fileName + if recordName != "" { + req.Name = &recordName } if size > 0 { req.Size = &size @@ -274,8 +279,8 @@ func (s *IndexService) Upsert(ctx context.Context, did, objectURL, fileName stri if size > 0 { payload.Size = &size } - if fileName != "" { - payload.FileName = &fileName + if recordName != "" { + payload.Name = &recordName } if sha256sum != "" { h := internalapi.HashInfo{"sha256": sha256sum} diff --git a/client/services/index_service_test.go b/client/services/index_service_test.go index 63540b8b..57c9e885 100644 --- a/client/services/index_service_test.go +++ b/client/services/index_service_test.go @@ -38,7 +38,7 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { size := int64(12) authz := map[string][]string{"p1": {}} rec := testRecordForURL("did-list", "s3://bucket/object", authz) - rec.FileName = &name + rec.Name = &name rec.Size = &size records := []internalapi.InternalRecord{rec} writeJSON(t, w, http.StatusOK, internalapi.ListRecordsResponse{Records: &records}) @@ -85,14 +85,14 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { hashes := internalapi.HashInfo{"md5": "md5sum"} authz := map[string][]string{"existing": {}} rec := toRecordResponse(testRecordForURL("did-update", "s3://bucket/existing", authz)) - rec.FileName = &fileName + rec.Name = &fileName rec.Size = &size rec.Hashes = &hashes writeJSON(t, w, http.StatusOK, internalapi.InternalRecordResponse{ Did: rec.Did, AccessMethods: rec.AccessMethods, ControlledAccess: rec.ControlledAccess, - FileName: rec.FileName, + Name: rec.Name, Size: rec.Size, Hashes: rec.Hashes, }) @@ -125,7 +125,7 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { listSize := int64(12) listAuthz := map[string][]string{"p1": {}} listRec := testRecordForURL("did-list", "s3://bucket/object", listAuthz) - listRec.FileName = &listName + listRec.Name = &listName listRec.Size = &listSize listRecords := []internalapi.InternalRecord{listRec} listResp, err := json.Marshal(internalapi.ListRecordsResponse{Records: &listRecords}) @@ -150,12 +150,12 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { createSize := int64(55) createAuthz := map[string][]string{"p1": {}} createRec := testRecordForURL("did-new", "s3://bucket/created", createAuthz) - createRec.FileName = &createFile + createRec.Name = &createFile createRec.Size = &createSize if _, err := service.Create(ctx, createRec); err != nil { t.Fatalf("Create returned error: %v", err) } - if lastCreated.Did != "did-new" || lastCreated.FileName == nil || *lastCreated.FileName != "created.txt" { + if lastCreated.Did != "did-new" || lastCreated.Name == nil || *lastCreated.Name != "created.txt" { t.Fatalf("unexpected create payload: %+v", lastCreated) } @@ -181,14 +181,14 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { t.Fatalf("unexpected remove controlled access payload: %+v", lastRemoveControlled) } - if _, err := service.List(ctx, ListRecordsOptions{Hash: "sha", URL: "s3://bucket/path", Organization: "org", ProjectID: "proj", Path: "nested", Limit: 3, Start: "did-100"}); err != nil { + if _, err := service.List(ctx, ListRecordsOptions{Hash: "sha", URL: "s3://bucket/path", Organization: "org", ProjectID: "proj", Limit: 3, Start: "did-100"}); err != nil { t.Fatalf("List returned error: %v", err) } query, err := url.ParseQuery(strings.TrimPrefix(requester.builder.Url, "/index?")) if err != nil { t.Fatalf("parse list query: %v", err) } - if query.Get("hash") != "sha" || query.Get("url") != "s3://bucket/path" || query.Get("organization") != "org" || query.Get("project") != "proj" || query.Get("path") != "nested" || query.Get("limit") != "3" || query.Get("start") != "did-100" || query.Get("page") != "" { + if query.Get("hash") != "sha" || query.Get("url") != "s3://bucket/path" || query.Get("organization") != "org" || query.Get("project") != "proj" || query.Get("limit") != "3" || query.Get("start") != "did-100" || query.Get("page") != "" { t.Fatalf("unexpected list query values: %v", query) } @@ -253,7 +253,7 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { if err != nil { t.Fatalf("Upsert existing returned error: %v", err) } - if lastUpdated.FileName == nil || *lastUpdated.FileName != "new.txt" { + if lastUpdated.Name == nil || *lastUpdated.Name != "new.txt" { t.Fatalf("expected updated file name, got %+v", lastUpdated) } if lastUpdated.Size == nil || *lastUpdated.Size != 123 { @@ -279,7 +279,7 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { if lastCreated.Did != "did-create" || lastCreated.ControlledAccess == nil || len(*lastCreated.ControlledAccess) != 1 || lastCreated.AccessMethods == nil || len(*lastCreated.AccessMethods) != 1 { t.Fatalf("unexpected create-on-upsert payload: %+v", lastCreated) } - if lastCreated.FileName == nil || *lastCreated.FileName != "created.txt" || lastCreated.Size == nil || *lastCreated.Size != 99 { + if lastCreated.Name == nil || *lastCreated.Name != "created.txt" || lastCreated.Size == nil || *lastCreated.Size != 99 { t.Fatalf("unexpected create-on-upsert sizing: %+v", lastCreated) } diff --git a/client/services/metrics_service.go b/client/services/metrics_service.go index 6f0d0ad9..d6afd7eb 100644 --- a/client/services/metrics_service.go +++ b/client/services/metrics_service.go @@ -74,46 +74,6 @@ func (s *MetricsService) File(ctx context.Context, objectID string) (metricsapi. return *resp.JSON200, nil } -func (s *MetricsService) StorageSummary(ctx context.Context, opts StorageSummaryOptions) (metricsapi.StoragePathSummary, error) { - params := &metricsapi.GetStorageSummaryParams{ - Organization: metricsapi.StorageOrganization(opts.Organization), - Project: metricsapi.StorageProject(opts.ProjectID), - Path: stringPtr[metricsapi.Path](opts.Path), - } - resp, err := s.gen.GetStorageSummaryWithResponse(ctx, params) - if err != nil { - return metricsapi.StoragePathSummary{}, err - } - if resp.JSON200 == nil { - return metricsapi.StoragePathSummary{}, fmt.Errorf("failed to get storage metrics summary: %d", resp.StatusCode()) - } - return *resp.JSON200, nil -} - -func (s *MetricsService) StorageChildren(ctx context.Context, opts StorageChildrenOptions) (metricsapi.StoragePathChildrenResponse, error) { - params := &metricsapi.ListStorageChildrenParams{ - Organization: metricsapi.StorageOrganization(opts.Organization), - Project: metricsapi.StorageProject(opts.ProjectID), - Path: stringPtr[metricsapi.Path](opts.Path), - SortBy: stringPtr[metricsapi.ListStorageChildrenParamsSortBy](opts.SortBy), - SortOrder: stringPtr[metricsapi.ListStorageChildrenParamsSortOrder](opts.SortOrder), - } - if opts.Limit > 0 { - params.Limit = &opts.Limit - } - if opts.Offset > 0 { - params.Offset = &opts.Offset - } - resp, err := s.gen.ListStorageChildrenWithResponse(ctx, params) - if err != nil { - return metricsapi.StoragePathChildrenResponse{}, err - } - if resp.JSON200 == nil { - return metricsapi.StoragePathChildrenResponse{}, fmt.Errorf("failed to list storage metrics children: %d", resp.StatusCode()) - } - return *resp.JSON200, nil -} - func (s *MetricsService) TransferSummary(ctx context.Context, opts TransferMetricsOptions) (models.TransferAttributionSummary, error) { params, err := transferSummaryParams(opts) if err != nil { diff --git a/client/services/services_test.go b/client/services/services_test.go index b07db86f..24c5492c 100644 --- a/client/services/services_test.go +++ b/client/services/services_test.go @@ -130,21 +130,15 @@ func (f *fakeBucketClient) DeleteProjectDataWithResponse(ctx context.Context, or } type fakeMetricsClient struct { - summaryResp *metricsapi.GetMetricsSummaryResponse - summaryErr error - summaryParams *metricsapi.GetMetricsSummaryParams - storageSummaryResp *metricsapi.GetStorageSummaryResponse - storageSummaryErr error - storageSummaryParams *metricsapi.GetStorageSummaryParams - filesResp *metricsapi.ListMetricsFilesResponse - filesErr error - filesParams *metricsapi.ListMetricsFilesParams - storageChildrenResp *metricsapi.ListStorageChildrenResponse - storageChildrenErr error - storageChildrenParams *metricsapi.ListStorageChildrenParams - fileResp *metricsapi.GetMetricsFileResponse - fileErr error - fileObjectID string + summaryResp *metricsapi.GetMetricsSummaryResponse + summaryErr error + summaryParams *metricsapi.GetMetricsSummaryParams + filesResp *metricsapi.ListMetricsFilesResponse + filesErr error + filesParams *metricsapi.ListMetricsFilesParams + fileResp *metricsapi.GetMetricsFileResponse + fileErr error + fileObjectID string } func (f *fakeMetricsClient) ListMetricsFilesWithResponse(ctx context.Context, params *metricsapi.ListMetricsFilesParams, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.ListMetricsFilesResponse, error) { @@ -162,16 +156,6 @@ func (f *fakeMetricsClient) GetMetricsSummaryWithResponse(ctx context.Context, p return f.summaryResp, f.summaryErr } -func (f *fakeMetricsClient) GetStorageSummaryWithResponse(ctx context.Context, params *metricsapi.GetStorageSummaryParams, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.GetStorageSummaryResponse, error) { - f.storageSummaryParams = params - return f.storageSummaryResp, f.storageSummaryErr -} - -func (f *fakeMetricsClient) ListStorageChildrenWithResponse(ctx context.Context, params *metricsapi.ListStorageChildrenParams, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.ListStorageChildrenResponse, error) { - f.storageChildrenParams = params - return f.storageChildrenResp, f.storageChildrenErr -} - func (f *fakeMetricsClient) RecordProviderTransferEventsWithBodyWithResponse(ctx context.Context, params *metricsapi.RecordProviderTransferEventsParams, contentType string, body io.Reader, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.RecordProviderTransferEventsResponse, error) { return &metricsapi.RecordProviderTransferEventsResponse{HTTPResponse: &http.Response{StatusCode: http.StatusNotImplemented}}, nil } diff --git a/client/services/types.go b/client/services/types.go index 4f99d5fc..29fbe37a 100644 --- a/client/services/types.go +++ b/client/services/types.go @@ -30,7 +30,7 @@ type ListRecordsOptions struct { type UploadURLRequest struct { FileID string - FileName string + Key string ExpiresIn int Organization string Project string diff --git a/cmd/cli_basic_commands_test.go b/cmd/cli_basic_commands_test.go index bf97751d..f43cce82 100644 --- a/cmd/cli_basic_commands_test.go +++ b/cmd/cli_basic_commands_test.go @@ -813,8 +813,8 @@ func TestSyfonCopyProjectCommand_AcrossInstances(t *testing.T) { if err != nil { t.Fatalf("failed to get copied target record: %v", err) } - if targetRec.FileName == nil || *targetRec.FileName != fileName { - t.Fatalf("unexpected target filename: %+v", targetRec.FileName) + if targetRec.Name == nil || *targetRec.Name != fileName { + t.Fatalf("unexpected target name: %+v", targetRec.Name) } if targetRec.Hashes == nil || (*targetRec.Hashes)["sha256"] != checksum { t.Fatalf("unexpected target hashes: %+v", targetRec.Hashes) @@ -879,7 +879,7 @@ func TestSyfonCopyProjectRefsCommand(t *testing.T) { AccessMethods: sourceRec.AccessMethods, ControlledAccess: sourceRec.ControlledAccess, Description: &description, - FileName: sourceRec.FileName, + Name: sourceRec.Name, Hashes: sourceRec.Hashes, Organization: sourceRec.Organization, Project: sourceRec.Project, @@ -909,8 +909,8 @@ func TestSyfonCopyProjectRefsCommand(t *testing.T) { if err != nil { t.Fatalf("failed to get copied target record: %v", err) } - if targetRec.FileName == nil || *targetRec.FileName != fileName { - t.Fatalf("unexpected target filename: %+v", targetRec.FileName) + if targetRec.Name == nil || *targetRec.Name != fileName { + t.Fatalf("unexpected target name: %+v", targetRec.Name) } if targetRec.Description == nil || *targetRec.Description != description { t.Fatalf("unexpected target description: %+v", targetRec.Description) diff --git a/cmd/copyproject/main.go b/cmd/copyproject/main.go index fb6d9cdb..d3a584ca 100644 --- a/cmd/copyproject/main.go +++ b/cmd/copyproject/main.go @@ -375,8 +375,8 @@ func hasPathPrefix(candidate, prefix []string) bool { func copyRecord(ctx context.Context, cmd *cobra.Command, sourceClient, targetClient services.SyfonClient, rec internalapi.InternalRecord, targetBucket, targetProjectPath string, dstResource string, current, total int, tempDir string) error { did := rec.Did fileName := "" - if rec.FileName != nil { - fileName = *rec.FileName + if rec.Name != nil { + fileName = *rec.Name } size := int64(0) if rec.Size != nil { diff --git a/cmd/download/main.go b/cmd/download/main.go index db2f1cf5..017ff373 100644 --- a/cmd/download/main.go +++ b/cmd/download/main.go @@ -17,7 +17,6 @@ import ( syupload "github.com/calypr/syfon/client/transfer/upload" "github.com/calypr/syfon/cmd/cliauth" "github.com/calypr/syfon/cmd/transferprogress" - "github.com/calypr/syfon/internal/common" "github.com/spf13/cobra" ) @@ -47,10 +46,8 @@ var Cmd = &cobra.Command{ return fmt.Errorf("resolve output filename from record: %w", err) } name := did - if rec.FileName != nil { - if pretty := common.DownloadFilename(*rec.FileName); pretty != "" { - name = pretty - } + if rec.Name != nil && strings.TrimSpace(*rec.Name) != "" { + name = strings.TrimSpace(*rec.Name) } if rec.Size != nil { expectedSize = *rec.Size diff --git a/cmd/list/main.go b/cmd/list/main.go index 0a5f2e31..a3857602 100644 --- a/cmd/list/main.go +++ b/cmd/list/main.go @@ -40,7 +40,6 @@ var Cmd = &cobra.Command{ records, err := listRecords(cmd.Context(), c.Index(), syfonclient.ListRecordsOptions{ Limit: listLimit, Page: listPage, - Path: strings.TrimSpace(listPath), Start: strings.TrimSpace(listStart), Organization: strings.TrimSpace(listOrganization), ProjectID: strings.TrimSpace(listProject), @@ -61,8 +60,8 @@ var Cmd = &cobra.Command{ for _, rec := range records { did := strings.TrimSpace(rec.Did) name := "-" - if rec.FileName != nil { - name = strings.TrimSpace(*rec.FileName) + if rec.Name != nil { + name = strings.TrimSpace(*rec.Name) } size := int64(0) if rec.Size != nil { @@ -93,94 +92,23 @@ func init() { } func listRecords(ctx context.Context, lister indexLister, opts syfonclient.ListRecordsOptions, recursive bool) ([]internalapi.InternalRecord, error) { - if !recursive { - resp, err := lister.List(ctx, opts) - if err != nil { - return nil, err - } - if resp.Records == nil { - return nil, nil - } - return *resp.Records, nil + if strings.TrimSpace(opts.Path) != "" { + return nil, fmt.Errorf("path-based listing is no longer supported") } - if strings.TrimSpace(opts.Organization) == "" || strings.TrimSpace(opts.ProjectID) == "" { - return nil, fmt.Errorf("--recursive requires both --organization and --project") + if recursive { + return nil, fmt.Errorf("--recursive requires both --organization and --project") + } } - if strings.TrimSpace(opts.Start) != "" || opts.Page != 0 { - return nil, fmt.Errorf("--recursive does not support --start or --page") + if recursive { + return nil, fmt.Errorf("path-based recursive listing is no longer supported") } - - remaining := opts.Limit - if remaining <= 0 { - remaining = 100 + resp, err := lister.List(ctx, opts) + if err != nil { + return nil, err } - - queue := []string{strings.TrimSpace(opts.Path)} - seenDirs := map[string]struct{}{} - seenDIDs := map[string]struct{}{} - records := make([]internalapi.InternalRecord, 0, remaining) - - for len(queue) > 0 && remaining > 0 { - currentPath := strings.TrimSpace(queue[0]) - queue = queue[1:] - if _, ok := seenDirs[currentPath]; ok { - continue - } - seenDirs[currentPath] = struct{}{} - - start := "" - for remaining > 0 { - pageLimit := remaining - resp, err := lister.List(ctx, syfonclient.ListRecordsOptions{ - Limit: pageLimit, - Start: start, - Path: currentPath, - Organization: opts.Organization, - ProjectID: opts.ProjectID, - }) - if err != nil { - return nil, err - } - - pageCount := 0 - if resp.Records != nil { - for _, rec := range *resp.Records { - did := strings.TrimSpace(rec.Did) - if did == "" { - continue - } - if _, ok := seenDIDs[did]; ok { - continue - } - seenDIDs[did] = struct{}{} - records = append(records, rec) - remaining-- - pageCount++ - start = did - if remaining == 0 { - break - } - } - } - - if resp.Directories != nil { - for _, dir := range *resp.Directories { - nextPath := strings.TrimSpace(dir.Path) - if nextPath == "" { - continue - } - if _, ok := seenDirs[nextPath]; !ok { - queue = append(queue, nextPath) - } - } - } - - if pageCount == 0 || pageCount < pageLimit { - break - } - } + if resp.Records == nil { + return nil, nil } - - return records, nil + return *resp.Records, nil } diff --git a/cmd/list/main_test.go b/cmd/list/main_test.go index 0eebda1f..fb0c0f6d 100644 --- a/cmd/list/main_test.go +++ b/cmd/list/main_test.go @@ -18,40 +18,21 @@ type fakeIndexLister struct { func (f *fakeIndexLister) List(_ context.Context, opts syfonclient.ListRecordsOptions) (internalapi.ListRecordsResponse, error) { f.calls = append(f.calls, opts) - key := fmt.Sprintf("%s|%s|%s|%d|%s", opts.Organization, opts.ProjectID, opts.Path, opts.Limit, opts.Start) + key := fmt.Sprintf("%s|%s|%d|%s", opts.Organization, opts.ProjectID, opts.Limit, opts.Start) if resp, ok := f.responses[key]; ok { return resp, nil } return internalapi.ListRecordsResponse{}, fmt.Errorf("unexpected list call: %s", key) } -func TestListRecordsRecursiveWalksDirectories(t *testing.T) { +func TestListRecordsNonRecursive(t *testing.T) { rootRecords := []internalapi.InternalRecord{ - {Did: "did-root", FileName: ptr("README.md")}, + {Did: "did-root", Name: ptr("README.md")}, + {Did: "did-data", Name: ptr("file1.txt")}, } - rootDirs := []internalapi.IndexDirectory{ - {Name: "data", Path: "data"}, - {Name: "nested", Path: "nested"}, - } - dataRecords := []internalapi.InternalRecord{ - {Did: "did-data", FileName: ptr("data/file1.txt")}, - } - dataDirs := []internalapi.IndexDirectory{ - {Name: "more", Path: "data/more"}, - } - moreRecords := []internalapi.InternalRecord{ - {Did: "did-more", FileName: ptr("data/more/file2.txt")}, - } - nestedRecords := []internalapi.InternalRecord{ - {Did: "did-nested", FileName: ptr("nested/file3.txt")}, - } - lister := &fakeIndexLister{ responses: map[string]internalapi.ListRecordsResponse{ - "Ellrott_Lab|hla2vec||10|": {Records: &rootRecords, Directories: &rootDirs}, - "Ellrott_Lab|hla2vec|data|9|": {Records: &dataRecords, Directories: &dataDirs}, - "Ellrott_Lab|hla2vec|nested|8|": {Records: &nestedRecords}, - "Ellrott_Lab|hla2vec|data/more|7|": {Records: &moreRecords}, + "Ellrott_Lab|hla2vec|10|": {Records: &rootRecords}, }, } @@ -59,7 +40,7 @@ func TestListRecordsRecursiveWalksDirectories(t *testing.T) { Organization: "Ellrott_Lab", ProjectID: "hla2vec", Limit: 10, - }, true) + }, false) if err != nil { t.Fatalf("listRecords returned error: %v", err) } @@ -68,28 +49,30 @@ func TestListRecordsRecursiveWalksDirectories(t *testing.T) { for _, rec := range records { got = append(got, strings.TrimSpace(rec.Did)) } - want := []string{"did-root", "did-data", "did-nested", "did-more"} + want := []string{"did-root", "did-data"} if !slices.Equal(got, want) { - t.Fatalf("unexpected recursive record set: got %v want %v", got, want) + t.Fatalf("unexpected record set: got %v want %v", got, want) } } -func TestListRecordsRecursiveRequiresScopedProject(t *testing.T) { - _, err := listRecords(context.Background(), &fakeIndexLister{}, syfonclient.ListRecordsOptions{Limit: 10}, true) - if err == nil || !strings.Contains(err.Error(), "--recursive requires both --organization and --project") { - t.Fatalf("expected recursive scope error, got %v", err) - } -} - -func TestListRecordsRecursiveRejectsStartAndPage(t *testing.T) { +func TestListRecordsRejectsRecursive(t *testing.T) { _, err := listRecords(context.Background(), &fakeIndexLister{}, syfonclient.ListRecordsOptions{ Organization: "Ellrott_Lab", ProjectID: "hla2vec", - Start: "did-1", Limit: 10, }, true) - if err == nil || !strings.Contains(err.Error(), "--recursive does not support --start or --page") { - t.Fatalf("expected recursive pagination error, got %v", err) + if err == nil || !strings.Contains(err.Error(), "path-based recursive listing is no longer supported") { + t.Fatalf("expected recursive unsupported error, got %v", err) + } +} + +func TestListRecordsRejectsPathFilter(t *testing.T) { + _, err := listRecords(context.Background(), &fakeIndexLister{}, syfonclient.ListRecordsOptions{ + Path: "nested", + Limit: 10, + }, false) + if err == nil || !strings.Contains(err.Error(), "path-based listing is no longer supported") { + t.Fatalf("expected path unsupported error, got %v", err) } } diff --git a/cmd/projectcopy/helpers.go b/cmd/projectcopy/helpers.go index a9932adb..962e2f93 100644 --- a/cmd/projectcopy/helpers.go +++ b/cmd/projectcopy/helpers.go @@ -266,7 +266,7 @@ func internalRecordFromResponse(rec internalapi.InternalRecordResponse) internal ControlledAccess: rec.ControlledAccess, CreatedTime: rec.CreatedTime, Description: rec.Description, - FileName: rec.FileName, + Name: rec.Name, Hashes: rec.Hashes, Organization: rec.Organization, Project: rec.Project, @@ -278,63 +278,32 @@ func internalRecordFromResponse(rec internalapi.InternalRecordResponse) internal func listAllProjectRecords(ctx context.Context, index *services.IndexService, org, project string) ([]internalapi.InternalRecord, error) { var records []internalapi.InternalRecord - queue := []string{""} - seenDirs := map[string]struct{}{} - seenDIDs := map[string]struct{}{} - - for len(queue) > 0 { - currentPath := queue[0] - queue = queue[1:] - if _, ok := seenDirs[currentPath]; ok { - continue + start := "" + for { + resp, err := index.List(ctx, services.ListRecordsOptions{ + Limit: 1000, + Start: start, + Organization: org, + ProjectID: project, + }) + if err != nil { + return nil, err } - seenDirs[currentPath] = struct{}{} - - start := "" - for { - resp, err := index.List(ctx, services.ListRecordsOptions{ - Limit: 1000, - Start: start, - Path: currentPath, - Organization: org, - ProjectID: project, - }) - if err != nil { - return nil, err - } - - pageCount := 0 - if resp.Records != nil { - for _, rec := range *resp.Records { - did := strings.TrimSpace(rec.Did) - if did == "" { - continue - } - if _, ok := seenDIDs[did]; ok { - continue - } - seenDIDs[did] = struct{}{} - records = append(records, rec) - pageCount++ - start = did - } - } - - if resp.Directories != nil { - for _, dir := range *resp.Directories { - nextPath := strings.TrimSpace(dir.Path) - if nextPath == "" { - continue - } - if _, ok := seenDirs[nextPath]; !ok { - queue = append(queue, nextPath) - } - } - } - - if pageCount == 0 || pageCount < 1000 { - break + if resp.Records == nil || len(*resp.Records) == 0 { + break + } + pageCount := 0 + for _, rec := range *resp.Records { + did := strings.TrimSpace(rec.Did) + if did == "" { + continue } + records = append(records, rec) + start = did + pageCount++ + } + if pageCount < 1000 { + break } } return records, nil diff --git a/cmd/projectcopy/helpers_test.go b/cmd/projectcopy/helpers_test.go index c67d9cc3..517ff63f 100644 --- a/cmd/projectcopy/helpers_test.go +++ b/cmd/projectcopy/helpers_test.go @@ -187,20 +187,13 @@ func writeProfileConfig(t *testing.T, home, content string) { } func testIndexService() *services.IndexService { - listByPath := map[string]internalapi.ListRecordsResponse{ + listPages := map[string]internalapi.ListRecordsResponse{ "": { Records: &[]internalapi.InternalRecord{ { Did: "did-1", ControlledAccess: &[]string{"/organization/syfon/project/e2e"}, }, - }, - Directories: &[]internalapi.IndexDirectory{ - {Name: "nested", Path: "nested"}, - }, - }, - "nested": { - Records: &[]internalapi.InternalRecord{ { Did: "did-2", ControlledAccess: &[]string{"/organization/syfon/project/e2e"}, @@ -233,8 +226,8 @@ func testIndexService() *services.IndexService { }, }, stubRequester{ - listResponse: func(path string) (internalapi.ListRecordsResponse, error) { - if resp, ok := listByPath[path]; ok { + listResponse: func(start string) (internalapi.ListRecordsResponse, error) { + if resp, ok := listPages[start]; ok { return resp, nil } return internalapi.ListRecordsResponse{}, nil @@ -256,7 +249,7 @@ func (s stubInternalClient) InternalGetWithResponse(ctx context.Context, id stri } type stubRequester struct { - listResponse func(path string) (internalapi.ListRecordsResponse, error) + listResponse func(start string) (internalapi.ListRecordsResponse, error) } func (s stubRequester) Do(ctx context.Context, method, path string, body, out any, opts ...request.RequestOption) error { @@ -267,15 +260,15 @@ func (s stubRequester) Do(ctx context.Context, method, path string, body, out an for _, opt := range opts { opt(builder) } - listPath := "" + start := "" if strings.Contains(builder.Url, "?") { values, err := url.ParseQuery(strings.TrimPrefix(strings.SplitN(builder.Url, "?", 2)[1], "?")) if err != nil { return err } - listPath = values.Get("path") + start = values.Get("start") } - resp, err := s.listResponse(listPath) + resp, err := s.listResponse(start) if err != nil { return err } diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index 59dffcca..20254def 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -66,13 +66,7 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { hash := c.Query("hash") hashType := c.Query("hash_type") objectURL := strings.TrimSpace(c.Query("url")) - hasPathQuery := c.Request().URI().QueryArgs().Has("path") - pathQuery := c.Query("path") - if hash != "" { - if hasPathQuery { - return c.Status(fiber.StatusBadRequest).SendString("path is only supported for exact organization/project listings") - } hashType, hash = common.ParseHashQuery(hash, hashType) filterOrg := strings.TrimSpace(c.Query("organization")) filterProject := strings.TrimSpace(c.Query("project")) @@ -102,45 +96,10 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { if !hasScope { filterOrg, filterProject = "", "" } - if hasPathQuery && (filterOrg == "" || filterProject == "") { - return c.Status(fiber.StatusBadRequest).SendString("organization and project are required when path is set") - } - if hasPathQuery && objectURL != "" { - return c.Status(fiber.StatusBadRequest).SendString("path is only supported for exact organization/project listings") - } - if _, _, err := common.NormalizeBrowsePath(pathQuery); err != nil { - return c.Status(fiber.StatusBadRequest).SendString(err.Error()) - } - limit, start, offset, err := parseInternalListPaginationFiber(c) if err != nil { return c.Status(fiber.StatusBadRequest).SendString(err.Error()) } - if hasPathQuery { - ids, directories, err := om.ListObjectIDsPageByPath(c.Context(), filterOrg, filterProject, pathQuery, "read", start, limit, offset) - if err != nil { - return apiutil.HandleError(c, err) - } - objs, err := om.GetBulkObjects(c.Context(), ids, "read") - if err != nil { - return apiutil.HandleError(c, err) - } - records := make([]internalapi.InternalRecord, 0, len(objs)) - for _, obj := range objs { - records = append(records, core.InternalObjectToInternalRecord(obj)) - } - respDirectories := make([]internalapi.IndexDirectory, 0, len(directories)) - for _, directory := range directories { - respDirectories = append(respDirectories, internalapi.IndexDirectory{ - Name: directory.Name, - Path: directory.Path, - }) - } - return c.JSON(internalapi.ListRecordsResponse{ - Directories: &respDirectories, - Records: &records, - }) - } var ids []string if objectURL != "" { diff --git a/internal/api/internaldrs/index_additional_test.go b/internal/api/internaldrs/index_additional_test.go index 48a23383..4f7bda81 100644 --- a/internal/api/internaldrs/index_additional_test.go +++ b/internal/api/internaldrs/index_additional_test.go @@ -81,18 +81,18 @@ func TestHandleInternalBulkDocuments_InvalidBodyAndMissingIDs(t *testing.T) { } } -func TestHandleInternalList_PathBrowseValidation(t *testing.T) { +func TestHandleInternalList_IgnoresLegacyPathValidation(t *testing.T) { om := core.NewObjectManager(&testutils.MockDatabase{}, &testutils.MockUrlManager{}) req := httptest.NewRequest(http.MethodGet, "/index?path=nested", nil) rr := doInternalDRSTestRequest(req, om) - if rr.Code != http.StatusBadRequest { - t.Fatalf("expected 400 when path is set without exact scope, got %d body=%s", rr.Code, rr.Body.String()) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200 when legacy path query is ignored, got %d body=%s", rr.Code, rr.Body.String()) } req = httptest.NewRequest(http.MethodGet, "/index?organization=org&project=proj&path=../nested", nil) rr = doInternalDRSTestRequest(req, om) - if rr.Code != http.StatusBadRequest { - t.Fatalf("expected 400 for invalid path, got %d body=%s", rr.Code, rr.Body.String()) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200 when invalid legacy path query is ignored, got %d body=%s", rr.Code, rr.Body.String()) } } diff --git a/internal/api/internaldrs/index_test.go b/internal/api/internaldrs/index_test.go index 789f5c80..3e20a024 100644 --- a/internal/api/internaldrs/index_test.go +++ b/internal/api/internaldrs/index_test.go @@ -333,7 +333,7 @@ func TestHandleInternalList_LimitIsCappedAtTenThousand(t *testing.T) { } } -func TestHandleInternalList_PathBrowseReturnsImmediateDirectoriesAndFiles(t *testing.T) { +func TestHandleInternalList_IgnoresLegacyPathQuery(t *testing.T) { now := time.Now().UTC() mockDB := &testutils.MockDatabase{ Objects: map[string]*drs.DrsObject{ @@ -366,11 +366,8 @@ func TestHandleInternalList_PathBrowseReturnsImmediateDirectoriesAndFiles(t *tes if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { t.Fatalf("decode response: %v", err) } - if payload.Directories == nil || len(*payload.Directories) != 1 || (*payload.Directories)[0].Name != "deep" || (*payload.Directories)[0].Path != "nested/deep" { - t.Fatalf("unexpected directories: %+v", payload.Directories) - } - if payload.Records == nil || len(*payload.Records) != 1 || (*payload.Records)[0].Did != "obj-a" { - t.Fatalf("unexpected paged records: %+v", payload.Records) + if payload.Records == nil || len(*payload.Records) != 1 { + t.Fatalf("expected one paged record, got %+v", payload.Records) } req = httptest.NewRequest(http.MethodGet, "/index?organization=org-a&project=proj-a&path=nested&limit=1&start=obj-a", nil) @@ -382,14 +379,12 @@ func TestHandleInternalList_PathBrowseReturnsImmediateDirectoriesAndFiles(t *tes body, _ := io.ReadAll(resp.Body) t.Fatalf("expected 200, got %d body=%s", resp.StatusCode, string(body)) } + payload = internalapi.ListRecordsResponse{} if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - t.Fatalf("decode start response: %v", err) - } - if payload.Directories == nil || len(*payload.Directories) != 1 { - t.Fatalf("expected directories on paged response, got %+v", payload.Directories) + t.Fatalf("decode paged response: %v", err) } - if payload.Records == nil || len(*payload.Records) != 1 || (*payload.Records)[0].Did != "obj-d" { - t.Fatalf("expected obj-d after start cursor, got %+v", payload.Records) + if payload.Records == nil || len(*payload.Records) != 1 { + t.Fatalf("expected one paged record with start cursor, got %+v", payload.Records) } } diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go index 60361c4d..6c8adbda 100644 --- a/internal/api/internaldrs/repair_cleanup.go +++ b/internal/api/internaldrs/repair_cleanup.go @@ -259,7 +259,6 @@ func clientRecordFromServer(rec serverinternalapi.InternalRecord) clientinternal Size: rec.Size, CreatedTime: rec.CreatedTime, Description: rec.Description, - FileName: rec.FileName, Name: rec.Name, Version: rec.Version, UpdatedTime: rec.UpdatedTime, @@ -278,7 +277,6 @@ func serverRecordFromClient(rec clientinternalapi.InternalRecord) serverinternal Size: rec.Size, CreatedTime: rec.CreatedTime, Description: rec.Description, - FileName: rec.FileName, Name: rec.Name, Version: rec.Version, UpdatedTime: rec.UpdatedTime, @@ -297,7 +295,6 @@ func clientRecordResponseFromServer(rec serverinternalapi.InternalRecordResponse Size: rec.Size, CreatedTime: rec.CreatedTime, Description: rec.Description, - FileName: rec.FileName, Name: rec.Name, Version: rec.Version, UpdatedTime: rec.UpdatedTime, diff --git a/internal/api/internaldrs/transfer.go b/internal/api/internaldrs/transfer.go index 3ddaa913..6d7cde18 100644 --- a/internal/api/internaldrs/transfer.go +++ b/internal/api/internaldrs/transfer.go @@ -209,7 +209,7 @@ func handleInternalUploadURLFiber(om *core.ObjectManager) fiber.Handler { } else { target, err = om.ResolveCanonicalStorageTarget(c.Context(), core.CanonicalStorageTargetRequest{ Object: obj, - Key: strings.TrimSpace(common.StringVal(params.FileName)), + Key: strings.TrimSpace(common.StringVal(params.Key)), PreferChecksum: true, }) } @@ -228,7 +228,7 @@ func handleInternalUploadURLFiber(om *core.ObjectManager) fiber.Handler { } return c.JSON(internalapi.InternalSignedURL{Url: &signedURL}) } else { - if requestedKey := strings.Trim(strings.TrimSpace(firstNonEmpty(common.StringVal(params.FileName), c.Query("file_name"), c.Query("filename"))), "/"); requestedKey != "" { + if requestedKey := strings.Trim(strings.TrimSpace(firstNonEmpty(common.StringVal(params.Key), c.Query("key"))), "/"); requestedKey != "" { key = requestedKey } target, err := resolveUploadTarget(c.Context(), om, common.StringVal(params.Organization), common.StringVal(params.Project), key) @@ -258,7 +258,7 @@ func handleInternalUploadURLFiber(om *core.ObjectManager) fiber.Handler { } func uploadKeyForExistingObject(obj *models.InternalObject, params internalapi.InternalUploadURLParams) string { - if key := strings.Trim(strings.TrimSpace(common.StringVal(params.FileName)), "/"); key != "" { + if key := strings.Trim(strings.TrimSpace(common.StringVal(params.Key)), "/"); key != "" { return key } if obj != nil { @@ -296,7 +296,7 @@ func handleInternalUploadBulkFiber(om *core.ObjectManager) fiber.Handler { results := make([]internalapi.InternalUploadBulkResult, 0, len(req.Requests)) for _, item := range req.Requests { - res := internalapi.InternalUploadBulkResult{FileId: item.FileId, FileName: item.FileName} + res := internalapi.InternalUploadBulkResult{FileId: item.FileId, Key: item.Key} if item.FileId == "" { errMsg := "FileId is required" res.Error = &errMsg @@ -323,7 +323,7 @@ func handleInternalUploadBulkFiber(om *core.ObjectManager) fiber.Handler { target, err := om.ResolveCanonicalStorageTarget(c.Context(), core.CanonicalStorageTargetRequest{ Object: obj, - Key: strings.TrimSpace(common.StringVal(item.FileName)), + Key: strings.TrimSpace(common.StringVal(item.Key)), PreferChecksum: true, }) if err != nil { @@ -353,7 +353,7 @@ func handleInternalUploadBulkFiber(om *core.ObjectManager) fiber.Handler { } else { res.Url = &signedURL res.Bucket = &bucket - res.FileName = &key + res.Key = &key res.Status = http.StatusOK } results = append(results, res) @@ -380,11 +380,11 @@ func handleInternalMultipartInitFiber(om *core.ObjectManager) fiber.Handler { key := "" if req.Guid != nil { key = strings.TrimSpace(*req.Guid) - } else if req.FileName != nil { - key = strings.TrimSpace(*req.FileName) + } else if req.Key != nil { + key = strings.TrimSpace(*req.Key) } if key == "" { - return c.Status(fiber.StatusBadRequest).SendString("key/guid/file_name is required") + return c.Status(fiber.StatusBadRequest).SendString("key/guid is required") } if strings.Contains(key, "/") { diff --git a/internal/api/internaldrs/upload_routes_test.go b/internal/api/internaldrs/upload_routes_test.go index c73b9bff..1b69b798 100644 --- a/internal/api/internaldrs/upload_routes_test.go +++ b/internal/api/internaldrs/upload_routes_test.go @@ -89,7 +89,7 @@ func TestHandleInternalUploadBlank_RequiresScope(t *testing.T) { func TestHandleInternalUploadURL_MissingObjectResolvesOrganizationProjectScope(t *testing.T) { mockUM := &capturingMultipartURLManager{} req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/new-guid?organization=syfon&project=e2e&file_name=payload.bin", nil), + httptest.NewRequest(http.MethodGet, "/data/upload/new-guid?organization=syfon&project=e2e&key=payload.bin", nil), map[string]string{"file_id": "new-guid"}, ) rr := doInternalDRSTestRequest(req, core.NewObjectManager(&testutils.MockDatabase{ @@ -123,7 +123,7 @@ func TestHandleInternalMultipartInit(t *testing.T) { guid := "multipart-guid" org := "syfon" project := "e2e" - body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Guid: &guid, FileName: &fileName, Organization: &org, Project: &project}) + body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Guid: &guid, Key: &fileName, Organization: &org, Project: &project}) rr := doInternalDRSTestRequest(httptest.NewRequest(http.MethodPost, "/data/multipart/init", bytes.NewBuffer(body)), core.NewObjectManager(&testutils.MockDatabase{ Objects: map[string]*drs.DrsObject{}, Credentials: map[string]models.S3Credential{"b1": {Bucket: "b1"}}, @@ -139,7 +139,7 @@ func TestHandleInternalMultipartInit(t *testing.T) { func TestHandleInternalMultipartInit_RequiresScopeForNewUpload(t *testing.T) { fileName := "test.bam" guid := "multipart-guid" - body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Guid: &guid, FileName: &fileName}) + body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Guid: &guid, Key: &fileName}) rr := doInternalDRSTestRequest(httptest.NewRequest(http.MethodPost, "/data/multipart/init", bytes.NewBuffer(body)), core.NewObjectManager(&testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}}, &testutils.MockUrlManager{})) if rr.Code != http.StatusBadRequest { t.Fatalf("expected 400, got %d", rr.Code) @@ -197,7 +197,7 @@ func TestHandleInternalMultipartInit_PreservesRequestedKey(t *testing.T) { func TestHandleInternalMultipartInit_MintsUUIDForChecksumInput(t *testing.T) { checksum := strings.Repeat("a", 64) - body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{FileName: &checksum}) + body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Key: &checksum}) mockDB := &testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}} rr := doInternalDRSTestRequest(httptest.NewRequest(http.MethodPost, "/data/multipart/init", bytes.NewBuffer(body)), core.NewObjectManager(mockDB, &testutils.MockUrlManager{})) if rr.Code != http.StatusBadRequest { @@ -407,7 +407,7 @@ func TestHandleInternalUploadURL_ResolvesRegisteredScopedObjectID(t *testing.T) mockUM := &capturingMultipartURLManager{} om = core.NewObjectManager(database, mockUM) req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/"+registered.Id+"?file_name=program-root/"+oid, nil), + httptest.NewRequest(http.MethodGet, "/data/upload/"+registered.Id+"?key=program-root/"+oid, nil), map[string]string{"file_id": registered.Id}, ) rr := doInternalDRSTestRequest(req, om) @@ -616,7 +616,7 @@ func TestHandleInternalUploadURL_UsesExplicitObjectKeyForExistingObject(t *testi } mockUM := &capturingMultipartURLManager{} req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/7b9de5b9-19b2-536f-abcc-fe2a146c4eb5?file_name=program-root/"+checksum, nil), + httptest.NewRequest(http.MethodGet, "/data/upload/7b9de5b9-19b2-536f-abcc-fe2a146c4eb5?key=program-root/"+checksum, nil), map[string]string{"file_id": "7b9de5b9-19b2-536f-abcc-fe2a146c4eb5"}, ) rr := doInternalDRSTestRequest(req, core.NewObjectManager(db, mockUM)) @@ -670,7 +670,7 @@ func TestHandleInternalUploadURL_ExplicitScopeOverridesMalformedExistingObjectUR } mockUM := &capturingMultipartURLManager{} req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/f781273b-52eb-5ac2-a484-775235eef303?organization=syfon&project=e2e&file_name=project-subpath/"+checksum, nil), + httptest.NewRequest(http.MethodGet, "/data/upload/f781273b-52eb-5ac2-a484-775235eef303?organization=syfon&project=e2e&key=project-subpath/"+checksum, nil), map[string]string{"file_id": "f781273b-52eb-5ac2-a484-775235eef303"}, ) rr := doInternalDRSTestRequest(req, core.NewObjectManager(db, mockUM)) @@ -719,7 +719,7 @@ func TestHandleInternalUploadURL_ExplicitScopeIgnoresConflictingObjectMetadata(t } mockUM := &capturingMultipartURLManager{} req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/4f74e0c2-3c80-5c19-b47c-061b300ae270?organization=syfon&project=e2e&file_name="+checksum, nil), + httptest.NewRequest(http.MethodGet, "/data/upload/4f74e0c2-3c80-5c19-b47c-061b300ae270?organization=syfon&project=e2e&key="+checksum, nil), map[string]string{"file_id": "4f74e0c2-3c80-5c19-b47c-061b300ae270"}, ) rr := doInternalDRSTestRequest(req, core.NewObjectManager(db, mockUM)) diff --git a/internal/api/metrics/storage_summary.go b/internal/api/metrics/storage_summary.go deleted file mode 100644 index f4075990..00000000 --- a/internal/api/metrics/storage_summary.go +++ /dev/null @@ -1,140 +0,0 @@ -package metrics - -import ( - "context" - "net/http" - "strings" - - "github.com/calypr/syfon/apigen/server/metricsapi" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" - "github.com/calypr/syfon/internal/storagemetrics" -) - -func (s *MetricsServer) GetStorageSummary(ctx context.Context, request metricsapi.GetStorageSummaryRequestObject) (metricsapi.GetStorageSummaryResponseObject, error) { - access, statusCode, ok := s.checkAuth(ctx) - if !ok { - return getStorageSummaryAuthResponse(statusCode), nil - } - if !access.isScoped() || strings.TrimSpace(access.project) == "" { - return metricsapi.GetStorageSummary400Response{}, nil - } - normalizedPath, _, err := common.NormalizeBrowsePath(pointerString(request.Params.Path)) - if err != nil { - return metricsapi.GetStorageSummary400Response{}, nil - } - summary, err := s.database.GetStoragePathSummary(ctx, access.organization, access.project, normalizedPath) - if err != nil { - return metricsapi.GetStorageSummary500Response{}, nil - } - return metricsapi.GetStorageSummary200JSONResponse(storageSummaryToGenerated(summary)), nil -} - -func (s *MetricsServer) ListStorageChildren(ctx context.Context, request metricsapi.ListStorageChildrenRequestObject) (metricsapi.ListStorageChildrenResponseObject, error) { - limit := 200 - if request.Params.Limit != nil { - limit = *request.Params.Limit - } - offset := 0 - if request.Params.Offset != nil { - offset = *request.Params.Offset - } - if limit < 1 || limit > 1000 || offset < 0 { - return metricsapi.ListStorageChildren400Response{}, nil - } - access, statusCode, ok := s.checkAuth(ctx) - if !ok { - return getStorageChildrenAuthResponse(statusCode), nil - } - if !access.isScoped() || strings.TrimSpace(access.project) == "" { - return metricsapi.ListStorageChildren400Response{}, nil - } - normalizedPath, _, err := common.NormalizeBrowsePath(pointerString(request.Params.Path)) - if err != nil { - return metricsapi.ListStorageChildren400Response{}, nil - } - sortBy, sortOrder, err := storagemetrics.NormalizeStorageChildrenSort(pointerString(request.Params.SortBy), pointerString(request.Params.SortOrder)) - if err != nil { - return metricsapi.ListStorageChildren400Response{}, nil - } - items, err := s.database.ListStoragePathChildren(ctx, access.organization, access.project, normalizedPath, limit, offset, sortBy, sortOrder) - if err != nil { - return metricsapi.ListStorageChildren500Response{}, nil - } - return metricsapi.ListStorageChildren200JSONResponse(storageChildrenToGenerated(access.organization, access.project, normalizedPath, items)), nil -} - -func getStorageSummaryAuthResponse(statusCode int) metricsapi.GetStorageSummaryResponseObject { - switch statusCode { - case http.StatusUnauthorized: - return metricsapi.GetStorageSummary401Response{} - case http.StatusForbidden: - return metricsapi.GetStorageSummary403Response{} - default: - return metricsapi.GetStorageSummary400Response{} - } -} - -func getStorageChildrenAuthResponse(statusCode int) metricsapi.ListStorageChildrenResponseObject { - switch statusCode { - case http.StatusUnauthorized: - return metricsapi.ListStorageChildren401Response{} - case http.StatusForbidden: - return metricsapi.ListStorageChildren403Response{} - default: - return metricsapi.ListStorageChildren400Response{} - } -} - -func storageSummaryToGenerated(v models.StoragePathSummary) metricsapi.StoragePathSummary { - return metricsapi.StoragePathSummary{ - Organization: &v.Organization, - Project: &v.Project, - Path: &v.Path, - FileCount: &v.FileCount, - RecordCount: &v.RecordCount, - TotalBytes: &v.TotalBytes, - DownloadCount: &v.DownloadCount, - DirectChildCount: &v.DirectChildCount, - DuplicatePathCount: &v.DuplicatePathCount, - LatestUpdateTime: v.LatestUpdateTime, - LastDownloadTime: v.LastDownloadTime, - } -} - -func storageChildrenToGenerated(organization, project, path string, items []models.StoragePathChild) metricsapi.StoragePathChildrenResponse { - out := metricsapi.StoragePathChildrenResponse{ - Organization: &organization, - Project: &project, - Path: &path, - } - if len(items) == 0 { - empty := make([]metricsapi.StoragePathChild, 0) - out.Items = &empty - return out - } - generated := make([]metricsapi.StoragePathChild, 0, len(items)) - for _, item := range items { - childType := metricsapi.StoragePathChildType(item.Type) - generated = append(generated, metricsapi.StoragePathChild{ - Name: &item.Name, - Path: &item.Path, - Type: &childType, - FileCount: &item.FileCount, - RecordCount: &item.RecordCount, - TotalBytes: &item.TotalBytes, - LatestUpdateTime: item.LatestUpdateTime, - DownloadCount: &item.DownloadCount, - LastDownloadTime: item.LastDownloadTime, - }) - } - out.Items = &generated - return out -} - -func pointerString[T ~string](v *T) string { - if v == nil { - return "" - } - return string(*v) -} diff --git a/internal/api/metrics/storage_summary_test.go b/internal/api/metrics/storage_summary_test.go deleted file mode 100644 index a7b95861..00000000 --- a/internal/api/metrics/storage_summary_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package metrics - -import ( - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/calypr/syfon/apigen/server/drs" - "github.com/calypr/syfon/apigen/server/metricsapi" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" - "github.com/calypr/syfon/internal/testutils" - "github.com/gofiber/fiber/v3" -) - -func TestStorageMetricsRoutes_SummaryAndChildren(t *testing.T) { - now := time.Now().UTC() - db := &testutils.MockDatabase{ - Objects: map[string]*drs.DrsObject{ - "obj-1": {Id: "obj-1", Name: common.Ptr("data/a.txt"), Size: 10, UpdatedTime: &now}, - "obj-2": {Id: "obj-2", Name: common.Ptr("data/a.txt"), Size: 20, UpdatedTime: &now}, - "obj-3": {Id: "obj-3", Name: common.Ptr("data/nested/b.txt"), Size: 30, UpdatedTime: &now}, - }, - ObjectAuthz: map[string]map[string][]string{ - "obj-1": {"cbds": {"end_to_end_test"}}, - "obj-2": {"cbds": {"end_to_end_test"}}, - "obj-3": {"cbds": {"end_to_end_test"}}, - }, - Usage: map[string]models.FileUsage{ - "obj-1": {ObjectID: "obj-1", DownloadCount: 2, LastDownloadTime: timePtr(now.Add(-3 * time.Hour))}, - "obj-2": {ObjectID: "obj-2", DownloadCount: 4, LastDownloadTime: timePtr(now.Add(-time.Hour))}, - "obj-3": {ObjectID: "obj-3", DownloadCount: 1, LastDownloadTime: timePtr(now.Add(-2 * time.Hour))}, - }, - } - app := fiber.New() - app.Use(func(c fiber.Ctx) error { - if mode := c.Get("X-Test-Auth-Mode"); mode != "" { - var privs map[string]map[string]bool - if privsJSON := c.Get("X-Test-Privileges"); privsJSON != "" { - _ = json.Unmarshal([]byte(privsJSON), &privs) - } - ctx := metricsTestContext(c.Context(), mode, c.Get("X-Test-Auth-Header") == "true", c.Get("X-Test-Auth-Header") == "true", privs) - c.SetContext(ctx) - } - return c.Next() - }) - RegisterMetricsRoutes(app, db) - - privs, _ := json.Marshal(map[string]map[string]bool{ - "/programs/cbds/projects/end_to_end_test": {"read": true}, - }) - - t.Run("summary", func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/index/v1/metrics/storage/summary?organization=cbds&project=end_to_end_test&path=data", nil) - req.Header.Set("X-Test-Auth-Mode", "gen3") - req.Header.Set("X-Test-Auth-Header", "true") - req.Header.Set("X-Test-Privileges", string(privs)) - httpResp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - body, _ := io.ReadAll(httpResp.Body) - if httpResp.StatusCode != http.StatusOK { - t.Fatalf("expected 200, got %d body=%s", httpResp.StatusCode, string(body)) - } - var resp metricsapi.StoragePathSummary - if err := json.Unmarshal(body, &resp); err != nil { - t.Fatalf("decode: %v", err) - } - if resp.RecordCount == nil || *resp.RecordCount != 3 || resp.FileCount == nil || *resp.FileCount != 2 || resp.DuplicatePathCount == nil || *resp.DuplicatePathCount != 1 { - t.Fatalf("unexpected storage summary: %+v", resp) - } - if resp.DownloadCount == nil || *resp.DownloadCount != 7 { - t.Fatalf("unexpected download count: %+v", resp) - } - if resp.LastDownloadTime == nil || !resp.LastDownloadTime.Equal(now.Add(-time.Hour)) { - t.Fatalf("unexpected last download time: %+v", resp.LastDownloadTime) - } - }) - - t.Run("children", func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/index/v1/metrics/storage/children?organization=cbds&project=end_to_end_test&path=data&sort_by=bytes&sort_order=desc", nil) - req.Header.Set("X-Test-Auth-Mode", "gen3") - req.Header.Set("X-Test-Auth-Header", "true") - req.Header.Set("X-Test-Privileges", string(privs)) - httpResp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - body, _ := io.ReadAll(httpResp.Body) - if httpResp.StatusCode != http.StatusOK { - t.Fatalf("expected 200, got %d body=%s", httpResp.StatusCode, string(body)) - } - var resp metricsapi.StoragePathChildrenResponse - if err := json.Unmarshal(body, &resp); err != nil { - t.Fatalf("decode: %v", err) - } - if resp.Items == nil || len(*resp.Items) != 2 { - t.Fatalf("unexpected children response: %+v", resp) - } - first := (*resp.Items)[0] - if first.Name == nil || *first.Name != "nested" || first.Type == nil || *first.Type != metricsapi.Directory { - t.Fatalf("unexpected first child: %+v", first) - } - if first.DownloadCount == nil || *first.DownloadCount != 1 { - t.Fatalf("unexpected first child download count: %+v", first) - } - }) -} - -func TestStorageMetricsRoutes_Validation(t *testing.T) { - app := fiber.New() - RegisterMetricsRoutes(app, &testutils.MockDatabase{}) - - req := httptest.NewRequest(http.MethodGet, "/index/v1/metrics/storage/children?organization=cbds&project=end_to_end_test&path=../bad", nil) - httpResp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if httpResp.StatusCode != http.StatusBadRequest { - body, _ := io.ReadAll(httpResp.Body) - t.Fatalf("expected 400, got %d body=%s", httpResp.StatusCode, string(body)) - } -} - -func timePtr(t time.Time) *time.Time { - return &t -} diff --git a/internal/core/converters.go b/internal/core/converters.go index ce4db2fa..ee7177cd 100644 --- a/internal/core/converters.go +++ b/internal/core/converters.go @@ -2,7 +2,6 @@ package core import ( "fmt" - "path" "strings" "time" @@ -252,15 +251,6 @@ func InternalRecordToInternalObject(r internalapi.InternalRecord, now time.Time) obj.UpdatedTime = &updatedTime if r.Name != nil && strings.TrimSpace(*r.Name) != "" { obj.Name = common.Ptr(strings.TrimSpace(*r.Name)) - } else if r.FileName != nil { - fileName := strings.TrimSpace(*r.FileName) - if fileName != "" { - base := path.Base(strings.Trim(fileName, "/")) - if base == "." || base == "/" { - base = fileName - } - obj.Name = common.Ptr(base) - } } if v := r.Version; v != nil { obj.Version = v @@ -287,9 +277,7 @@ func InternalRecordToInternalObject(r internalapi.InternalRecord, now time.Time) internalObj := models.InternalObject{ DrsObject: obj, Authorizations: authzMap, - Properties: map[string]interface{}{ - "file_name": common.StringVal(r.FileName), - }, + Properties: map[string]interface{}{}, } return EnforceCanonicalProjectScope(internalObj, common.StringVal(r.Organization), common.StringVal(r.Project)) } @@ -308,16 +296,11 @@ func parseInternalRecordTime(raw *string, fallback time.Time) time.Time { // InternalObjectToInternalRecord converts our internal domain model back to an API record. func InternalObjectToInternalRecord(obj models.InternalObject) internalapi.InternalRecord { - fileName := common.StringVal(obj.Name) - if raw, ok := obj.Properties["file_name"].(string); ok && strings.TrimSpace(raw) != "" { - fileName = strings.TrimSpace(raw) - } res := internalapi.InternalRecord{ Did: obj.Id, Size: &obj.Size, CreatedTime: common.Ptr(obj.CreatedTime.Format(time.RFC3339)), Description: obj.Description, - FileName: common.Ptr(fileName), Name: obj.Name, Version: obj.Version, AccessMethods: obj.AccessMethods, @@ -348,7 +331,6 @@ func InternalObjectToInternalRecordResponse(obj models.InternalObject) internala Size: rec.Size, CreatedTime: rec.CreatedTime, Description: rec.Description, - FileName: rec.FileName, Name: rec.Name, Version: rec.Version, UpdatedTime: rec.UpdatedTime, diff --git a/internal/db/postgres/metrics.go b/internal/db/postgres/metrics.go index 7d993d74..d5162f71 100644 --- a/internal/db/postgres/metrics.go +++ b/internal/db/postgres/metrics.go @@ -560,7 +560,7 @@ func (db *PostgresDB) listStorageMetricRowsByScope(ctx context.Context, organiza for rows.Next() { var row models.DrsObjectRecord var lastDownload sql.NullTime - if err := rows.Scan(&row.ID, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload, &row.Name, &row.FileName); err != nil { + if err := rows.Scan(&row.ID, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload, &row.Name, &row.Path); err != nil { return nil, err } if lastDownload.Valid { diff --git a/internal/db/postgres/objects.go b/internal/db/postgres/objects.go index c67f71a7..cd2ee6e4 100644 --- a/internal/db/postgres/objects.go +++ b/internal/db/postgres/objects.go @@ -17,8 +17,11 @@ import ( "github.com/lib/pq" ) -func postgresStoredFileName(obj *models.InternalObject) string { +func postgresStoredPath(obj *models.InternalObject) string { if obj != nil && obj.Properties != nil { + if raw, ok := obj.Properties["path"].(string); ok && strings.TrimSpace(raw) != "" { + return strings.TrimSpace(raw) + } if raw, ok := obj.Properties["file_name"].(string); ok && strings.TrimSpace(raw) != "" { return strings.TrimSpace(raw) } @@ -31,18 +34,18 @@ func postgresStoredFileName(obj *models.InternalObject) string { func postgresLoadedNames(name, fileName sql.NullString) (string, string) { loadedName := strings.TrimSpace(name.String) - loadedFileName := strings.TrimSpace(fileName.String) - if loadedFileName == "" { - loadedFileName = loadedName + loadedPath := strings.TrimSpace(fileName.String) + if loadedPath == "" { + loadedPath = loadedName loadedName = "" } - if loadedName == "" && loadedFileName != "" { - loadedName = path.Base(strings.Trim(loadedFileName, "/")) + if loadedName == "" && loadedPath != "" { + loadedName = path.Base(strings.Trim(loadedPath, "/")) if loadedName == "." || loadedName == "/" || loadedName == "" { - loadedName = loadedFileName + loadedName = loadedPath } } - return loadedName, loadedFileName + return loadedName, loadedPath } func (db *PostgresDB) DeleteObject(ctx context.Context, id string) error { @@ -162,7 +165,7 @@ retryLookup: if err != nil { return nil, fmt.Errorf("failed to fetch record: %w", err) } - r.Name, r.FileName = postgresLoadedNames(name, fileName) + r.Name, r.Path = postgresLoadedNames(name, fileName) r.Version = version.String r.Description = description.String objectID := r.ID @@ -183,8 +186,8 @@ retryLookup: }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(r.FileName) != "" { - obj.Properties["file_name"] = r.FileName + if strings.TrimSpace(r.Path) != "" { + obj.Properties["path"] = r.Path } // 2. Fetch storage access methods. urlRows, err := db.db.QueryContext(ctx, "SELECT url, type FROM drs_object_access_method WHERE object_id = $1", lookupID) @@ -260,7 +263,7 @@ func (db *PostgresDB) CreateObject(ctx context.Context, obj *models.InternalObje _, err = tx.ExecContext(ctx, ` INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), postgresStoredFileName(obj), common.StringVal(obj.Version), common.StringVal(obj.Description), + obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), postgresStoredPath(obj), common.StringVal(obj.Version), common.StringVal(obj.Description), ) if err != nil { return fmt.Errorf("failed to insert drs_object: %w", err) @@ -269,7 +272,7 @@ func (db *PostgresDB) CreateObject(ctx context.Context, obj *models.InternalObje if err := insertControlledAccessTx(ctx, tx, obj.Id, objectAccessResources(obj)); err != nil { return err } - if err := postgresRebuildBrowseRowsForObjectTx(ctx, tx, obj.Id, postgresStoredFileName(obj), objectAccessResources(obj)); err != nil { + if err := postgresRebuildBrowseRowsForObjectTx(ctx, tx, obj.Id, postgresStoredPath(obj), objectAccessResources(obj)); err != nil { return fmt.Errorf("failed to update browse index: %w", err) } @@ -341,7 +344,7 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte createdTimes = append(createdTimes, obj.CreatedTime) updatedTimes = append(updatedTimes, common.TimeVal(obj.UpdatedTime)) names = append(names, common.StringVal(obj.Name)) - fileNames = append(fileNames, postgresStoredFileName(&obj)) + fileNames = append(fileNames, postgresStoredPath(&obj)) versions = append(versions, common.StringVal(obj.Version)) descriptions = append(descriptions, common.StringVal(obj.Description)) @@ -350,7 +353,7 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte controlledObjectIDs = append(controlledObjectIDs, obj.Id) controlledResources = append(controlledResources, resource) } - browseRows = append(browseRows, postgresBrowseRowsForObject(obj.Id, postgresStoredFileName(&obj), objectAccessResources(&obj))...) + browseRows = append(browseRows, postgresBrowseRowsForObject(obj.Id, postgresStoredPath(&obj), objectAccessResources(&obj))...) if obj.AccessMethods != nil { for _, am := range *obj.AccessMethods { if am.AccessUrl == nil || am.AccessUrl.Url == "" { @@ -1179,7 +1182,7 @@ func (db *PostgresDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []st return nil, err } - loadedName, loadedFileName := postgresLoadedNames(name, fileName) + loadedName, loadedPath := postgresLoadedNames(name, fileName) objectsByID[id] = &models.InternalObject{ DrsObject: drs.DrsObject{ Id: id, @@ -1193,8 +1196,8 @@ func (db *PostgresDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []st }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(loadedFileName) != "" { - objectsByID[id].Properties["file_name"] = loadedFileName + if strings.TrimSpace(loadedPath) != "" { + objectsByID[id].Properties["path"] = loadedPath } } diff --git a/internal/db/sqlite/metrics.go b/internal/db/sqlite/metrics.go index beeeeb69..cedcd127 100644 --- a/internal/db/sqlite/metrics.go +++ b/internal/db/sqlite/metrics.go @@ -659,7 +659,7 @@ func (db *SqliteDB) listStorageMetricRowsByScope(ctx context.Context, organizati for rows.Next() { var row models.DrsObjectRecord var lastDownload sql.NullTime - if err := rows.Scan(&row.ID, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload, &row.Name, &row.FileName); err != nil { + if err := rows.Scan(&row.ID, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload, &row.Name, &row.Path); err != nil { return nil, err } if lastDownload.Valid { diff --git a/internal/db/sqlite/objects.go b/internal/db/sqlite/objects.go index f92fa9c9..2024c187 100644 --- a/internal/db/sqlite/objects.go +++ b/internal/db/sqlite/objects.go @@ -16,8 +16,11 @@ import ( "github.com/calypr/syfon/internal/models" ) -func sqliteStoredFileName(obj *models.InternalObject) string { +func sqliteStoredPath(obj *models.InternalObject) string { if obj != nil && obj.Properties != nil { + if raw, ok := obj.Properties["path"].(string); ok && strings.TrimSpace(raw) != "" { + return strings.TrimSpace(raw) + } if raw, ok := obj.Properties["file_name"].(string); ok && strings.TrimSpace(raw) != "" { return strings.TrimSpace(raw) } @@ -30,18 +33,18 @@ func sqliteStoredFileName(obj *models.InternalObject) string { func sqliteLoadedNames(name, fileName sql.NullString) (string, string) { loadedName := strings.TrimSpace(name.String) - loadedFileName := strings.TrimSpace(fileName.String) - if loadedFileName == "" { - loadedFileName = loadedName + loadedPath := strings.TrimSpace(fileName.String) + if loadedPath == "" { + loadedPath = loadedName loadedName = "" } - if loadedName == "" && loadedFileName != "" { - loadedName = path.Base(strings.Trim(loadedFileName, "/")) + if loadedName == "" && loadedPath != "" { + loadedName = path.Base(strings.Trim(loadedPath, "/")) if loadedName == "." || loadedName == "/" || loadedName == "" { - loadedName = loadedFileName + loadedName = loadedPath } } - return loadedName, loadedFileName + return loadedName, loadedPath } func (db *SqliteDB) DeleteObject(ctx context.Context, id string) error { @@ -163,7 +166,7 @@ retryLookup: if err != nil { return nil, fmt.Errorf("failed to fetch record: %w", err) } - r.Name, r.FileName = sqliteLoadedNames(name, fileName) + r.Name, r.Path = sqliteLoadedNames(name, fileName) r.Version = version.String r.Description = description.String objectID := r.ID @@ -184,8 +187,8 @@ retryLookup: }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(r.FileName) != "" { - obj.Properties["file_name"] = r.FileName + if strings.TrimSpace(r.Path) != "" { + obj.Properties["path"] = r.Path } // 2. Fetch storage access methods. @@ -262,7 +265,7 @@ func (db *SqliteDB) CreateObject(ctx context.Context, obj *models.InternalObject _, err = tx.ExecContext(ctx, ` INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), sqliteStoredFileName(obj), common.StringVal(obj.Version), common.StringVal(obj.Description), + obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), sqliteStoredPath(obj), common.StringVal(obj.Version), common.StringVal(obj.Description), ) if err != nil { return fmt.Errorf("failed to insert drs_object: %w", err) @@ -271,7 +274,7 @@ func (db *SqliteDB) CreateObject(ctx context.Context, obj *models.InternalObject if err := insertControlledAccessTx(ctx, tx, obj.Id, objectAccessResources(obj)); err != nil { return err } - if err := sqliteRebuildBrowseRowsForObjectTx(ctx, tx, obj.Id, sqliteStoredFileName(obj), objectAccessResources(obj)); err != nil { + if err := sqliteRebuildBrowseRowsForObjectTx(ctx, tx, obj.Id, sqliteStoredPath(obj), objectAccessResources(obj)); err != nil { return fmt.Errorf("failed to update browse index: %w", err) } @@ -331,13 +334,13 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern for _, obj := range objects { ids = append(ids, obj.Id) - mainArgs = append(mainArgs, obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), sqliteStoredFileName(&obj), common.StringVal(obj.Version), common.StringVal(obj.Description)) + mainArgs = append(mainArgs, obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), sqliteStoredPath(&obj), common.StringVal(obj.Version), common.StringVal(obj.Description)) seenAccess := make(map[string]struct{}) for _, resource := range objectAccessResources(&obj) { controlledArgs = append(controlledArgs, obj.Id, resource) } - browseRows = append(browseRows, sqliteBrowseRowsForObject(obj.Id, sqliteStoredFileName(&obj), objectAccessResources(&obj))...) + browseRows = append(browseRows, sqliteBrowseRowsForObject(obj.Id, sqliteStoredPath(&obj), objectAccessResources(&obj))...) if obj.AccessMethods != nil { for _, am := range *obj.AccessMethods { if am.AccessUrl == nil || am.AccessUrl.Url == "" { @@ -1191,7 +1194,7 @@ func (db *SqliteDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []stri return nil, err } - loadedName, loadedFileName := sqliteLoadedNames(name, fileName) + loadedName, loadedPath := sqliteLoadedNames(name, fileName) objectsByID[id] = &models.InternalObject{ DrsObject: drs.DrsObject{ Id: id, @@ -1205,8 +1208,8 @@ func (db *SqliteDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []stri }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(loadedFileName) != "" { - objectsByID[id].Properties["file_name"] = loadedFileName + if strings.TrimSpace(loadedPath) != "" { + objectsByID[id].Properties["path"] = loadedPath } } diff --git a/internal/models/object.go b/internal/models/object.go index 19ecd034..c03d7a58 100644 --- a/internal/models/object.go +++ b/internal/models/object.go @@ -2,6 +2,7 @@ package models import ( "encoding/json" + "strings" "time" "github.com/calypr/syfon/apigen/server/drs" @@ -36,6 +37,7 @@ func (o *InternalObject) UnmarshalJSON(data []byte) error { drs.DrsObject Did string `json:"did,omitempty"` Hashes map[string]string `json:"hashes,omitempty"` + Path *string `json:"path,omitempty"` FileName *string `json:"file_name,omitempty"` } @@ -69,6 +71,11 @@ func (o *InternalObject) UnmarshalJSON(data []byte) error { o.MimeType = wire.MimeType o.SelfUri = wire.SelfUri o.Version = wire.Version + if wire.Path != nil && strings.TrimSpace(*wire.Path) != "" { + o.Properties["path"] = strings.TrimSpace(*wire.Path) + } else if wire.FileName != nil && strings.TrimSpace(*wire.FileName) != "" { + o.Properties["path"] = strings.TrimSpace(*wire.FileName) + } if wire.ControlledAccess != nil { o.Authorizations = common.ControlledAccessToAuthzMap(*wire.ControlledAccess) @@ -117,12 +124,6 @@ func (o InternalObject) MarshalJSON() ([]byte, error) { } // Ensure Gen3 compatibility fields are also present. out["did"] = o.Id - if fileName, ok := out["file_name"].(string); ok && fileName != "" { - out["file_name"] = fileName - } else if o.Name != nil { - out["file_name"] = *o.Name - } - if len(o.Checksums) > 0 { hashes := make(map[string]string, len(o.Checksums)) for _, c := range o.Checksums { @@ -141,7 +142,7 @@ func (o InternalObject) MarshalJSON() ([]byte, error) { func isRetiredInternalAuthField(key string) bool { switch key { - case "auth", "authz", "authorizations", "urls": + case "auth", "authz", "authorizations", "urls", "file_name", "path": return true default: return false diff --git a/internal/models/object_test.go b/internal/models/object_test.go index 5b355260..d3ecac5c 100644 --- a/internal/models/object_test.go +++ b/internal/models/object_test.go @@ -38,12 +38,12 @@ func TestInternalObjectUnmarshalLegacyAndMarshalCompatibility(t *testing.T) { if out["did"] != "did-1" { t.Fatalf("expected did in output, got %v", out["did"]) } - if out["file_name"] != "legacy-name.txt" { - t.Fatalf("expected file_name in output, got %v", out["file_name"]) - } if _, ok := out["name"]; ok { t.Fatalf("did not expect name in output when only file_name was provided") } + if _, ok := out["file_name"]; ok { + t.Fatalf("did not expect file_name in output") + } if _, ok := out["unknown_field"]; !ok { t.Fatalf("expected unknown field preservation in output") } diff --git a/internal/models/storage.go b/internal/models/storage.go index 6b7c12d0..5ffb1b44 100644 --- a/internal/models/storage.go +++ b/internal/models/storage.go @@ -265,7 +265,7 @@ type DrsObjectRecord struct { DownloadCount int64 LastDownloadTime *time.Time Name string - FileName string + Path string Version string Description string } diff --git a/internal/storagemetrics/storage_metrics.go b/internal/storagemetrics/storage_metrics.go index 20c6814b..6c67e2f5 100644 --- a/internal/storagemetrics/storage_metrics.go +++ b/internal/storagemetrics/storage_metrics.go @@ -140,7 +140,7 @@ func AggregateStoragePathChildren(path string, rows []models.DrsObjectRecord, li } func storageMetricPathInfo(row models.DrsObjectRecord) (common.BrowsePathInfo, bool) { - raw := strings.TrimSpace(row.FileName) + raw := strings.TrimSpace(row.Path) if raw == "" { raw = strings.TrimSpace(row.Name) } diff --git a/internal/storagemetrics/storage_metrics_test.go b/internal/storagemetrics/storage_metrics_test.go index 2e116e97..c986143f 100644 --- a/internal/storagemetrics/storage_metrics_test.go +++ b/internal/storagemetrics/storage_metrics_test.go @@ -11,7 +11,7 @@ func TestAggregateStoragePathSummary(t *testing.T) { now := time.Now().UTC() rows := []models.DrsObjectRecord{ {Name: "data/a.txt", Size: 10, UpdatedTime: now.Add(-time.Hour), DownloadCount: 2, LastDownloadTime: timePtr(now.Add(-3 * time.Hour))}, - {FileName: "data/a.txt", Size: 20, UpdatedTime: now, DownloadCount: 4, LastDownloadTime: timePtr(now.Add(-30 * time.Minute))}, + {Path: "data/a.txt", Size: 20, UpdatedTime: now, DownloadCount: 4, LastDownloadTime: timePtr(now.Add(-30 * time.Minute))}, {Name: "data/nested/b.txt", Size: 30, UpdatedTime: now.Add(-2 * time.Hour), DownloadCount: 1, LastDownloadTime: timePtr(now.Add(-90 * time.Minute))}, {Name: "other/c.txt", Size: 40, UpdatedTime: now.Add(-3 * time.Hour)}, {Name: "bad/../path", Size: 50, UpdatedTime: now}, From 610582fc1bed8e8efc05e1e3d477bbb2eb2f31c6 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 29 Jun 2026 10:40:15 -0700 Subject: [PATCH 06/24] cleanup old code, fix some deadcode go tool issues --- apigen/client/internalapi/internal.gen.go | 387 ------- apigen/openapi/internal.openapi.yaml | 249 ----- apigen/openapi/metrics.openapi.yaml | 98 -- apigen/server/internalapi/internal.gen.go | 297 ------ client/client.go | 3 - client/common/constants.go | 1 - client/services/drs_service_test.go | 4 +- client/services/index_service.go | 3 - client/services/interface.go | 1 - client/services/repair_service.go | 103 -- client/services/repair_service_test.go | 64 -- client/services/types.go | 39 - cmd/list/main.go | 6 +- cmd/list/main_test.go | 9 +- cmd/migrate/main.go | 302 ------ cmd/migrate/main_test.go | 79 -- cmd/repair/main.go | 380 ------- cmd/repair/main_test.go | 73 -- cmd/root.go | 2 - cmd/server/endpoints_mock_test.go | 4 +- cmd/server/server_integration_test.go | 2 +- docs/frontend-handoff-storage-metrics.md | 272 ----- docs/implementation-plan-storage-summary.md | 414 -------- docs/operator-guide-migration-indexd.md | 83 -- docs/pr-summary-feature-optimize.md | 1 - ...em-solution-org-project-storage-metrics.md | 103 -- ...roblem-solution-project-hierarchy-index.md | 2 +- internal/api/docs/swagger.go | 88 -- internal/api/internaldrs/bucket_helpers.go | 10 - .../api/internaldrs/bucket_policy_test.go | 4 - internal/api/internaldrs/index.go | 43 - internal/api/internaldrs/index_test.go | 49 - internal/api/internaldrs/repair_cleanup.go | 81 -- internal/api/lfs/lfs.go | 7 - internal/api/lfs/reset_test.go | 8 + internal/api/metrics/transfer_ingest.go | 110 -- internal/api/metrics/transfer_ingest_test.go | 111 -- internal/api/routeutil/routeutil.go | 13 - internal/common/browse_path.go | 96 -- internal/common/browse_path_test.go | 41 - internal/common/routes.go | 33 +- internal/converters/converters.go | 19 +- internal/core/converters.go | 23 +- internal/core/object_reads.go | 24 - internal/core/storage_cleanup.go | 49 - internal/db/interface.go | 16 - internal/db/postgres/browse_index.go | 163 --- internal/db/postgres/metrics.go | 178 ---- internal/db/postgres/metrics_sqlmock_test.go | 126 --- internal/db/postgres/object_schema.sql | 22 +- internal/db/postgres/objects.go | 92 +- internal/db/postgres/objects_sqlmock_test.go | 86 +- internal/db/postgres/postgres_schema_test.go | 10 +- internal/db/sqlite/browse_index.go | 142 --- internal/db/sqlite/metrics.go | 178 ---- internal/db/sqlite/objects.go | 87 +- internal/db/sqlite/sqlite.go | 78 +- internal/db/sqlite/sqlite_test.go | 145 +-- internal/models/object.go | 21 +- internal/models/object_test.go | 39 +- internal/models/storage.go | 50 - internal/repair/format.go | 89 -- internal/repair/project_diff_service.go | 166 --- internal/repair/project_diff_service_test.go | 90 -- internal/repair/project_diff_types.go | 46 - internal/repair/storage_cleanup_service.go | 971 ------------------ .../repair/storage_cleanup_service_test.go | 663 ------------ internal/repair/storage_cleanup_types.go | 121 --- internal/storagemetrics/storage_metrics.go | 248 ----- .../storagemetrics/storage_metrics_test.go | 96 -- internal/testutils/mocks.go | 220 ---- internal/version/version.go | 11 - migrate/client.go | 379 ------- migrate/dump.go | 155 --- migrate/dump_test.go | 47 - migrate/preflight.go | 205 ---- migrate/run.go | 153 --- migrate/run_test.go | 287 ------ migrate/transform.go | 258 ----- migrate/transform_test.go | 77 -- migrate/types.go | 96 -- mkdocs.yml | 1 - 82 files changed, 201 insertions(+), 9401 deletions(-) delete mode 100644 client/services/repair_service.go delete mode 100644 client/services/repair_service_test.go delete mode 100644 cmd/migrate/main.go delete mode 100644 cmd/migrate/main_test.go delete mode 100644 cmd/repair/main.go delete mode 100644 cmd/repair/main_test.go delete mode 100644 docs/frontend-handoff-storage-metrics.md delete mode 100644 docs/implementation-plan-storage-summary.md delete mode 100644 docs/operator-guide-migration-indexd.md delete mode 100644 docs/problem-solution-org-project-storage-metrics.md create mode 100644 internal/api/lfs/reset_test.go delete mode 100644 internal/api/metrics/transfer_ingest_test.go delete mode 100644 internal/common/browse_path.go delete mode 100644 internal/common/browse_path_test.go delete mode 100644 internal/core/storage_cleanup.go delete mode 100644 internal/db/postgres/browse_index.go delete mode 100644 internal/db/sqlite/browse_index.go delete mode 100644 internal/repair/format.go delete mode 100644 internal/repair/project_diff_service.go delete mode 100644 internal/repair/project_diff_service_test.go delete mode 100644 internal/repair/project_diff_types.go delete mode 100644 internal/repair/storage_cleanup_service.go delete mode 100644 internal/repair/storage_cleanup_service_test.go delete mode 100644 internal/repair/storage_cleanup_types.go delete mode 100644 internal/storagemetrics/storage_metrics.go delete mode 100644 internal/storagemetrics/storage_metrics_test.go delete mode 100644 migrate/client.go delete mode 100644 migrate/dump.go delete mode 100644 migrate/dump_test.go delete mode 100644 migrate/preflight.go delete mode 100644 migrate/run.go delete mode 100644 migrate/run_test.go delete mode 100644 migrate/transform.go delete mode 100644 migrate/transform_test.go delete mode 100644 migrate/types.go diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index 0dca3c92..ea0f532d 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -12,7 +12,6 @@ import ( "net/http" "net/url" "strings" - "time" externalRef0 "github.com/calypr/syfon/apigen/client/drs" "github.com/oapi-codegen/runtime" @@ -195,108 +194,6 @@ type ListRecordsResponse struct { Records *[]InternalRecord `json:"records,omitempty"` } -// StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. -type StorageCleanupAccessProbe struct { - Bucket *string `json:"bucket,omitempty"` - ErrorKind *string `json:"error_kind,omitempty"` - StorageMessage *string `json:"storage_message,omitempty"` - StorageStatus *string `json:"storage_status,omitempty"` - Url *string `json:"url,omitempty"` -} - -// StorageCleanupApplyRequest defines model for StorageCleanupApplyRequest. -type StorageCleanupApplyRequest struct { - CheckStorage *bool `json:"check_storage,omitempty"` - DeleteRepoOrphans *bool `json:"delete_repo_orphans,omitempty"` - DeleteStaleDuplicates *bool `json:"delete_stale_duplicates,omitempty"` - DryRun *bool `json:"dry_run,omitempty"` - ExpectedPaths *[]string `json:"expected_paths,omitempty"` - Organization string `json:"organization"` - PathPrefix *string `json:"path_prefix,omitempty"` - Project string `json:"project"` - SelectedFindingKinds *[]string `json:"selected_finding_kinds,omitempty"` - SelectedObjectIds *[]string `json:"selected_object_ids,omitempty"` - SelectedPaths *[]string `json:"selected_paths,omitempty"` -} - -// StorageCleanupApplyResult defines model for StorageCleanupApplyResult. -type StorageCleanupApplyResult struct { - DeletedRecordIds *[]string `json:"deleted_record_ids,omitempty"` - DryRun *bool `json:"dry_run,omitempty"` - RepoDeletePaths *[]string `json:"repo_delete_paths,omitempty"` - Report *StorageCleanupReport `json:"report,omitempty"` - Skipped *[]StorageCleanupSkipped `json:"skipped,omitempty"` - StoragePurgeResults *[]StorageCleanupStoragePurgeResult `json:"storage_purge_results,omitempty"` -} - -// StorageCleanupAuditRequest defines model for StorageCleanupAuditRequest. -type StorageCleanupAuditRequest struct { - CheckStorage *bool `json:"check_storage,omitempty"` - ExpectedPaths *[]string `json:"expected_paths,omitempty"` - Organization string `json:"organization"` - PathPrefix *string `json:"path_prefix,omitempty"` - Project string `json:"project"` -} - -// StorageCleanupFinding defines model for StorageCleanupFinding. -type StorageCleanupFinding struct { - ChecksumCount *int `json:"checksum_count,omitempty"` - CleanupScope *string `json:"cleanup_scope,omitempty"` - Kind *string `json:"kind,omitempty"` - LegacyUrlTemplateDetected *bool `json:"legacy_url_template_detected,omitempty"` - Message *string `json:"message,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - RecommendedAction *string `json:"recommended_action,omitempty"` - Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` - RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` - Severity *string `json:"severity,omitempty"` - SizeCount *int `json:"size_count,omitempty"` - StructuralReason *string `json:"structural_reason,omitempty"` -} - -// StorageCleanupRecordAudit defines model for StorageCleanupRecordAudit. -type StorageCleanupRecordAudit struct { - AccessProbes *[]StorageCleanupAccessProbe `json:"access_probes,omitempty"` - CleanupScope *string `json:"cleanup_scope,omitempty"` - CurrentAccessUrls *[]string `json:"current_access_urls,omitempty"` - DownloadCount *int64 `json:"download_count,omitempty"` - LastDownloadTime *time.Time `json:"last_download_time,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - ObjectId *string `json:"object_id,omitempty"` - Size *int64 `json:"size,omitempty"` - StorageMessage *string `json:"storage_message,omitempty"` - StorageStatus *string `json:"storage_status,omitempty"` - UpdatedTime *time.Time `json:"updated_time,omitempty"` -} - -// StorageCleanupReport defines model for StorageCleanupReport. -type StorageCleanupReport struct { - ClassifiedPaths *int `json:"classified_paths,omitempty"` - Findings *[]StorageCleanupFinding `json:"findings,omitempty"` - Organization *string `json:"organization,omitempty"` - PathPrefix *string `json:"path_prefix,omitempty"` - Project *string `json:"project,omitempty"` - Scanned *int `json:"scanned,omitempty"` - ScannedPaths *int `json:"scanned_paths,omitempty"` - Summary *map[string]int `json:"summary,omitempty"` - UnclassifiedPaths *int `json:"unclassified_paths,omitempty"` -} - -// StorageCleanupSkipped defines model for StorageCleanupSkipped. -type StorageCleanupSkipped struct { - Kind *string `json:"kind,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - ObjectId *string `json:"object_id,omitempty"` - Reason *string `json:"reason,omitempty"` -} - -// StorageCleanupStoragePurgeResult defines model for StorageCleanupStoragePurgeResult. -type StorageCleanupStoragePurgeResult struct { - Message *string `json:"message,omitempty"` - ObjectId *string `json:"object_id,omitempty"` - Purged *bool `json:"purged,omitempty"` -} - // InternalDownloadParams defines parameters for InternalDownload. type InternalDownloadParams struct { Redirect *bool `form:"redirect,omitempty" json:"redirect,omitempty"` @@ -348,12 +245,6 @@ type InternalMultipartInitJSONRequestBody = InternalMultipartInitRequest // InternalMultipartUploadJSONRequestBody defines body for InternalMultipartUpload for application/json ContentType. type InternalMultipartUploadJSONRequestBody = InternalMultipartUploadRequest -// InternalStorageCleanupApplyJSONRequestBody defines body for InternalStorageCleanupApply for application/json ContentType. -type InternalStorageCleanupApplyJSONRequestBody = StorageCleanupApplyRequest - -// InternalStorageCleanupAuditJSONRequestBody defines body for InternalStorageCleanupAudit for application/json ContentType. -type InternalStorageCleanupAuditJSONRequestBody = StorageCleanupAuditRequest - // InternalUploadBlankJSONRequestBody defines body for InternalUploadBlank for application/json ContentType. type InternalUploadBlankJSONRequestBody = InternalUploadBlankRequest @@ -540,16 +431,6 @@ type ClientInterface interface { InternalMultipartUpload(ctx context.Context, body InternalMultipartUploadJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // InternalStorageCleanupApplyWithBody request with any body - InternalStorageCleanupApplyWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) - - InternalStorageCleanupApply(ctx context.Context, body InternalStorageCleanupApplyJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - - // InternalStorageCleanupAuditWithBody request with any body - InternalStorageCleanupAuditWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) - - InternalStorageCleanupAudit(ctx context.Context, body InternalStorageCleanupAuditJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // InternalUploadBlankWithBody request with any body InternalUploadBlankWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -712,54 +593,6 @@ func (c *Client) InternalMultipartUpload(ctx context.Context, body InternalMulti return c.Client.Do(req) } -func (c *Client) InternalStorageCleanupApplyWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewInternalStorageCleanupApplyRequestWithBody(c.Server, contentType, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) InternalStorageCleanupApply(ctx context.Context, body InternalStorageCleanupApplyJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewInternalStorageCleanupApplyRequest(c.Server, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) InternalStorageCleanupAuditWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewInternalStorageCleanupAuditRequestWithBody(c.Server, contentType, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) InternalStorageCleanupAudit(ctx context.Context, body InternalStorageCleanupAuditJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewInternalStorageCleanupAuditRequest(c.Server, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - func (c *Client) InternalUploadBlankWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewInternalUploadBlankRequestWithBody(c.Server, contentType, body) if err != nil { @@ -1316,86 +1149,6 @@ func NewInternalMultipartUploadRequestWithBody(server string, contentType string return req, nil } -// NewInternalStorageCleanupApplyRequest calls the generic InternalStorageCleanupApply builder with application/json body -func NewInternalStorageCleanupApplyRequest(server string, body InternalStorageCleanupApplyJSONRequestBody) (*http.Request, error) { - var bodyReader io.Reader - buf, err := json.Marshal(body) - if err != nil { - return nil, err - } - bodyReader = bytes.NewReader(buf) - return NewInternalStorageCleanupApplyRequestWithBody(server, "application/json", bodyReader) -} - -// NewInternalStorageCleanupApplyRequestWithBody generates requests for InternalStorageCleanupApply with any type of body -func NewInternalStorageCleanupApplyRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/data/repair/storage-cleanup/apply") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("POST", queryURL.String(), body) - if err != nil { - return nil, err - } - - req.Header.Add("Content-Type", contentType) - - return req, nil -} - -// NewInternalStorageCleanupAuditRequest calls the generic InternalStorageCleanupAudit builder with application/json body -func NewInternalStorageCleanupAuditRequest(server string, body InternalStorageCleanupAuditJSONRequestBody) (*http.Request, error) { - var bodyReader io.Reader - buf, err := json.Marshal(body) - if err != nil { - return nil, err - } - bodyReader = bytes.NewReader(buf) - return NewInternalStorageCleanupAuditRequestWithBody(server, "application/json", bodyReader) -} - -// NewInternalStorageCleanupAuditRequestWithBody generates requests for InternalStorageCleanupAudit with any type of body -func NewInternalStorageCleanupAuditRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/data/repair/storage-cleanup/audit") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("POST", queryURL.String(), body) - if err != nil { - return nil, err - } - - req.Header.Add("Content-Type", contentType) - - return req, nil -} - // NewInternalUploadBlankRequest calls the generic InternalUploadBlank builder with application/json body func NewInternalUploadBlankRequest(server string, body InternalUploadBlankJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -2336,16 +2089,6 @@ type ClientWithResponsesInterface interface { InternalMultipartUploadWithResponse(ctx context.Context, body InternalMultipartUploadJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalMultipartUploadResponse, error) - // InternalStorageCleanupApplyWithBodyWithResponse request with any body - InternalStorageCleanupApplyWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalStorageCleanupApplyResponse, error) - - InternalStorageCleanupApplyWithResponse(ctx context.Context, body InternalStorageCleanupApplyJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalStorageCleanupApplyResponse, error) - - // InternalStorageCleanupAuditWithBodyWithResponse request with any body - InternalStorageCleanupAuditWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalStorageCleanupAuditResponse, error) - - InternalStorageCleanupAuditWithResponse(ctx context.Context, body InternalStorageCleanupAuditJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalStorageCleanupAuditResponse, error) - // InternalUploadBlankWithBodyWithResponse request with any body InternalUploadBlankWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalUploadBlankResponse, error) @@ -2521,50 +2264,6 @@ func (r InternalMultipartUploadResponse) StatusCode() int { return 0 } -type InternalStorageCleanupApplyResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *StorageCleanupApplyResult -} - -// Status returns HTTPResponse.Status -func (r InternalStorageCleanupApplyResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r InternalStorageCleanupApplyResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -type InternalStorageCleanupAuditResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *StorageCleanupReport -} - -// Status returns HTTPResponse.Status -func (r InternalStorageCleanupAuditResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r InternalStorageCleanupAuditResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - type InternalUploadBlankResponse struct { Body []byte HTTPResponse *http.Response @@ -2964,40 +2663,6 @@ func (c *ClientWithResponses) InternalMultipartUploadWithResponse(ctx context.Co return ParseInternalMultipartUploadResponse(rsp) } -// InternalStorageCleanupApplyWithBodyWithResponse request with arbitrary body returning *InternalStorageCleanupApplyResponse -func (c *ClientWithResponses) InternalStorageCleanupApplyWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalStorageCleanupApplyResponse, error) { - rsp, err := c.InternalStorageCleanupApplyWithBody(ctx, contentType, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseInternalStorageCleanupApplyResponse(rsp) -} - -func (c *ClientWithResponses) InternalStorageCleanupApplyWithResponse(ctx context.Context, body InternalStorageCleanupApplyJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalStorageCleanupApplyResponse, error) { - rsp, err := c.InternalStorageCleanupApply(ctx, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseInternalStorageCleanupApplyResponse(rsp) -} - -// InternalStorageCleanupAuditWithBodyWithResponse request with arbitrary body returning *InternalStorageCleanupAuditResponse -func (c *ClientWithResponses) InternalStorageCleanupAuditWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalStorageCleanupAuditResponse, error) { - rsp, err := c.InternalStorageCleanupAuditWithBody(ctx, contentType, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseInternalStorageCleanupAuditResponse(rsp) -} - -func (c *ClientWithResponses) InternalStorageCleanupAuditWithResponse(ctx context.Context, body InternalStorageCleanupAuditJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalStorageCleanupAuditResponse, error) { - rsp, err := c.InternalStorageCleanupAudit(ctx, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseInternalStorageCleanupAuditResponse(rsp) -} - // InternalUploadBlankWithBodyWithResponse request with arbitrary body returning *InternalUploadBlankResponse func (c *ClientWithResponses) InternalUploadBlankWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalUploadBlankResponse, error) { rsp, err := c.InternalUploadBlankWithBody(ctx, contentType, body, reqEditors...) @@ -3333,58 +2998,6 @@ func ParseInternalMultipartUploadResponse(rsp *http.Response) (*InternalMultipar return response, nil } -// ParseInternalStorageCleanupApplyResponse parses an HTTP response from a InternalStorageCleanupApplyWithResponse call -func ParseInternalStorageCleanupApplyResponse(rsp *http.Response) (*InternalStorageCleanupApplyResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &InternalStorageCleanupApplyResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest StorageCleanupApplyResult - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - } - - return response, nil -} - -// ParseInternalStorageCleanupAuditResponse parses an HTTP response from a InternalStorageCleanupAuditWithResponse call -func ParseInternalStorageCleanupAuditResponse(rsp *http.Response) (*InternalStorageCleanupAuditResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &InternalStorageCleanupAuditResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest StorageCleanupReport - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - } - - return response, nil -} - // ParseInternalUploadBlankResponse parses an HTTP response from a InternalUploadBlankWithResponse call func ParseInternalUploadBlankResponse(rsp *http.Response) (*InternalUploadBlankResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/openapi/internal.openapi.yaml b/apigen/openapi/internal.openapi.yaml index bc6b2e87..f0df1ade 100644 --- a/apigen/openapi/internal.openapi.yaml +++ b/apigen/openapi/internal.openapi.yaml @@ -633,58 +633,6 @@ paths: description: Forbidden '500': description: Internal server error - /data/repair/storage-cleanup/audit: - post: - tags: - - internal-data - operationId: internalStorageCleanupAudit - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/StorageCleanupAuditRequest' - responses: - '200': - description: Storage cleanup audit report - content: - application/json: - schema: - $ref: '#/components/schemas/StorageCleanupReport' - '400': - description: Validation error - '401': - description: Authentication required - '403': - description: Forbidden - '500': - description: Internal server error - /data/repair/storage-cleanup/apply: - post: - tags: - - internal-data - operationId: internalStorageCleanupApply - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/StorageCleanupApplyRequest' - responses: - '200': - description: Storage cleanup apply result - content: - application/json: - schema: - $ref: '#/components/schemas/StorageCleanupApplyResult' - '400': - description: Validation error - '401': - description: Authentication required - '403': - description: Forbidden - '500': - description: Internal server error components: schemas: ControlledAccessRemoveRequest: @@ -796,203 +744,6 @@ components: type: array items: type: string - StorageCleanupAuditRequest: - type: object - required: - - organization - - project - properties: - organization: - type: string - project: - type: string - path_prefix: - type: string - expected_paths: - type: array - items: - type: string - check_storage: - type: boolean - StorageCleanupApplyRequest: - type: object - required: - - organization - - project - properties: - organization: - type: string - project: - type: string - path_prefix: - type: string - expected_paths: - type: array - items: - type: string - delete_stale_duplicates: - type: boolean - delete_repo_orphans: - type: boolean - dry_run: - type: boolean - selected_paths: - type: array - items: - type: string - selected_object_ids: - type: array - items: - type: string - selected_finding_kinds: - type: array - items: - type: string - check_storage: - type: boolean - StorageCleanupReport: - type: object - properties: - organization: - type: string - project: - type: string - path_prefix: - type: string - scanned: - type: integer - scanned_paths: - type: integer - classified_paths: - type: integer - unclassified_paths: - type: integer - summary: - type: object - additionalProperties: - type: integer - findings: - type: array - items: - $ref: '#/components/schemas/StorageCleanupFinding' - StorageCleanupFinding: - type: object - properties: - kind: - type: string - severity: - type: string - normalized_path: - type: string - message: - type: string - recommended_action: - type: string - repo_delete_candidate: - type: boolean - cleanup_scope: - type: string - structural_reason: - type: string - checksum_count: - type: integer - size_count: - type: integer - legacy_url_template_detected: - type: boolean - records: - type: array - items: - $ref: '#/components/schemas/StorageCleanupRecordAudit' - StorageCleanupAccessProbe: - type: object - properties: - url: - type: string - bucket: - type: string - storage_status: - type: string - storage_message: - type: string - error_kind: - type: string - StorageCleanupRecordAudit: - type: object - properties: - object_id: - type: string - normalized_path: - type: string - size: - type: integer - format: int64 - updated_time: - type: string - format: date-time - download_count: - type: integer - format: int64 - last_download_time: - type: string - format: date-time - current_access_urls: - type: array - items: - type: string - access_probes: - type: array - items: - $ref: '#/components/schemas/StorageCleanupAccessProbe' - cleanup_scope: - type: string - storage_status: - type: string - storage_message: - type: string - StorageCleanupApplyResult: - type: object - properties: - report: - $ref: '#/components/schemas/StorageCleanupReport' - dry_run: - type: boolean - deleted_record_ids: - type: array - items: - type: string - storage_purge_results: - type: array - items: - $ref: '#/components/schemas/StorageCleanupStoragePurgeResult' - repo_delete_paths: - type: array - items: - type: string - skipped: - type: array - items: - $ref: '#/components/schemas/StorageCleanupSkipped' - StorageCleanupStoragePurgeResult: - type: object - properties: - object_id: - type: string - purged: - type: boolean - message: - type: string - StorageCleanupSkipped: - type: object - properties: - normalized_path: - type: string - object_id: - type: string - kind: - type: string - reason: - type: string DeleteByQueryResponse: type: object properties: diff --git a/apigen/openapi/metrics.openapi.yaml b/apigen/openapi/metrics.openapi.yaml index 088b32bd..5d1c502e 100644 --- a/apigen/openapi/metrics.openapi.yaml +++ b/apigen/openapi/metrics.openapi.yaml @@ -226,26 +226,6 @@ components: schema: type: string description: Project scope filter. Requires organization when set. - StorageOrganization: - in: query - name: organization - required: true - schema: - type: string - description: Organization scope for the storage subtree query. - StorageProject: - in: query - name: project - required: true - schema: - type: string - description: Project scope for the storage subtree query. - Path: - in: query - name: path - schema: - type: string - description: Optional storage subtree path. Omit for the project root. Direction: in: query name: direction @@ -352,84 +332,6 @@ components: type: integer offset: type: integer - StoragePathSummary: - type: object - properties: - organization: - type: string - project: - type: string - path: - type: string - file_count: - type: integer - format: int64 - record_count: - type: integer - format: int64 - total_bytes: - type: integer - format: int64 - download_count: - type: integer - format: int64 - direct_child_count: - type: integer - format: int64 - duplicate_path_count: - type: integer - format: int64 - latest_update_time: - type: string - format: date-time - nullable: true - last_download_time: - type: string - format: date-time - nullable: true - StoragePathChild: - type: object - properties: - name: - type: string - path: - type: string - type: - type: string - enum: [file, directory] - file_count: - type: integer - format: int64 - record_count: - type: integer - format: int64 - total_bytes: - type: integer - format: int64 - download_count: - type: integer - format: int64 - latest_update_time: - type: string - format: date-time - nullable: true - last_download_time: - type: string - format: date-time - nullable: true - StoragePathChildrenResponse: - type: object - properties: - organization: - type: string - project: - type: string - path: - type: string - items: - type: array - items: - $ref: '#/components/schemas/StoragePathChild' ProviderTransferDirection: type: string enum: diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 7e4bf918..5e5d2956 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "io" - "time" externalRef0 "github.com/calypr/syfon/apigen/server/drs" "github.com/gofiber/fiber/v3" @@ -193,108 +192,6 @@ type ListRecordsResponse struct { Records *[]InternalRecord `json:"records,omitempty"` } -// StorageCleanupAccessProbe defines model for StorageCleanupAccessProbe. -type StorageCleanupAccessProbe struct { - Bucket *string `json:"bucket,omitempty"` - ErrorKind *string `json:"error_kind,omitempty"` - StorageMessage *string `json:"storage_message,omitempty"` - StorageStatus *string `json:"storage_status,omitempty"` - Url *string `json:"url,omitempty"` -} - -// StorageCleanupApplyRequest defines model for StorageCleanupApplyRequest. -type StorageCleanupApplyRequest struct { - CheckStorage *bool `json:"check_storage,omitempty"` - DeleteRepoOrphans *bool `json:"delete_repo_orphans,omitempty"` - DeleteStaleDuplicates *bool `json:"delete_stale_duplicates,omitempty"` - DryRun *bool `json:"dry_run,omitempty"` - ExpectedPaths *[]string `json:"expected_paths,omitempty"` - Organization string `json:"organization"` - PathPrefix *string `json:"path_prefix,omitempty"` - Project string `json:"project"` - SelectedFindingKinds *[]string `json:"selected_finding_kinds,omitempty"` - SelectedObjectIds *[]string `json:"selected_object_ids,omitempty"` - SelectedPaths *[]string `json:"selected_paths,omitempty"` -} - -// StorageCleanupApplyResult defines model for StorageCleanupApplyResult. -type StorageCleanupApplyResult struct { - DeletedRecordIds *[]string `json:"deleted_record_ids,omitempty"` - DryRun *bool `json:"dry_run,omitempty"` - RepoDeletePaths *[]string `json:"repo_delete_paths,omitempty"` - Report *StorageCleanupReport `json:"report,omitempty"` - Skipped *[]StorageCleanupSkipped `json:"skipped,omitempty"` - StoragePurgeResults *[]StorageCleanupStoragePurgeResult `json:"storage_purge_results,omitempty"` -} - -// StorageCleanupAuditRequest defines model for StorageCleanupAuditRequest. -type StorageCleanupAuditRequest struct { - CheckStorage *bool `json:"check_storage,omitempty"` - ExpectedPaths *[]string `json:"expected_paths,omitempty"` - Organization string `json:"organization"` - PathPrefix *string `json:"path_prefix,omitempty"` - Project string `json:"project"` -} - -// StorageCleanupFinding defines model for StorageCleanupFinding. -type StorageCleanupFinding struct { - ChecksumCount *int `json:"checksum_count,omitempty"` - CleanupScope *string `json:"cleanup_scope,omitempty"` - Kind *string `json:"kind,omitempty"` - LegacyUrlTemplateDetected *bool `json:"legacy_url_template_detected,omitempty"` - Message *string `json:"message,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - RecommendedAction *string `json:"recommended_action,omitempty"` - Records *[]StorageCleanupRecordAudit `json:"records,omitempty"` - RepoDeleteCandidate *bool `json:"repo_delete_candidate,omitempty"` - Severity *string `json:"severity,omitempty"` - SizeCount *int `json:"size_count,omitempty"` - StructuralReason *string `json:"structural_reason,omitempty"` -} - -// StorageCleanupRecordAudit defines model for StorageCleanupRecordAudit. -type StorageCleanupRecordAudit struct { - AccessProbes *[]StorageCleanupAccessProbe `json:"access_probes,omitempty"` - CleanupScope *string `json:"cleanup_scope,omitempty"` - CurrentAccessUrls *[]string `json:"current_access_urls,omitempty"` - DownloadCount *int64 `json:"download_count,omitempty"` - LastDownloadTime *time.Time `json:"last_download_time,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - ObjectId *string `json:"object_id,omitempty"` - Size *int64 `json:"size,omitempty"` - StorageMessage *string `json:"storage_message,omitempty"` - StorageStatus *string `json:"storage_status,omitempty"` - UpdatedTime *time.Time `json:"updated_time,omitempty"` -} - -// StorageCleanupReport defines model for StorageCleanupReport. -type StorageCleanupReport struct { - ClassifiedPaths *int `json:"classified_paths,omitempty"` - Findings *[]StorageCleanupFinding `json:"findings,omitempty"` - Organization *string `json:"organization,omitempty"` - PathPrefix *string `json:"path_prefix,omitempty"` - Project *string `json:"project,omitempty"` - Scanned *int `json:"scanned,omitempty"` - ScannedPaths *int `json:"scanned_paths,omitempty"` - Summary *map[string]int `json:"summary,omitempty"` - UnclassifiedPaths *int `json:"unclassified_paths,omitempty"` -} - -// StorageCleanupSkipped defines model for StorageCleanupSkipped. -type StorageCleanupSkipped struct { - Kind *string `json:"kind,omitempty"` - NormalizedPath *string `json:"normalized_path,omitempty"` - ObjectId *string `json:"object_id,omitempty"` - Reason *string `json:"reason,omitempty"` -} - -// StorageCleanupStoragePurgeResult defines model for StorageCleanupStoragePurgeResult. -type StorageCleanupStoragePurgeResult struct { - Message *string `json:"message,omitempty"` - ObjectId *string `json:"object_id,omitempty"` - Purged *bool `json:"purged,omitempty"` -} - // InternalDownloadParams defines parameters for InternalDownload. type InternalDownloadParams struct { Redirect *bool `form:"redirect,omitempty" json:"redirect,omitempty"` @@ -346,12 +243,6 @@ type InternalMultipartInitJSONRequestBody = InternalMultipartInitRequest // InternalMultipartUploadJSONRequestBody defines body for InternalMultipartUpload for application/json ContentType. type InternalMultipartUploadJSONRequestBody = InternalMultipartUploadRequest -// InternalStorageCleanupApplyJSONRequestBody defines body for InternalStorageCleanupApply for application/json ContentType. -type InternalStorageCleanupApplyJSONRequestBody = StorageCleanupApplyRequest - -// InternalStorageCleanupAuditJSONRequestBody defines body for InternalStorageCleanupAudit for application/json ContentType. -type InternalStorageCleanupAuditJSONRequestBody = StorageCleanupAuditRequest - // InternalUploadBlankJSONRequestBody defines body for InternalUploadBlank for application/json ContentType. type InternalUploadBlankJSONRequestBody = InternalUploadBlankRequest @@ -462,12 +353,6 @@ type ServerInterface interface { // (POST /data/multipart/upload) InternalMultipartUpload(c fiber.Ctx) error - // (POST /data/repair/storage-cleanup/apply) - InternalStorageCleanupApply(c fiber.Ctx) error - - // (POST /data/repair/storage-cleanup/audit) - InternalStorageCleanupAudit(c fiber.Ctx) error - // (POST /data/upload) InternalUploadBlank(c fiber.Ctx) error @@ -621,18 +506,6 @@ func (siw *ServerInterfaceWrapper) InternalMultipartUpload(c fiber.Ctx) error { return siw.Handler.InternalMultipartUpload(c) } -// InternalStorageCleanupApply operation middleware -func (siw *ServerInterfaceWrapper) InternalStorageCleanupApply(c fiber.Ctx) error { - - return siw.Handler.InternalStorageCleanupApply(c) -} - -// InternalStorageCleanupAudit operation middleware -func (siw *ServerInterfaceWrapper) InternalStorageCleanupAudit(c fiber.Ctx) error { - - return siw.Handler.InternalStorageCleanupAudit(c) -} - // InternalUploadBlank operation middleware func (siw *ServerInterfaceWrapper) InternalUploadBlank(c fiber.Ctx) error { @@ -1005,10 +878,6 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Post(options.BaseURL+"/data/multipart/upload", wrapper.InternalMultipartUpload) - router.Post(options.BaseURL+"/data/repair/storage-cleanup/apply", wrapper.InternalStorageCleanupApply) - - router.Post(options.BaseURL+"/data/repair/storage-cleanup/audit", wrapper.InternalStorageCleanupAudit) - router.Post(options.BaseURL+"/data/upload", wrapper.InternalUploadBlank) router.Post(options.BaseURL+"/data/upload/bulk", wrapper.InternalUploadBulk) @@ -1311,104 +1180,6 @@ func (response InternalMultipartUpload500Response) VisitInternalMultipartUploadR return nil } -type InternalStorageCleanupApplyRequestObject struct { - Body *InternalStorageCleanupApplyJSONRequestBody -} - -type InternalStorageCleanupApplyResponseObject interface { - VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error -} - -type InternalStorageCleanupApply200JSONResponse StorageCleanupApplyResult - -func (response InternalStorageCleanupApply200JSONResponse) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { - ctx.Response().Header.Set("Content-Type", "application/json") - ctx.Status(200) - - return ctx.JSON(&response) -} - -type InternalStorageCleanupApply400Response struct { -} - -func (response InternalStorageCleanupApply400Response) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { - ctx.Status(400) - return nil -} - -type InternalStorageCleanupApply401Response struct { -} - -func (response InternalStorageCleanupApply401Response) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { - ctx.Status(401) - return nil -} - -type InternalStorageCleanupApply403Response struct { -} - -func (response InternalStorageCleanupApply403Response) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { - ctx.Status(403) - return nil -} - -type InternalStorageCleanupApply500Response struct { -} - -func (response InternalStorageCleanupApply500Response) VisitInternalStorageCleanupApplyResponse(ctx fiber.Ctx) error { - ctx.Status(500) - return nil -} - -type InternalStorageCleanupAuditRequestObject struct { - Body *InternalStorageCleanupAuditJSONRequestBody -} - -type InternalStorageCleanupAuditResponseObject interface { - VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error -} - -type InternalStorageCleanupAudit200JSONResponse StorageCleanupReport - -func (response InternalStorageCleanupAudit200JSONResponse) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { - ctx.Response().Header.Set("Content-Type", "application/json") - ctx.Status(200) - - return ctx.JSON(&response) -} - -type InternalStorageCleanupAudit400Response struct { -} - -func (response InternalStorageCleanupAudit400Response) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { - ctx.Status(400) - return nil -} - -type InternalStorageCleanupAudit401Response struct { -} - -func (response InternalStorageCleanupAudit401Response) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { - ctx.Status(401) - return nil -} - -type InternalStorageCleanupAudit403Response struct { -} - -func (response InternalStorageCleanupAudit403Response) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { - ctx.Status(403) - return nil -} - -type InternalStorageCleanupAudit500Response struct { -} - -func (response InternalStorageCleanupAudit500Response) VisitInternalStorageCleanupAuditResponse(ctx fiber.Ctx) error { - ctx.Status(500) - return nil -} - type InternalUploadBlankRequestObject struct { Body *InternalUploadBlankJSONRequestBody } @@ -2133,12 +1904,6 @@ type StrictServerInterface interface { // (POST /data/multipart/upload) InternalMultipartUpload(ctx context.Context, request InternalMultipartUploadRequestObject) (InternalMultipartUploadResponseObject, error) - // (POST /data/repair/storage-cleanup/apply) - InternalStorageCleanupApply(ctx context.Context, request InternalStorageCleanupApplyRequestObject) (InternalStorageCleanupApplyResponseObject, error) - - // (POST /data/repair/storage-cleanup/audit) - InternalStorageCleanupAudit(ctx context.Context, request InternalStorageCleanupAuditRequestObject) (InternalStorageCleanupAuditResponseObject, error) - // (POST /data/upload) InternalUploadBlank(ctx context.Context, request InternalUploadBlankRequestObject) (InternalUploadBlankResponseObject, error) @@ -2347,68 +2112,6 @@ func (sh *strictHandler) InternalMultipartUpload(ctx fiber.Ctx) error { return nil } -// InternalStorageCleanupApply operation middleware -func (sh *strictHandler) InternalStorageCleanupApply(ctx fiber.Ctx) error { - var request InternalStorageCleanupApplyRequestObject - - var body InternalStorageCleanupApplyJSONRequestBody - if err := ctx.Bind().Body(&body); err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } - request.Body = &body - - handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { - return sh.ssi.InternalStorageCleanupApply(ctx.Context(), request.(InternalStorageCleanupApplyRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "InternalStorageCleanupApply") - } - - response, err := handler(ctx, request) - - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } else if validResponse, ok := response.(InternalStorageCleanupApplyResponseObject); ok { - if err := validResponse.VisitInternalStorageCleanupApplyResponse(ctx); err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } - } else if response != nil { - return fmt.Errorf("unexpected response type: %T", response) - } - return nil -} - -// InternalStorageCleanupAudit operation middleware -func (sh *strictHandler) InternalStorageCleanupAudit(ctx fiber.Ctx) error { - var request InternalStorageCleanupAuditRequestObject - - var body InternalStorageCleanupAuditJSONRequestBody - if err := ctx.Bind().Body(&body); err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } - request.Body = &body - - handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { - return sh.ssi.InternalStorageCleanupAudit(ctx.Context(), request.(InternalStorageCleanupAuditRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "InternalStorageCleanupAudit") - } - - response, err := handler(ctx, request) - - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } else if validResponse, ok := response.(InternalStorageCleanupAuditResponseObject); ok { - if err := validResponse.VisitInternalStorageCleanupAuditResponse(ctx); err != nil { - return fiber.NewError(fiber.StatusBadRequest, err.Error()) - } - } else if response != nil { - return fmt.Errorf("unexpected response type: %T", response) - } - return nil -} - // InternalUploadBlank operation middleware func (sh *strictHandler) InternalUploadBlank(ctx fiber.Ctx) error { var request InternalUploadBlankRequestObject diff --git a/client/client.go b/client/client.go index 9ef1ccd2..4b9a6516 100644 --- a/client/client.go +++ b/client/client.go @@ -47,7 +47,6 @@ type Client struct { drs *syfonclient.DRSService buckets *syfonclient.BucketsService metrics *syfonclient.MetricsService - repair *syfonclient.RepairService lfs *syfonclient.LFSService // Generated schema-specific clients @@ -209,7 +208,6 @@ func (c *Client) initServices() error { c.data = syfonclient.NewDataService(c.internalGen, c.requestor, l, c.drs) c.buckets = syfonclient.NewBucketsService(c.bucketGen) c.metrics = syfonclient.NewMetricsService(c.metricsGen) - c.repair = syfonclient.NewRepairService(c.requestor) return nil } @@ -228,7 +226,6 @@ func (c *Client) Index() *syfonclient.IndexService { return c.index } func (c *Client) DRS() *syfonclient.DRSService { return c.drs } func (c *Client) Buckets() *syfonclient.BucketsService { return c.buckets } func (c *Client) Metrics() *syfonclient.MetricsService { return c.metrics } -func (c *Client) Repair() *syfonclient.RepairService { return c.repair } func (c *Client) LFS() *syfonclient.LFSService { return c.lfs } // Schema-specific generated clients diff --git a/client/common/constants.go b/client/common/constants.go index ff69102d..bdeb549c 100644 --- a/client/common/constants.go +++ b/client/common/constants.go @@ -110,7 +110,6 @@ const ( QueryParamURL = "url" QueryParamHashType = "hash_type" QueryParamBucket = "bucket" - QueryParamFileName = "file_name" QueryParamExpiresIn = "expires_in" QueryParamRedirect = "redirect" QueryParamInactiveDays = "inactive_days" diff --git a/client/services/drs_service_test.go b/client/services/drs_service_test.go index 69aa2c34..e0a579ae 100644 --- a/client/services/drs_service_test.go +++ b/client/services/drs_service_test.go @@ -42,7 +42,7 @@ func TestDRSServiceResolveAndList(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/objects/broken": w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) - _, _ = io.WriteString(w, `{"error":{"message":"column \"file_name\" does not exist"}}`) + _, _ = io.WriteString(w, `{"error":{"message":"schema is stale"}}`) case r.Method == http.MethodGet && r.URL.Path == "/objects/obj-1/access/acc-1": writeJSON(t, w, http.StatusOK, drsapi.AccessURL{Url: "https://signed.example/access"}) case r.Method == http.MethodGet && r.URL.Path == "/objects/checksum/abc": @@ -121,7 +121,7 @@ func TestDRSServiceResolveAndList(t *testing.T) { if _, err := service.GetObject(ctx, "missing"); !errors.Is(err, ErrObjectNotFound) { t.Fatalf("expected ErrObjectNotFound, got %v", err) } - if _, err := service.GetObject(ctx, "broken"); err == nil || !strings.Contains(err.Error(), `column "file_name" does not exist`) { + if _, err := service.GetObject(ctx, "broken"); err == nil || !strings.Contains(err.Error(), "schema is stale") { t.Fatalf("expected server error detail for broken object lookup, got %v", err) } diff --git a/client/services/index_service.go b/client/services/index_service.go index 5d66e139..6db2e888 100644 --- a/client/services/index_service.go +++ b/client/services/index_service.go @@ -108,9 +108,6 @@ func (s *IndexService) List(ctx context.Context, opts ListRecordsOptions) (inter if opts.ProjectID != "" { params.Set("project", opts.ProjectID) } - if opts.Path != "" { - params.Set("path", opts.Path) - } if opts.Limit != 0 { params.Set("limit", fmt.Sprintf("%d", opts.Limit)) } diff --git a/client/services/interface.go b/client/services/interface.go index ffa5049d..1c92105b 100644 --- a/client/services/interface.go +++ b/client/services/interface.go @@ -8,6 +8,5 @@ type SyfonClient interface { DRS() *DRSService Buckets() *BucketsService Metrics() *MetricsService - Repair() *RepairService LFS() *LFSService } diff --git a/client/services/repair_service.go b/client/services/repair_service.go deleted file mode 100644 index 57d64e91..00000000 --- a/client/services/repair_service.go +++ /dev/null @@ -1,103 +0,0 @@ -package services - -import ( - "context" - "net/http" - - "github.com/calypr/syfon/client/request" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/repair" -) - -type RepairService struct { - requestor request.Requester -} - -func NewRepairService(r request.Requester) *RepairService { - return &RepairService{requestor: r} -} - -func (s *RepairService) ProjectDiffAudit(ctx context.Context, opts ProjectDiffAuditOptions) (repair.ProjectDiffReport, error) { - req := repair.ProjectDiffAuditRequest{ - Organization: opts.Organization, - Project: opts.ProjectID, - PathPrefix: opts.PathPrefix, - ExpectedPaths: append([]string(nil), opts.ExpectedPaths...), - } - var out repair.ProjectDiffReport - if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairProjectDiff, req, &out); err != nil { - return repair.ProjectDiffReport{}, err - } - return out, nil -} - -func (s *RepairService) StorageCleanupAudit(ctx context.Context, opts StorageCleanupAuditOptions) (repair.StorageCleanupReport, error) { - req := repair.StorageCleanupAuditRequest{ - Organization: opts.Organization, - Project: opts.ProjectID, - PathPrefix: opts.PathPrefix, - ExpectedPaths: append([]string(nil), opts.ExpectedPaths...), - SelectedPaths: append([]string(nil), opts.SelectedPaths...), - CheckStorage: opts.CheckStorage, - } - var out repair.StorageCleanupReport - if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairCleanupAudit, req, &out); err != nil { - return repair.StorageCleanupReport{}, err - } - return out, nil -} - -func (s *RepairService) StorageCleanupApply(ctx context.Context, opts StorageCleanupApplyOptions) (repair.StorageCleanupApplyResult, error) { - kinds := make([]repair.FindingKind, 0, len(opts.SelectedFindingKinds)) - for _, kind := range opts.SelectedFindingKinds { - kinds = append(kinds, repair.FindingKind(kind)) - } - req := repair.StorageCleanupApplyRequest{ - Organization: opts.Organization, - Project: opts.ProjectID, - PathPrefix: opts.PathPrefix, - ExpectedPaths: append([]string(nil), opts.ExpectedPaths...), - DeleteStaleDuplicates: opts.DeleteStaleDuplicates, - DeleteRepoOrphans: opts.DeleteRepoOrphans, - DryRun: opts.DryRun, - SelectedPaths: append([]string(nil), opts.SelectedPaths...), - SelectedObjectIDs: append([]string(nil), opts.SelectedObjectIDs...), - SelectedFindingKinds: kinds, - CheckStorage: opts.CheckStorage, - } - var out repair.StorageCleanupApplyResult - if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairCleanupApply, req, &out); err != nil { - return repair.StorageCleanupApplyResult{}, err - } - return out, nil -} - -func (s *RepairService) ScopeAudit(ctx context.Context, opts ScopeRepairOptions) (repair.Report, error) { - req := repair.Options{ - Organization: opts.Organization, - Project: opts.ProjectID, - CheckStorage: opts.CheckStorage, - Limit: opts.Limit, - PageSize: opts.PageSize, - } - var out repair.Report - if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairScopeAudit, req, &out); err != nil { - return repair.Report{}, err - } - return out, nil -} - -func (s *RepairService) ScopeApply(ctx context.Context, opts ScopeRepairOptions) (repair.ApplyResult, error) { - req := repair.Options{ - Organization: opts.Organization, - Project: opts.ProjectID, - CheckStorage: opts.CheckStorage, - Limit: opts.Limit, - PageSize: opts.PageSize, - } - var out repair.ApplyResult - if err := s.requestor.Do(ctx, http.MethodPost, common.RouteInternalRepairScopeApply, req, &out); err != nil { - return repair.ApplyResult{}, err - } - return out, nil -} diff --git a/client/services/repair_service_test.go b/client/services/repair_service_test.go deleted file mode 100644 index 229757a1..00000000 --- a/client/services/repair_service_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package services - -import ( - "context" - "testing" - - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/repair" -) - -func TestRepairServiceScopeAuditUsesScopeRoute(t *testing.T) { - requester := &fakeRequester{responseJSON: []byte(`{"organization":"org","project":"proj","scanned":1}`)} - service := NewRepairService(requester) - - report, err := service.ScopeAudit(context.Background(), ScopeRepairOptions{ - Organization: "org", - ProjectID: "proj", - CheckStorage: true, - Limit: 10, - PageSize: 5, - }) - if err != nil { - t.Fatalf("ScopeAudit returned error: %v", err) - } - if requester.path != common.RouteInternalRepairScopeAudit { - t.Fatalf("expected scope audit path %q, got %q", common.RouteInternalRepairScopeAudit, requester.path) - } - body, ok := requester.body.(repair.Options) - if !ok { - t.Fatalf("expected repair.Options body, got %T", requester.body) - } - if body.Organization != "org" || body.Project != "proj" || body.Limit != 10 || body.PageSize != 5 || !body.CheckStorage { - t.Fatalf("unexpected scope audit body: %+v", body) - } - if report.Organization != "org" || report.Project != "proj" || report.Scanned != 1 { - t.Fatalf("unexpected scope audit report: %+v", report) - } -} - -func TestRepairServiceScopeApplyUsesScopeRoute(t *testing.T) { - requester := &fakeRequester{responseJSON: []byte(`{"report":{"organization":"org","project":"proj","scanned":1},"mutated":1,"auto_fixable":1}`)} - service := NewRepairService(requester) - - result, err := service.ScopeApply(context.Background(), ScopeRepairOptions{ - Organization: "org", - ProjectID: "proj", - }) - if err != nil { - t.Fatalf("ScopeApply returned error: %v", err) - } - if requester.path != common.RouteInternalRepairScopeApply { - t.Fatalf("expected scope apply path %q, got %q", common.RouteInternalRepairScopeApply, requester.path) - } - body, ok := requester.body.(repair.Options) - if !ok { - t.Fatalf("expected repair.Options body, got %T", requester.body) - } - if body.Organization != "org" || body.Project != "proj" { - t.Fatalf("unexpected scope apply body: %+v", body) - } - if result.Mutated != 1 || result.AutoFixable != 1 || result.Report.Organization != "org" { - t.Fatalf("unexpected scope apply result: %+v", result) - } -} diff --git a/client/services/types.go b/client/services/types.go index 29fbe37a..350dc87b 100644 --- a/client/services/types.go +++ b/client/services/types.go @@ -22,7 +22,6 @@ type ListRecordsOptions struct { URL string Organization string ProjectID string - Path string Limit int Start string Page int @@ -80,41 +79,3 @@ type TransferMetricsOptions struct { ReconciliationStatus string AllowStale bool } - -type StorageCleanupAuditOptions struct { - Organization string - ProjectID string - PathPrefix string - ExpectedPaths []string - SelectedPaths []string - CheckStorage *bool -} - -type ProjectDiffAuditOptions struct { - Organization string - ProjectID string - PathPrefix string - ExpectedPaths []string -} - -type StorageCleanupApplyOptions struct { - Organization string - ProjectID string - PathPrefix string - ExpectedPaths []string - DeleteStaleDuplicates bool - DeleteRepoOrphans bool - DryRun bool - SelectedPaths []string - SelectedObjectIDs []string - SelectedFindingKinds []string - CheckStorage *bool -} - -type ScopeRepairOptions struct { - Organization string - ProjectID string - CheckStorage bool - Limit int - PageSize int -} diff --git a/cmd/list/main.go b/cmd/list/main.go index a3857602..cfa6295c 100644 --- a/cmd/list/main.go +++ b/cmd/list/main.go @@ -43,7 +43,7 @@ var Cmd = &cobra.Command{ Start: strings.TrimSpace(listStart), Organization: strings.TrimSpace(listOrganization), ProjectID: strings.TrimSpace(listProject), - }, listRecursive) + }, listRecursive, strings.TrimSpace(listPath)) if err != nil { return err } @@ -91,8 +91,8 @@ func init() { Cmd.Flags().StringVar(&listProject, "project", "", "Optional project filter") } -func listRecords(ctx context.Context, lister indexLister, opts syfonclient.ListRecordsOptions, recursive bool) ([]internalapi.InternalRecord, error) { - if strings.TrimSpace(opts.Path) != "" { +func listRecords(ctx context.Context, lister indexLister, opts syfonclient.ListRecordsOptions, recursive bool, listPath string) ([]internalapi.InternalRecord, error) { + if strings.TrimSpace(listPath) != "" { return nil, fmt.Errorf("path-based listing is no longer supported") } if strings.TrimSpace(opts.Organization) == "" || strings.TrimSpace(opts.ProjectID) == "" { diff --git a/cmd/list/main_test.go b/cmd/list/main_test.go index fb0c0f6d..c2e82172 100644 --- a/cmd/list/main_test.go +++ b/cmd/list/main_test.go @@ -40,7 +40,7 @@ func TestListRecordsNonRecursive(t *testing.T) { Organization: "Ellrott_Lab", ProjectID: "hla2vec", Limit: 10, - }, false) + }, false, "") if err != nil { t.Fatalf("listRecords returned error: %v", err) } @@ -60,17 +60,14 @@ func TestListRecordsRejectsRecursive(t *testing.T) { Organization: "Ellrott_Lab", ProjectID: "hla2vec", Limit: 10, - }, true) + }, true, "") if err == nil || !strings.Contains(err.Error(), "path-based recursive listing is no longer supported") { t.Fatalf("expected recursive unsupported error, got %v", err) } } func TestListRecordsRejectsPathFilter(t *testing.T) { - _, err := listRecords(context.Background(), &fakeIndexLister{}, syfonclient.ListRecordsOptions{ - Path: "nested", - Limit: 10, - }, false) + _, err := listRecords(context.Background(), &fakeIndexLister{}, syfonclient.ListRecordsOptions{Limit: 10}, false, "nested") if err == nil || !strings.Contains(err.Error(), "path-based listing is no longer supported") { t.Fatalf("expected path unsupported error, got %v", err) } diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go deleted file mode 100644 index 69ed6856..00000000 --- a/cmd/migrate/main.go +++ /dev/null @@ -1,302 +0,0 @@ -package migratecmd - -import ( - "context" - "errors" - "fmt" - "net/http" - "strings" - "time" - - conf "github.com/calypr/syfon/client/config" - "github.com/calypr/syfon/cmd/cliauth" - "github.com/calypr/syfon/migrate" - "github.com/spf13/cobra" -) - -var Cmd = &cobra.Command{ - Use: "migrate", - Short: "Export and import Gen3 Indexd records for Syfon migration", - Long: "Export records from a Gen3-mounted Indexd API into a local SQLite dump, then import that dump into Syfon using the controlled_access-aware /index/bulk loader.", -} - -var ( - dumpPath string - sourceProfile string - targetProfile string - sourceToken string - targetToken string - targetBasicUser string - targetBasicPassword string - batchSize int - limit int - sweeps int - dryRun bool - skipAuthPreflight bool - defaultAuthz []string -) - -var exportCmd = &cobra.Command{ - Use: "export", - Short: "Export records from Gen3 Indexd into a SQLite dump", - RunE: func(cmd *cobra.Command, args []string) error { - sourceURL, err := indexdURLFromServer(cmd) - if err != nil { - return err - } - if strings.TrimSpace(sourceProfile) == "" { - sourceProfile = cliauth.ResolvedProfile() - } - dumpPath = normalizedDumpPath(dumpPath) - - sourceAuth, err := authFromInputs(sourceProfile, sourceToken) - if err != nil { - return fmt.Errorf("source auth: %w", err) - } - httpClient := migrationHTTPClient() - source, err := migrate.NewHTTPClient(sourceURL, sourceAuth, httpClient) - if err != nil { - return err - } - var dump *migrate.SQLiteDump - if !dryRun { - dump, err = migrate.OpenSQLiteDump(dumpPath) - if err != nil { - return err - } - defer dump.Close() - } - - if dryRun { - cmd.Println("dry-run: no records will be written to the dump") - } - displayDump := dumpPath - if dryRun { - displayDump = "" - } - cmd.Printf("migration export starting: indexd=%s dump=%s batch=%d limit=%d sweeps=%d\n", sourceURL, displayDump, batchSize, limit, sweeps) - stats, err := migrate.Run(context.Background(), source, dump, migrate.Config{ - BatchSize: batchSize, - Limit: limit, - DryRun: dryRun, - DefaultAuthz: defaultAuthz, - Sweeps: sweeps, - }) - if err != nil { - return err - } - count := 0 - if dump != nil { - var countErr error - count, countErr = dump.Count(context.Background()) - if countErr != nil { - return countErr - } - } - if dryRun { - cmd.Printf("migration export dry-run complete: fetched=%d transformed=%d valid=%d skipped=%d errors=%d unique_ids=%d dump_records=0\n", - stats.Fetched, - stats.Transformed, - stats.Loaded, - stats.Skipped, - stats.Errors, - stats.CountOfUniqueIDs, - ) - return nil - } - cmd.Printf("migration export complete: fetched=%d transformed=%d dumped=%d skipped=%d errors=%d unique_ids=%d dump_records=%d\n", - stats.Fetched, - stats.Transformed, - stats.Loaded, - stats.Skipped, - stats.Errors, - stats.CountOfUniqueIDs, - count, - ) - return nil - }, -} - -var importCmd = &cobra.Command{ - Use: "import", - Short: "Import a migration SQLite dump into Syfon", - RunE: func(cmd *cobra.Command, args []string) error { - if strings.TrimSpace(dumpPath) == "" { - return fmt.Errorf("--dump is required") - } - targetURL, err := targetServerURL(cmd) - if err != nil { - return err - } - if strings.TrimSpace(targetProfile) == "" { - targetProfile = sourceProfile - } - if strings.TrimSpace(targetProfile) == "" { - targetProfile = cliauth.ResolvedProfile() - } - targetAuth, err := targetAuthFromInputs(targetProfile, targetToken, targetBasicUser, targetBasicPassword) - if err != nil { - return fmt.Errorf("target auth: %w", err) - } - target, err := migrate.NewHTTPClient(targetURL, targetAuth, migrationHTTPClient()) - if err != nil { - return err - } - dump, err := migrate.OpenExistingSQLiteDump(dumpPath) - if err != nil { - return err - } - defer dump.Close() - - cmd.Printf("migration import starting: dump=%s syfon=%s batch=%d\n", dumpPath, targetURL, batchSize) - if !skipAuthPreflight && strings.TrimSpace(targetAuth.BearerToken) != "" { - cmd.Println("migration import preflight: checking target create privileges") - report, err := migrate.PreflightImport(context.Background(), dump, target, batchSize) - if err != nil { - var preflightErr *migrate.ImportPreflightError - if errors.As(err, &preflightErr) { - printImportPreflightFailure(cmd, preflightErr.Report) - return fmt.Errorf("target authorization preflight failed: missing create access for %d/%d records across %d scopes; copy/paste scope list printed above", - preflightErr.Report.MissingRecords, - preflightErr.Report.Records, - len(preflightErr.Report.MissingResources), - ) - } - return fmt.Errorf("target authorization preflight failed: %w", err) - } - cmd.Printf("migration import preflight complete: records=%d required_scopes=%d missing_scopes=0\n", - report.Records, - len(report.RequiredResources), - ) - } - stats, err := migrate.Import(context.Background(), dump, target, batchSize) - if err != nil { - return err - } - cmd.Printf("migration import complete: read=%d loaded=%d skipped=%d errors=%d\n", - stats.Fetched, - stats.Loaded, - stats.Skipped, - stats.Errors, - ) - return nil - }, -} - -func printImportPreflightFailure(cmd *cobra.Command, report migrate.ImportPreflightReport) { - scopes := migrate.FormatPreflightScopes(report.MissingResources) - cmd.Printf("migration import preflight failed: missing create access for %d/%d records across %d scopes\n", - report.MissingRecords, - report.Records, - len(scopes), - ) - if report.FirstDeniedRecord != "" { - cmd.Printf("first denied record: %s\n", report.FirstDeniedRecord) - } - cmd.Println() - cmd.Println("Copy/paste scope list:") - cmd.Println(strings.Join(scopes, ", ")) - cmd.Println() -} - -func init() { - Cmd.AddCommand(exportCmd) - Cmd.AddCommand(importCmd) - - exportCmd.Flags().StringVar(&dumpPath, "dump", "", "SQLite dump file to write (default ./indexd-records.sqlite)") - exportCmd.Flags().StringVar(&sourceProfile, "source-profile", "", "Gen3 profile for source reads from ~/.gen3/gen3_client_config.ini") - exportCmd.Flags().StringVar(&sourceToken, "source-token", "", "Bearer token for source reads; overrides --source-profile") - exportCmd.Flags().IntVar(&batchSize, "batch-size", 500, "Records to fetch per batch; capped at Indexd's 1024 max") - exportCmd.Flags().IntVar(&limit, "limit", 0, "Maximum number of unique records to export; 0 means all records") - exportCmd.Flags().IntVar(&sweeps, "sweeps", 1, "Full start-cursor sweeps to run; increase only for non-quiescent or inconsistent Indexd sources") - exportCmd.Flags().BoolVar(&dryRun, "dry-run", false, "Fetch, transform, and validate only; do not write the dump") - exportCmd.Flags().StringArrayVar(&defaultAuthz, "default-controlled-access", nil, "Default controlled_access resource for source records with no authz; repeatable") - exportCmd.Flags().StringArrayVar(&defaultAuthz, "default-authz", nil, "Deprecated alias for --default-controlled-access") - - importCmd.Flags().StringVar(&dumpPath, "dump", "", "SQLite dump file to read") - importCmd.Flags().StringVar(&targetProfile, "target-profile", "", "Gen3 profile for target writes") - importCmd.Flags().StringVar(&targetToken, "target-token", "", "Bearer token for target writes; overrides --target-profile") - importCmd.Flags().StringVar(&targetBasicUser, "target-basic-user", "", "Basic auth username for local target Syfon writes") - importCmd.Flags().StringVar(&targetBasicPassword, "target-basic-password", "", "Basic auth password for local target Syfon writes") - importCmd.Flags().IntVar(&batchSize, "batch-size", 500, "Records to load per controlled_access-aware /index/bulk request") - importCmd.Flags().BoolVar(&skipAuthPreflight, "skip-auth-preflight", false, "Skip target create-privilege preflight before importing") -} - -func indexdURLFromServer(cmd *cobra.Command) (string, error) { - server, err := targetServerURL(cmd) - if err != nil { - return "", err - } - return strings.TrimRight(server, "/") + "/index/index", nil -} - -func normalizedDumpPath(path string) string { - path = strings.TrimSpace(path) - if path == "" { - return "indexd-records.sqlite" - } - return path -} - -func migrationHTTPClient() *http.Client { - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSHandshakeTimeout = 60 * time.Second - transport.ResponseHeaderTimeout = 90 * time.Second - return &http.Client{ - Timeout: 3 * time.Minute, - Transport: transport, - } -} - -func targetServerURL(cmd *cobra.Command) (string, error) { - return cliauth.ResolveServerURL(cmd) -} - -func authFromInputs(profile, token string) (migrate.AuthConfig, error) { - token = strings.TrimSpace(token) - if token != "" { - return migrate.AuthConfig{BearerToken: token}, nil - } - - profile = strings.TrimSpace(profile) - if profile == "" { - return migrate.AuthConfig{}, nil - } - - credential, err := cliauth.LoadProfileCredential(context.Background(), profile) - if err != nil { - return migrate.AuthConfig{}, err - } - return authFromCredential(profile, credential) -} - -func authFromCredential(profile string, credential *conf.Credential) (migrate.AuthConfig, error) { - if credential == nil { - return migrate.AuthConfig{}, fmt.Errorf("profile %q resolved to nil credential", profile) - } - if strings.TrimSpace(credential.AccessToken) != "" { - return migrate.AuthConfig{BearerToken: credential.AccessToken}, nil - } - if strings.TrimSpace(credential.KeyID) != "" || strings.TrimSpace(credential.APIKey) != "" { - return migrate.AuthConfig{Basic: &migrate.BasicAuth{Username: credential.KeyID, Password: credential.APIKey}}, nil - } - return migrate.AuthConfig{}, fmt.Errorf("profile %q has no access_token or key_id/api_key", profile) -} - -func targetAuthFromInputs(profile, token, basicUser, basicPassword string) (migrate.AuthConfig, error) { - basicUser = strings.TrimSpace(basicUser) - basicPassword = strings.TrimSpace(basicPassword) - if basicUser != "" || basicPassword != "" { - if basicUser == "" || basicPassword == "" { - return migrate.AuthConfig{}, fmt.Errorf("--target-basic-user and --target-basic-password must be set together") - } - if strings.TrimSpace(token) != "" { - return migrate.AuthConfig{}, fmt.Errorf("--target-token cannot be combined with --target-basic-user/--target-basic-password") - } - if strings.TrimSpace(profile) != "" { - return migrate.AuthConfig{}, fmt.Errorf("--target-profile cannot be combined with --target-basic-user/--target-basic-password") - } - return migrate.AuthConfig{Basic: &migrate.BasicAuth{Username: basicUser, Password: basicPassword}}, nil - } - return authFromInputs(profile, token) -} diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go deleted file mode 100644 index 8d9693d9..00000000 --- a/cmd/migrate/main_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package migratecmd - -import ( - "testing" - - conf "github.com/calypr/syfon/client/config" - "github.com/spf13/cobra" -) - -func TestNormalizedDumpPathDefaultsExport(t *testing.T) { - if got := normalizedDumpPath(""); got != "indexd-records.sqlite" { - t.Fatalf("expected default dump path, got %q", got) - } - if got := normalizedDumpPath(" custom.sqlite "); got != "custom.sqlite" { - t.Fatalf("expected trimmed custom dump path, got %q", got) - } -} - -func TestIndexdURLFromServerUsesRootServerFlag(t *testing.T) { - root := &cobra.Command{Use: "syfon"} - root.PersistentFlags().String("server", "https://calypr-dev.ohsu.edu/", "") - cmd := &cobra.Command{Use: "export"} - root.AddCommand(cmd) - - got, err := indexdURLFromServer(cmd) - if err != nil { - t.Fatalf("indexdURLFromServer returned error: %v", err) - } - if got != "https://calypr-dev.ohsu.edu/index/index" { - t.Fatalf("unexpected indexd url: %q", got) - } -} - -func TestTargetAuthFromInputsSupportsBasicAuth(t *testing.T) { - auth, err := targetAuthFromInputs("", "", "drs-user", "drs-pass") - if err != nil { - t.Fatalf("targetAuthFromInputs returned error: %v", err) - } - if auth.Basic == nil || auth.Basic.Username != "drs-user" || auth.Basic.Password != "drs-pass" { - t.Fatalf("unexpected basic auth config: %+v", auth.Basic) - } -} - -func TestTargetAuthFromInputsRequiresBasicPair(t *testing.T) { - if _, err := targetAuthFromInputs("", "", "drs-user", ""); err == nil { - t.Fatal("expected missing password error") - } - if _, err := targetAuthFromInputs("", "", "", "drs-pass"); err == nil { - t.Fatal("expected missing username error") - } -} - -func TestTargetAuthFromInputsRejectsMixedAuth(t *testing.T) { - if _, err := targetAuthFromInputs("", "token", "drs-user", "drs-pass"); err == nil { - t.Fatal("expected token/basic conflict error") - } - if _, err := targetAuthFromInputs("profile", "", "drs-user", "drs-pass"); err == nil { - t.Fatal("expected profile/basic conflict error") - } -} - -func TestAuthFromCredentialPrefersBearerToken(t *testing.T) { - t.Parallel() - - auth, err := authFromCredential("calypr", &conf.Credential{ - Profile: "calypr", - AccessToken: "new-token", - APIKey: "refresh-jwt", - }) - if err != nil { - t.Fatalf("authFromCredential returned error: %v", err) - } - if auth.BearerToken != "new-token" { - t.Fatalf("expected bearer token, got %+v", auth) - } - if auth.Basic != nil { - t.Fatalf("expected bearer auth only, got basic %+v", auth.Basic) - } -} diff --git a/cmd/repair/main.go b/cmd/repair/main.go deleted file mode 100644 index 10d1cbcb..00000000 --- a/cmd/repair/main.go +++ /dev/null @@ -1,380 +0,0 @@ -package repair - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - syclient "github.com/calypr/syfon/client" - "github.com/calypr/syfon/client/services" - "github.com/calypr/syfon/cmd/cliauth" - sycommon "github.com/calypr/syfon/internal/common" - repairsvc "github.com/calypr/syfon/internal/repair" - "github.com/spf13/cobra" -) - -type auditRunner func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error -type applyRunner func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error -type scopeAuditRunner func(cmd *cobra.Command, opts repairsvc.Options) error -type scopeApplyRunner func(cmd *cobra.Command, opts repairsvc.Options) error - -type commandOptions struct { - Organization string - Project string - PathPrefix string - ExpectedPathsFile string - RepoRoot string - CheckStorage bool - Format string - DeleteStaleDuplicates bool - DeleteRepoOrphans bool - DryRun bool - SelectedPaths []string - SelectedObjectIDs []string - SelectedFindingKinds []string -} - -var Cmd = newCommand(runAudit, runApply, runScopeAudit, runScopeApply) - -func newCommand(audit auditRunner, apply applyRunner, scopeAudit scopeAuditRunner, scopeApply scopeApplyRunner) *cobra.Command { - var opts commandOptions - cmd := &cobra.Command{ - Use: "repair", - Short: "Audit and apply Syfon storage cleanup actions", - } - addSharedFlags(cmd, &opts) - cmd.AddCommand(&cobra.Command{ - Use: "audit", - Short: "Audit duplicate and orphaned storage paths", - RunE: func(cmd *cobra.Command, args []string) error { - req, err := normalizedAuditOptions(opts) - if err != nil { - return err - } - return audit(cmd, req) - }, - }) - cmd.AddCommand(&cobra.Command{ - Use: "apply", - Short: "Apply storage cleanup actions", - PreRunE: func(cmd *cobra.Command, args []string) error { - req, err := normalizedApplyOptions(opts) - if err != nil { - return err - } - if req.Organization == "" || req.Project == "" { - return fmt.Errorf("repair apply requires --organization and --project") - } - if !req.DeleteStaleDuplicates && !req.DeleteRepoOrphans { - return fmt.Errorf("repair apply requires --delete-stale-duplicates and/or --delete-repo-orphans") - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - req, err := normalizedApplyOptions(opts) - if err != nil { - return err - } - return apply(cmd, req) - }, - }) - cmd.AddCommand(&cobra.Command{ - Use: "scope-audit", - Short: "Audit records missing canonical controlled_access for a project scope", - PreRunE: func(cmd *cobra.Command, args []string) error { - req := normalizedScopeOptions(opts) - if req.Organization == "" || req.Project == "" { - return fmt.Errorf("repair scope-audit requires --organization and --project") - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - return scopeAudit(cmd, normalizedScopeOptions(opts)) - }, - }) - cmd.AddCommand(&cobra.Command{ - Use: "scope-apply", - Short: "Backfill canonical controlled_access for a project scope", - PreRunE: func(cmd *cobra.Command, args []string) error { - req := normalizedScopeOptions(opts) - if req.Organization == "" || req.Project == "" { - return fmt.Errorf("repair scope-apply requires --organization and --project") - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - return scopeApply(cmd, normalizedScopeOptions(opts)) - }, - }) - return cmd -} - -func addSharedFlags(cmd *cobra.Command, opts *commandOptions) { - cmd.PersistentFlags().StringVar(&opts.Organization, "organization", "", "Organization to scope the cleanup") - cmd.PersistentFlags().StringVar(&opts.Project, "project", "", "Project to scope the cleanup") - cmd.PersistentFlags().StringVar(&opts.PathPrefix, "path-prefix", "", "Optional path prefix under the project") - cmd.PersistentFlags().StringVar(&opts.ExpectedPathsFile, "expected-paths-file", "", "File containing expected normalized paths (newline-delimited or JSON array)") - cmd.PersistentFlags().StringVar(&opts.RepoRoot, "repo-root", "", "Local checkout root to walk for expected paths") - cmd.PersistentFlags().BoolVar(&opts.CheckStorage, "check-storage", true, "Probe storage object existence for candidate records") - cmd.PersistentFlags().StringVar(&opts.Format, "format", "text", "Output format: text or json") - cmd.PersistentFlags().BoolVar(&opts.DeleteStaleDuplicates, "delete-stale-duplicates", false, "Delete stale duplicate Syfon records") - cmd.PersistentFlags().BoolVar(&opts.DeleteRepoOrphans, "delete-repo-orphans", false, "Delete Syfon records and purge storage for repo-orphaned live objects") - cmd.PersistentFlags().BoolVar(&opts.DryRun, "dry-run", false, "Report actions without deleting records") - cmd.PersistentFlags().StringSliceVar(&opts.SelectedPaths, "selected-path", nil, "Limit apply to one or more normalized paths") - cmd.PersistentFlags().StringSliceVar(&opts.SelectedObjectIDs, "selected-object-id", nil, "Limit apply to one or more Syfon object IDs") - cmd.PersistentFlags().StringSliceVar(&opts.SelectedFindingKinds, "selected-finding-kind", nil, "Limit apply to one or more finding kinds") -} - -func normalizedAuditOptions(opts commandOptions) (repairsvc.StorageCleanupAuditRequest, error) { - expected, err := loadExpectedPaths(opts.ExpectedPathsFile, opts.RepoRoot) - if err != nil { - return repairsvc.StorageCleanupAuditRequest{}, err - } - checkStorage := opts.CheckStorage - return repairsvc.StorageCleanupAuditRequest{ - Organization: strings.TrimSpace(opts.Organization), - Project: strings.TrimSpace(opts.Project), - PathPrefix: strings.TrimSpace(opts.PathPrefix), - ExpectedPaths: expected, - CheckStorage: &checkStorage, - }, nil -} - -func normalizedApplyOptions(opts commandOptions) (repairsvc.StorageCleanupApplyRequest, error) { - expected, err := loadExpectedPaths(opts.ExpectedPathsFile, opts.RepoRoot) - if err != nil { - return repairsvc.StorageCleanupApplyRequest{}, err - } - checkStorage := opts.CheckStorage - kinds := make([]repairsvc.FindingKind, 0, len(opts.SelectedFindingKinds)) - for _, kind := range opts.SelectedFindingKinds { - kinds = append(kinds, repairsvc.FindingKind(strings.TrimSpace(kind))) - } - return repairsvc.StorageCleanupApplyRequest{ - Organization: strings.TrimSpace(opts.Organization), - Project: strings.TrimSpace(opts.Project), - PathPrefix: strings.TrimSpace(opts.PathPrefix), - ExpectedPaths: expected, - DeleteStaleDuplicates: opts.DeleteStaleDuplicates, - DeleteRepoOrphans: opts.DeleteRepoOrphans, - DryRun: opts.DryRun, - SelectedPaths: trimNonEmpty(opts.SelectedPaths), - SelectedObjectIDs: trimNonEmpty(opts.SelectedObjectIDs), - SelectedFindingKinds: kinds, - CheckStorage: &checkStorage, - }, nil -} - -func normalizedScopeOptions(opts commandOptions) repairsvc.Options { - return repairsvc.Options{ - Organization: strings.TrimSpace(opts.Organization), - Project: strings.TrimSpace(opts.Project), - CheckStorage: opts.CheckStorage, - } -} - -func runAudit(cmd *cobra.Command, req repairsvc.StorageCleanupAuditRequest) error { - client, err := buildClient(cmd) - if err != nil { - return err - } - report, err := client.Repair().StorageCleanupAudit(cmd.Context(), servicesAuditOptions(req)) - if err != nil { - return err - } - return repairsvc.WriteStorageCleanupAudit(cmd.OutOrStdout(), normalizedFormat(cmd), report) -} - -func runApply(cmd *cobra.Command, req repairsvc.StorageCleanupApplyRequest) error { - client, err := buildClient(cmd) - if err != nil { - return err - } - result, err := client.Repair().StorageCleanupApply(cmd.Context(), servicesApplyOptions(req)) - if err != nil { - return err - } - return repairsvc.WriteStorageCleanupApply(cmd.OutOrStdout(), normalizedFormat(cmd), result) -} - -func runScopeAudit(cmd *cobra.Command, req repairsvc.Options) error { - client, err := buildClient(cmd) - if err != nil { - return err - } - report, err := client.Repair().ScopeAudit(cmd.Context(), services.ScopeRepairOptions{ - Organization: req.Organization, - ProjectID: req.Project, - CheckStorage: req.CheckStorage, - Limit: req.Limit, - PageSize: req.PageSize, - }) - if err != nil { - return err - } - return repairsvc.WriteAudit(cmd.OutOrStdout(), normalizedFormat(cmd), report) -} - -func runScopeApply(cmd *cobra.Command, req repairsvc.Options) error { - client, err := buildClient(cmd) - if err != nil { - return err - } - result, err := client.Repair().ScopeApply(cmd.Context(), services.ScopeRepairOptions{ - Organization: req.Organization, - ProjectID: req.Project, - CheckStorage: req.CheckStorage, - Limit: req.Limit, - PageSize: req.PageSize, - }) - if err != nil { - return err - } - return repairsvc.WriteApply(cmd.OutOrStdout(), normalizedFormat(cmd), result) -} - -func buildClient(cmd *cobra.Command) (*syclient.Client, error) { - client, err := cliauth.NewServerClient(cmd) - if err != nil { - return nil, err - } - concrete, ok := client.(*syclient.Client) - if !ok { - return nil, fmt.Errorf("unexpected syfon client type %T", client) - } - return concrete, nil -} - -func normalizedFormat(cmd *cobra.Command) string { - format, _ := cmd.Flags().GetString("format") - format = strings.ToLower(strings.TrimSpace(format)) - if format == "" { - return "text" - } - return format -} - -func servicesAuditOptions(req repairsvc.StorageCleanupAuditRequest) services.StorageCleanupAuditOptions { - return services.StorageCleanupAuditOptions{ - Organization: req.Organization, - ProjectID: req.Project, - PathPrefix: req.PathPrefix, - ExpectedPaths: append([]string(nil), req.ExpectedPaths...), - CheckStorage: req.CheckStorage, - } -} - -func servicesApplyOptions(req repairsvc.StorageCleanupApplyRequest) services.StorageCleanupApplyOptions { - kinds := make([]string, 0, len(req.SelectedFindingKinds)) - for _, kind := range req.SelectedFindingKinds { - kinds = append(kinds, string(kind)) - } - return services.StorageCleanupApplyOptions{ - Organization: req.Organization, - ProjectID: req.Project, - PathPrefix: req.PathPrefix, - ExpectedPaths: append([]string(nil), req.ExpectedPaths...), - DeleteStaleDuplicates: req.DeleteStaleDuplicates, - DeleteRepoOrphans: req.DeleteRepoOrphans, - DryRun: req.DryRun, - SelectedPaths: append([]string(nil), req.SelectedPaths...), - SelectedObjectIDs: append([]string(nil), req.SelectedObjectIDs...), - SelectedFindingKinds: kinds, - CheckStorage: req.CheckStorage, - } -} - -func loadExpectedPaths(filePath, repoRoot string) ([]string, error) { - filePath = strings.TrimSpace(filePath) - repoRoot = strings.TrimSpace(repoRoot) - if filePath != "" && repoRoot != "" { - return nil, fmt.Errorf("--expected-paths-file and --repo-root are mutually exclusive") - } - switch { - case filePath != "": - return loadExpectedPathsFile(filePath) - case repoRoot != "": - return walkExpectedPaths(repoRoot) - default: - return nil, nil - } -} - -func loadExpectedPathsFile(path string) ([]string, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var raw []string - if json.Unmarshal(data, &raw) == nil { - return normalizeManifestPaths(raw) - } - lines := strings.Split(string(data), "\n") - return normalizeManifestPaths(lines) -} - -func walkExpectedPaths(root string) ([]string, error) { - root, err := filepath.Abs(root) - if err != nil { - return nil, err - } - out := make([]string, 0, 256) - err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - if d.Name() == ".git" { - return filepath.SkipDir - } - return nil - } - rel, err := filepath.Rel(root, path) - if err != nil { - return err - } - normalized := filepath.ToSlash(rel) - out = append(out, normalized) - return nil - }) - if err != nil { - return nil, err - } - return normalizeManifestPaths(out) -} - -func normalizeManifestPaths(values []string) ([]string, error) { - out := make([]string, 0, len(values)) - seen := map[string]bool{} - for _, raw := range values { - raw = strings.TrimSpace(raw) - if raw == "" { - continue - } - normalized, _, err := sycommon.NormalizeBrowsePath(raw) - if err != nil { - return nil, err - } - if seen[normalized] { - continue - } - seen[normalized] = true - out = append(out, normalized) - } - sort.Strings(out) - return out, nil -} - -func trimNonEmpty(values []string) []string { - out := make([]string, 0, len(values)) - for _, value := range values { - value = strings.TrimSpace(value) - if value != "" { - out = append(out, value) - } - } - return out -} diff --git a/cmd/repair/main_test.go b/cmd/repair/main_test.go deleted file mode 100644 index 7f6a45ca..00000000 --- a/cmd/repair/main_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package repair - -import ( - "testing" - - repairsvc "github.com/calypr/syfon/internal/repair" - "github.com/spf13/cobra" -) - -func TestApplyRequiresScope(t *testing.T) { - cmd := newCommand( - func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, - ) - cmd.SetArgs([]string{"apply", "--delete-stale-duplicates"}) - err := cmd.Execute() - if err == nil || err.Error() != "repair apply requires --organization and --project" { - t.Fatalf("expected scope validation error, got %v", err) - } -} - -func TestAuditCollectsCheckStorageFlag(t *testing.T) { - called := false - cmd := newCommand( - func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { - called = true - if opts.CheckStorage == nil || !*opts.CheckStorage { - t.Fatal("expected check-storage true") - } - return nil - }, - func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, - ) - cmd.SetArgs([]string{"audit", "--check-storage"}) - if err := cmd.Execute(); err != nil { - t.Fatalf("audit failed: %v", err) - } - if !called { - t.Fatal("expected audit runner call") - } -} - -func TestApplyRequiresDeleteMode(t *testing.T) { - cmd := newCommand( - func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, - ) - cmd.SetArgs([]string{"apply", "--organization", "org", "--project", "proj"}) - err := cmd.Execute() - if err == nil || err.Error() != "repair apply requires --delete-stale-duplicates and/or --delete-repo-orphans" { - t.Fatalf("expected delete mode validation error, got %v", err) - } -} - -func TestScopeApplyRequiresScope(t *testing.T) { - cmd := newCommand( - func(cmd *cobra.Command, opts repairsvc.StorageCleanupAuditRequest) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.StorageCleanupApplyRequest) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, - func(cmd *cobra.Command, opts repairsvc.Options) error { return nil }, - ) - cmd.SetArgs([]string{"scope-apply"}) - err := cmd.Execute() - if err == nil || err.Error() != "repair scope-apply requires --organization and --project" { - t.Fatalf("expected scope apply validation error, got %v", err) - } -} diff --git a/cmd/root.go b/cmd/root.go index bdaae05b..3551e21d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,7 +13,6 @@ import ( "github.com/calypr/syfon/cmd/download" listcmd "github.com/calypr/syfon/cmd/list" metricscmd "github.com/calypr/syfon/cmd/metrics" - migratecmd "github.com/calypr/syfon/cmd/migrate" "github.com/calypr/syfon/cmd/ping" "github.com/calypr/syfon/cmd/rm" "github.com/calypr/syfon/cmd/server" @@ -60,7 +59,6 @@ func init() { RootCmd.AddCommand(listcmd.Cmd) RootCmd.AddCommand(rm.Cmd) RootCmd.AddCommand(metricscmd.Cmd) - RootCmd.AddCommand(migratecmd.Cmd) RootCmd.AddCommand(copyproject.Cmd) RootCmd.AddCommand(copyprojectrefs.Cmd) RootCmd.AddCommand(deleteproject.Cmd) diff --git a/cmd/server/endpoints_mock_test.go b/cmd/server/endpoints_mock_test.go index fa06b963..3cb6072e 100644 --- a/cmd/server/endpoints_mock_test.go +++ b/cmd/server/endpoints_mock_test.go @@ -275,9 +275,9 @@ func requestBodyFor(method, template string) ([]byte, string) { } return []byte(`{}`), "application/json" case "/data/upload/bulk": - return []byte(`{"requests":[{"file_id":"sha-1","bucket":"test-bucket-1","file_name":"sha-1"}]}`), "application/json" + return []byte(`{"requests":[{"file_id":"sha-1","bucket":"test-bucket-1","name":"sha-1"}]}`), "application/json" case "/data/multipart/init": - return []byte(`{"guid":"sha-1","file_name":"sha-1","bucket":"test-bucket-1"}`), "application/json" + return []byte(`{"guid":"sha-1","name":"sha-1","bucket":"test-bucket-1"}`), "application/json" case "/data/multipart/upload": return []byte(`{"key":"sha-1","bucket":"test-bucket-1","uploadId":"mock-upload-id","partNumber":1}`), "application/json" case "/data/multipart/complete": diff --git a/cmd/server/server_integration_test.go b/cmd/server/server_integration_test.go index cc0fefbd..254fd39f 100644 --- a/cmd/server/server_integration_test.go +++ b/cmd/server/server_integration_test.go @@ -199,7 +199,7 @@ s3_credentials: // 4. Internal multipart init still works after credential preload. internalMultipartReq := map[string]interface{}{ "guid": guid, - "file_name": "test-multipart", + "name": "test-multipart", "organization": organization, "project": project, } diff --git a/docs/frontend-handoff-storage-metrics.md b/docs/frontend-handoff-storage-metrics.md deleted file mode 100644 index c2599532..00000000 --- a/docs/frontend-handoff-storage-metrics.md +++ /dev/null @@ -1,272 +0,0 @@ -# Frontend Handoff: Syfon Path-Level Storage Metrics - -This note describes the backend storage-metrics work that was just added in -`syfon` so the frontend can stop recursively walking `/index` and computing -directory totals client-side. - -## What Was Added - -Two new metrics endpoints now exist under the metrics namespace: - -1. `GET /index/v1/metrics/storage/summary` -2. `GET /index/v1/metrics/storage/children` - -These are project-scoped path metrics backed by Syfon’s existing -`drs_object_browse_index` path model plus `drs_object.size` and -`drs_object.updated_time`. - -The important part: the backend is reusing the same normalized browse-path -semantics as the existing path browsing queries, not inventing a second tree -definition. - -## Why Frontend Should Use This - -Old behavior: - -- page through all `/index` results for a project -- rebuild a directory tree in the client -- sum bytes and counts locally - -New behavior: - -- ask Syfon for one path summary -- ask Syfon for direct children of the current path -- render the result directly - -This removes: - -- full-project pagination just to render a folder row -- repeated client aggregation work -- large response payloads for simple summary views - -## Endpoint 1: Storage Summary - -Route: - -`GET /index/v1/metrics/storage/summary` - -Required query params: - -- `organization` -- `project` - -Optional query params: - -- `path` - -If `path` is omitted, the summary is for the project root. - -Example: - -```http -GET /index/v1/metrics/storage/summary?organization=cbds&project=end_to_end_test&path=data -``` - -Response shape: - -```json -{ - "organization": "cbds", - "project": "end_to_end_test", - "path": "data", - "file_count": 2, - "record_count": 3, - "total_bytes": 60, - "direct_child_count": 2, - "duplicate_path_count": 1, - "latest_update_time": "2026-06-24T22:00:00Z" -} -``` - -Field meanings: - -- `file_count`: distinct normalized file paths under the subtree -- `record_count`: total indexed records under the subtree -- `total_bytes`: sum of `drs_object.size` for all matching records -- `direct_child_count`: number of immediate children under the requested path -- `duplicate_path_count`: count of normalized file paths with more than one indexed record -- `latest_update_time`: max `drs_object.updated_time` seen in the subtree - -## Endpoint 2: Storage Children - -Route: - -`GET /index/v1/metrics/storage/children` - -Required query params: - -- `organization` -- `project` - -Optional query params: - -- `path` -- `limit` -- `offset` -- `sort_by` -- `sort_order` - -Valid `sort_by` values: - -- `name` -- `bytes` -- `updated_time` -- `records` - -Valid `sort_order` values: - -- `asc` -- `desc` - -Example: - -```http -GET /index/v1/metrics/storage/children?organization=cbds&project=end_to_end_test&path=data&sort_by=bytes&sort_order=desc -``` - -Response shape: - -```json -{ - "organization": "cbds", - "project": "end_to_end_test", - "path": "data", - "items": [ - { - "name": "nested", - "path": "data/nested", - "type": "directory", - "file_count": 1, - "record_count": 1, - "total_bytes": 30, - "latest_update_time": "2026-06-24T22:00:00Z" - }, - { - "name": "a.txt", - "path": "data/a.txt", - "type": "file", - "file_count": 1, - "record_count": 2, - "total_bytes": 30, - "latest_update_time": "2026-06-24T22:00:00Z" - } - ] -} -``` - -Field meanings: - -- `name`: direct child name only -- `path`: normalized full child path -- `type`: `file` or `directory` -- `file_count`: distinct normalized file paths represented by that child -- `record_count`: total records represented by that child -- `total_bytes`: summed `drs_object.size` -- `latest_update_time`: max update time across that child subtree - -## Path Semantics - -These endpoints intentionally follow Syfon’s existing browse-index path logic. - -Important rules: - -- path normalization strips duplicate `/` -- backslashes are normalized to `/` -- empty path means root -- `.` and `..` are rejected -- path aggregation is driven by the existing browse index - -So the frontend should treat returned `path` values as canonical and should not -try to re-normalize them differently. - -## Duplicate Records - -The backend does not pretend duplicate-path records are one file. - -If multiple indexed objects resolve to the same normalized path: - -- all contribute to `record_count` -- all contribute to `total_bytes` -- `file_count` still counts the distinct path once -- duplicate occupancy is surfaced through `duplicate_path_count` - -Frontend implication: - -- if `record_count > file_count`, there are duplicate indexed records inside that subtree -- if a child row has `record_count > file_count`, that child has duplicate path occupancy - -## Auth / Scope Requirements - -These routes are not global rollups. - -They currently require a concrete project scope: - -- `organization` must be set -- `project` must be set - -Do not call them for org-wide rollups yet. - -## Suggested Frontend Wiring - -For a file explorer/tree view: - -1. On entering a project root, call `/metrics/storage/summary` with no `path`. -2. Also call `/metrics/storage/children` with no `path`. -3. On clicking a directory, call both endpoints again with that directory `path`. -4. Use `summary.direct_child_count` for quick folder metadata if useful. -5. Use `children.items` to render rows without reconstructing the subtree client-side. - -For a summary card: - -- use `/metrics/storage/summary` -- render `file_count`, `record_count`, `total_bytes`, and optionally `duplicate_path_count` - -## Recommended UI Interpretation - -For a directory row: - -- primary size: `total_bytes` -- secondary count: `file_count` -- warning state if `record_count > file_count` - -For a file row: - -- size: `total_bytes` -- warning if `record_count > 1` - -## What This Does Not Do - -This is not a precomputed materialized quota table. - -It is still query-time aggregation, but it is much cheaper than loading every -project record into the frontend and aggregating there. - -This also does not add: - -- org-wide grouped rollups -- deletion/remediation workflows -- duplicate cleanup - -## Backend Files Touched - -Useful reference points if the other agent wants to inspect behavior: - -- `apigen/openapi/metrics.openapi.yaml` -- `internal/api/metrics/storage_summary.go` -- `internal/storagemetrics/storage_metrics.go` -- `internal/db/sqlite/metrics.go` -- `internal/db/postgres/metrics.go` -- `client/services/metrics_service.go` - -## Short Pasteable Summary - -Use Syfon’s new endpoints instead of recursively traversing `/index`: - -- `GET /index/v1/metrics/storage/summary?organization=ORG&project=PROJ&path=OPTIONAL` -- `GET /index/v1/metrics/storage/children?organization=ORG&project=PROJ&path=OPTIONAL&sort_by=name|bytes|updated_time|records&sort_order=asc|desc&limit=N&offset=N` - -These routes reuse the same normalized browse-index path semantics as existing -path browsing. `total_bytes` is summed from `drs_object.size`. `record_count` -counts all matching indexed records. `file_count` counts distinct normalized -paths. Duplicate path occupancy shows up when `record_count > file_count`, and -the summary also exposes `duplicate_path_count`. diff --git a/docs/implementation-plan-storage-summary.md b/docs/implementation-plan-storage-summary.md deleted file mode 100644 index 7a41fd9b..00000000 --- a/docs/implementation-plan-storage-summary.md +++ /dev/null @@ -1,414 +0,0 @@ -# Implementation Plan: Path-Level Storage Summary Metrics - -## Goal - -Add native Syfon metrics endpoints for storage summaries so clients can ask for: - -- directory size under `{organization, project, path}` -- direct-child breakdown for breadcrumb/tree navigation -- project-level totals without traversing every `/index` page -- organization/project-scoped storage metrics with documented semantics - -This plan replaces the current frontend workaround that recursively walks -`/index` and computes aggregates client-side. - -## Why This Is Needed - -Current metrics support in Syfon is real, but it is focused on: - -- per-object file usage counts -- transfer attribution and provider-observed transfer events -- scoped usage summaries by object - -It does **not** currently provide: - -- path-level storage rollups -- directory summaries by prefix -- direct-child storage listings for a tree UI -- a metrics model that returns bytes grouped by canonical path - -That gap forces the frontend to re-enumerate the object index and rebuild a -directory tree on every request, which is the wrong ownership boundary. - -## Scope - -This plan covers new metrics read APIs and the storage-side aggregation needed -to power them. - -This plan does **not** include: - -- bucket/object deletion workflows -- automatic duplicate-object remediation -- quota enforcement -- billing reconciliation changes beyond reusing existing file size/object scope data - -## Required Product Behaviors - -The resulting API should support these use cases efficiently: - -1. Project summary card: - - total files - - total bytes - - total distinct direct children at the current path - -2. Breadcrumb/tree view: - - list direct children under a path - - show each child type (`file` or `directory`) - - show aggregate bytes for each child - - show file coverage count for each child - - show latest update timestamp for each child - -3. Organization/project summary views: - - project totals without full `/index` traversal - - optional grouped rollups for organization-wide reporting later - -## Canonical Data Semantics - -These must be fixed before implementation to avoid frontend/backend drift. - -### Path identity - -Use a single canonical path field for storage-tree aggregation. - -Recommended rule: - -- canonical path = normalized `file_name` when present -- fallback = normalized `name` only when `file_name` is empty -- if neither is a usable path, exclude from path metrics and surface in logs/diagnostics - -Rationale: - -- `file_name` is already the closest thing to path-like user-facing placement -- `name` is too ambiguous to be the primary tree key -- access URLs should not define logical browse paths - -### Size semantics - -Use the indexed object size already stored in Syfon metadata. - -- `total_bytes` = sum of `drs_object.size` for objects included in the filtered scope -- do not call cloud storage to compute live bucket bytes for interactive reads - -Rationale: - -- consistent with existing Syfon object metadata -- cheap to query -- avoids coupling UI latency to provider APIs - -### Object/version semantics - -Count every indexed object that is currently visible in the scope/path query. - -That means: - -- if two distinct objects resolve to the same canonical path, both contribute -- duplicate-path objects should be visible as `record_count > 1` -- deletion/supersession semantics remain whatever the object index currently exposes - -Rationale: - -- this is a storage summary of Syfon’s indexed state, not an inferred “clean file tree” -- remediation decisions should happen after the API can prove duplicate path occupancy - -## Proposed API - -Add a new storage-summary family under the metrics namespace. - -### 1. Path summary - -`GET /index/v1/metrics/storage/summary` - -Query params: - -- `organization` required -- `project` required -- `path` optional, default root - -Response shape: - -```json -{ - "organization": "Ellrott_Lab", - "project": "embedding_rotation", - "path": "data", - "file_count": 6, - "record_count": 6, - "total_bytes": 50497716224, - "direct_child_count": 6, - "duplicate_path_count": 3, - "latest_update_time": "2026-05-27T22:00:00Z" -} -``` - -### 2. Path children - -`GET /index/v1/metrics/storage/children` - -Query params: - -- `organization` required -- `project` required -- `path` optional, default root -- `limit` optional -- `offset` optional -- `sort_by` optional: `bytes`, `name`, `updated_time`, `records` -- `sort_order` optional: `asc`, `desc` - -Response shape: - -```json -{ - "organization": "Ellrott_Lab", - "project": "embedding_rotation", - "path": "data", - "items": [ - { - "name": "tcga_HM450.hdf5", - "path": "data/tcga_HM450.hdf5", - "type": "file", - "file_count": 1, - "record_count": 2, - "total_bytes": 39943195852, - "latest_update_time": "2026-05-27T22:00:00Z" - }, - { - "name": "nested", - "path": "data/nested", - "type": "directory", - "file_count": 42, - "record_count": 42, - "total_bytes": 90123123, - "latest_update_time": "2026-05-20T18:14:00Z" - } - ] -} -``` - -### 3. Optional later: grouped project rollup - -`GET /index/v1/metrics/storage/projects` - -This is useful later for explorer summary tables, but it is not necessary to -replace the FileSummary workaround. Keep it out of phase 1 unless it is nearly -free once the query layer exists. - -## Backend Design - -### New model types - -Add new metrics models in `internal/models`, for example: - -- `StoragePathSummary` -- `StoragePathChild` -- `StoragePathChildrenResponse` - -Fields should include: - -- scope: `organization`, `project`, `path` -- counts: `file_count`, `record_count`, `direct_child_count`, `duplicate_path_count` -- bytes: `total_bytes` -- timestamps: `latest_update_time` - -### New DB interface - -Extend `internal/db/interface.go` with a dedicated storage metrics contract. - -Suggested interface: - -```go -type StorageMetricsStore interface { - GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) - ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) -} -``` - -This should remain separate from `FileUsageScopedLister`. - -Reason: - -- file usage metrics answer “how often was this object used?” -- storage summary metrics answer “how much indexed data is under this logical path?” - -Those are different dimensions and should not be conflated. - -### Query strategy - -Do the aggregation in SQL against indexed metadata, not by hydrating every -object through `ObjectManager`. - -The query layer needs to: - -1. resolve visible objects in `{organization, project}` -2. normalize canonical browse path -3. filter to the requested prefix -4. compute: - - direct child name - - whether the child is a file or directory - - aggregate bytes - - file count - - record count - - latest update time - -### Postgres / SQLite parity - -Implement in both: - -- `internal/db/postgres/metrics.go` -- `internal/db/sqlite/metrics.go` - -Keep one behavioral contract and verify with matching tests. - -## Normalization Rules - -Path normalization should be shared and explicit. - -Recommended helper package behavior: - -- trim whitespace -- strip leading `/` -- collapse repeated `/` -- treat empty path as root -- direct child of root: - - `foo.txt` -> file child `foo.txt` - - `dir/a.txt` -> directory child `dir` - -If a record has canonical path equal to the requested path exactly, treat it as -a file child only when querying the parent directory. Do not let a path -summarize itself as its own descendant. - -## Authorization - -Use the same auth pattern as the existing metrics server: - -- require auth under Gen3 mode -- allow scoped reads only for readable resources -- require exact `organization/project` scope for path storage endpoints in phase 1 - -Do **not** implement organization-wide aggregation across arbitrary readable -scopes in the first pass. The storage monitor page already works at exact -project scope, and that keeps the SQL simpler. - -## Phased Rollout - -### Phase 1: backend path summary for exact project scope - -Deliver: - -- OpenAPI additions for: - - `GET /index/v1/metrics/storage/summary` - - `GET /index/v1/metrics/storage/children` -- generated metrics bindings via `make gen` -- new model types -- DB interface additions -- Postgres + SQLite implementations -- metrics server handlers -- unit/integration tests - -Success criterion: - -- the frontend can replace recursive `/index` traversal for a single project/path - -### Phase 2: frontend migration - -Deliver: - -- replace `FileSummary` recursive fetch logic with metrics calls -- preserve duplicate-path visibility with `record_count` -- keep `/index` only for drill-down features that need raw record identity - -Success criterion: - -- `FileSummary` no longer walks every record to compute path totals - -### Phase 3: broader rollups and diagnostics - -Optional additions: - -- organization-wide grouped project totals -- duplicate-path diagnostic endpoint -- “top N duplicate paths” for cleanup/admin views - -Success criterion: - -- explorer/admin UIs can build storage summaries without re-aggregating client-side - -## Testing Plan - -Add tests at three layers. - -### DB tests - -For both Postgres and SQLite: - -- root path summary -- nested path summary -- direct-child breakdown -- duplicate canonical path aggregation -- mixed file/directory children -- empty path result - -### API tests - -Under `internal/api/metrics`: - -- auth required / forbidden scope -- bad query validation -- summary response shape -- children response shape -- pagination and sorting - -### Server parity tests - -Update endpoint registration tests to include the new routes. - -## Risks - -### Path ambiguity - -If `file_name` is missing or inconsistent, tree metrics will be noisy. - -Mitigation: - -- define canonical path rules once -- add tests covering `file_name` vs `name` -- optionally add a diagnostic counter for records excluded from path metrics - -### Duplicate-path confusion - -Aggregating by path can hide the fact that multiple objects occupy one path. - -Mitigation: - -- return both `file_count` and `record_count` -- return `duplicate_path_count` in summaries - -### SQL complexity drift - -It is easy to accidentally reimplement the object manager in SQL. - -Mitigation: - -- keep phase 1 scoped to exact `{organization, project, path}` -- avoid organization-wide multi-scope fanout until the project-path query is stable - -## Concrete Implementation Order - -1. Add new OpenAPI paths and schemas to `apigen/openapi/metrics.openapi.yaml`. -2. Run `make gen`. -3. Add `StoragePathSummary` and `StoragePathChild` models. -4. Extend `internal/db/interface.go` with a storage metrics interface. -5. Implement SQLite queries first in `internal/db/sqlite/metrics.go`. -6. Mirror the behavior in `internal/db/postgres/metrics.go`. -7. Add metrics handlers in `internal/api/metrics`. -8. Add tests for handlers and DB parity. -9. Switch the frontend `FileSummary` page to use the new endpoints. - -## Immediate Recommendation - -Implement phase 1 only on this branch. - -That is the smallest meaningful delivery because it: - -- removes the frontend recursive `/index` workaround -- establishes the right metrics ownership boundary in Syfon -- preserves future room for duplicate-path diagnostics without coupling them to cleanup workflows diff --git a/docs/operator-guide-migration-indexd.md b/docs/operator-guide-migration-indexd.md deleted file mode 100644 index 4f239501..00000000 --- a/docs/operator-guide-migration-indexd.md +++ /dev/null @@ -1,83 +0,0 @@ -# Operator Guide: Indexd to Syfon Migration - -`syfon migrate` is a two-step offline workflow: - -1. `syfon migrate export` reads records from a source Gen3-mounted Indexd API and writes a local SQLite dump. -2. `syfon migrate import` reads that dump and loads records into a target Syfon instance through Syfon's `controlled_access`-aware `/index/bulk` compatibility loader. - -No migration-specific Syfon server endpoint is required. - -## Export - -```bash -syfon migrate export \ - --server "https://source-gen3.example.org" \ - --source-profile source \ - --batch-size 500 -``` - -By default, export writes `./indexd-records.sqlite`. Use `--dump` only when you want a different output path. - -The exporter uses Indexd's DID cursor (`start=`) rather than page offsets and de-duplicates records by DID. A single sweep is sufficient for a quiet Indexd database; increase `--sweeps` only if the source is being modified during export or the deployment is known to return inconsistent pages. - -Export reads Indexd records from `/index/index`, so for `--server https://calypr-dev.ohsu.edu` it queries `https://calypr-dev.ohsu.edu/index/index`. - -Useful export flags: - -- `--limit 1000`: canary export with only the first N unique records. -- `--dry-run`: fetch, transform, and validate without writing the dump. -- `--default-controlled-access /programs/open`: apply `controlled_access` only when a source record has no `authz`. -- `--default-authz /programs/open`: deprecated alias for `--default-controlled-access`. -- `--source-token`: use a raw bearer token instead of a Gen3 profile. - -## Import - -```bash -syfon migrate import \ - --dump ./indexd-records.sqlite \ - --server "https://target-gen3-with-syfon.example.org" \ - --target-profile target \ - --batch-size 500 -``` - -The import command reads the SQLite dump in batches and posts to the existing `POST /index/bulk` endpoint using `controlled_access` plus `access_methods`. It does not recreate legacy per-path `auth` maps. - -For a local Syfon server protected by local basic auth, use: - -```bash -syfon migrate import \ - --server http://localhost:8080 \ - --dump ./indexd-records.sqlite \ - --target-basic-user drs-user \ - --target-basic-password drs-pass -``` - -## Validation - -After a canary or full run, spot-check a few records: - -```bash -curl -u drs-user:drs-pass "https://target-gen3-with-syfon.example.org/index?limit=10" | jq . -curl -s "https://target-gen3-with-syfon.example.org/index/" | jq . -curl -s -X POST "https://target-gen3-with-syfon.example.org/index/bulk/hashes" \ - -H "Content-Type: application/json" \ - -d '{"hashes":["sha256:"]}' | jq . -``` - -`GET /index` is paged. Use `limit` for the page size and `start=` for the next page; `page=` remains supported for compatibility but should not be used for large traversals. The server caps list pages at `10000` records, but interactive clients should usually stay near `1000-2000`. - -For exact project-scoped listings, the recommended query shape is a cursor walk over the controlled-access index: - -```sql -SELECT DISTINCT ca.object_id -FROM drs_object_controlled_access ca -WHERE ca.resource = $1 AND ca.object_id > $2 -ORDER BY ca.object_id -LIMIT $3 -``` - -The `DISTINCT` remains intentional because Syfon does not currently enforce uniqueness on `(resource, object_id)`. Postgres should still plan this as an index-driven scan on `drs_object_controlled_access(resource, object_id)` rather than a deep-offset walk. - -When validating a deployment, run `EXPLAIN (ANALYZE, BUFFERS)` on the scoped ID query and confirm later-page requests remain index-driven and do not regress toward large offset costs or fanout-heavy row counts. - -The migration preserves DIDs, hashes, URLs, file names, descriptions, versions, timestamps, access methods, and authz-derived `controlled_access` claims for normal Indexd object records. Deprecated Indexd fields such as `baseid`, `rev`, `acl`, `metadata`, `urls_metadata`, `form`, and `uploader` are intentionally not loaded into Syfon. diff --git a/docs/pr-summary-feature-optimize.md b/docs/pr-summary-feature-optimize.md index 8d663c42..ab205895 100644 --- a/docs/pr-summary-feature-optimize.md +++ b/docs/pr-summary-feature-optimize.md @@ -87,7 +87,6 @@ Several supporting docs in this branch explain the intended behavior around: - project hierarchy browsing - configurable `/index` page sizing -- org/project storage metrics ### 6) Client and CLI restructuring diff --git a/docs/problem-solution-org-project-storage-metrics.md b/docs/problem-solution-org-project-storage-metrics.md deleted file mode 100644 index fe62ac05..00000000 --- a/docs/problem-solution-org-project-storage-metrics.md +++ /dev/null @@ -1,103 +0,0 @@ -# Problem / Solution: Organization and Project Storage Metrics - -## Problem - -The frontend organization explorer wants compact summary statistics at the organization and project level, especially: - -- total file count per project -- total stored bytes per project -- organization-level rollups across accessible projects - -A frontend workaround that pages through `GET /index?organization=...&project=...` for every visible project and then sums record sizes in JavaScript is not sustainable. - -Problems with the frontend approach: - -- request volume grows linearly with the number of visible projects -- large projects require walking every page of the index listing just to compute a summary row -- page load time and UI responsiveness become coupled to full index traversal -- multiple clients would independently repeat the same aggregation work -- the summary is conceptually a metrics concern, not a file-listing concern - -## Desired Outcome - -Syfon should expose organization- and project-scoped aggregate metrics so clients can request summary stats directly without enumerating every record. - -At minimum, the metrics surface should support: - -- exact project summaries for `{organization, project}` -- organization rollups across all projects in that organization -- total file count -- total stored bytes - -## Proposed Direction - -Add org/project filters and aggregate responses to the Syfon metrics API. - -Possible shapes: - -1. Extend an existing metrics endpoint to accept `organization` and optional `project` filters. -2. Add a dedicated metrics endpoint for storage summaries, for example a summary route under the metrics namespace. -3. Support grouped responses so one request can return all project summaries for an organization. - -Example response shape: - -```json -{ - "organization": "aced", - "project": "evotypes", - "file_count": 12842, - "total_bytes": 9412381234 -} -``` - -Example grouped organization response: - -```json -{ - "organization": "aced", - "projects": [ - { - "project": "evotypes", - "file_count": 12842, - "total_bytes": 9412381234 - } - ], - "organization_totals": { - "file_count": 12842, - "total_bytes": 9412381234 - } -} -``` - -## Data Semantics - -Open questions that should be decided explicitly: - -- Whether `total_bytes` is the sum of indexed record sizes, object-store bytes, or another canonical storage measure. -- Whether deleted, superseded, or non-current versions should contribute to totals. -- Whether metrics should be computed from the index, persisted as pre-aggregated counters, or refreshed asynchronously. -- Whether authorization should filter the response to only scopes visible to the caller, or whether this endpoint is only used for already-authorized org/project scopes. - -## Why This Belongs in Metrics - -This is shared aggregate data with stable semantics and broad reuse potential. - -Potential consumers include: - -- organization/project explorer pages -- administrative dashboards -- quota and storage reporting -- operational monitoring - -A metrics endpoint keeps listing endpoints focused on record retrieval and keeps clients from rebuilding the same aggregation logic independently. - -## Acceptance Criteria - -- A client can request project-level totals without traversing every `/index` page. -- A client can request organization-level totals without traversing every project. -- Returned values have documented semantics for file count and bytes. -- The response is fast enough to support summary rows in interactive UIs. - -## Current Status - -Frontend aggregate-size display has been removed for now. The organization explorer should only render data that can be fetched efficiently until a Syfon metrics implementation exists. diff --git a/docs/problem-solution-project-hierarchy-index.md b/docs/problem-solution-project-hierarchy-index.md index c16908b6..7b4a368e 100644 --- a/docs/problem-solution-project-hierarchy-index.md +++ b/docs/problem-solution-project-hierarchy-index.md @@ -53,7 +53,7 @@ The hierarchy should be derived from authoritative object/bucket path informatio Open questions: -- Which field should be canonical for hierarchy derivation: bucket path, object key, `file_name`, or another normalized source? +- Which field should be canonical for hierarchy derivation: bucket path, object key, or another normalized source? - How should conflicts be handled when metadata names differ from object paths? - Should directories be materialized eagerly or derived lazily and cached? - How should renames or path changes be represented historically? diff --git a/internal/api/docs/swagger.go b/internal/api/docs/swagger.go index acc36332..986af9d0 100644 --- a/internal/api/docs/swagger.go +++ b/internal/api/docs/swagger.go @@ -104,82 +104,6 @@ func handleInternalOpenAPISpec(w http.ResponseWriter, r *http.Request) { } } -func findOpenAPISpecPath() (string, bool) { - candidates := []string{ - "apigen/openapi/openapi.yaml", - filepath.Join(filepath.Dir(os.Args[0]), "apigen", "openapi", "openapi.yaml"), - } - - if _, thisFile, _, ok := runtime.Caller(0); ok { - repoRoot := filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..", "..")) - candidates = append(candidates, filepath.Join(repoRoot, "apigen", "openapi", "openapi.yaml")) - } - - for _, path := range candidates { - if _, err := os.Stat(path); err == nil { - return path, true - } - } - return "", false -} - -func findLFSOpenAPISpecPath() (string, bool) { - candidates := []string{ - "apigen/openapi/lfs.openapi.yaml", - filepath.Join(filepath.Dir(os.Args[0]), "apigen", "openapi", "lfs.openapi.yaml"), - } - - if _, thisFile, _, ok := runtime.Caller(0); ok { - repoRoot := filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..", "..")) - candidates = append(candidates, filepath.Join(repoRoot, "apigen", "openapi", "lfs.openapi.yaml")) - } - - for _, path := range candidates { - if _, err := os.Stat(path); err == nil { - return path, true - } - } - return "", false -} - -func findCompatOpenAPISpecPath() (string, bool) { - candidates := []string{ - "apigen/openapi/compat.openapi.yaml", - filepath.Join(filepath.Dir(os.Args[0]), "apigen", "openapi", "compat.openapi.yaml"), - } - - if _, thisFile, _, ok := runtime.Caller(0); ok { - repoRoot := filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..", "..")) - candidates = append(candidates, filepath.Join(repoRoot, "apigen", "openapi", "compat.openapi.yaml")) - } - - for _, path := range candidates { - if _, err := os.Stat(path); err == nil { - return path, true - } - } - return "", false -} - -func findBucketOpenAPISpecPath() (string, bool) { - candidates := []string{ - "apigen/openapi/bucket.openapi.yaml", - filepath.Join(filepath.Dir(os.Args[0]), "apigen", "openapi", "bucket.openapi.yaml"), - } - - if _, thisFile, _, ok := runtime.Caller(0); ok { - repoRoot := filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..", "..")) - candidates = append(candidates, filepath.Join(repoRoot, "apigen", "openapi", "bucket.openapi.yaml")) - } - - for _, path := range candidates { - if _, err := os.Stat(path); err == nil { - return path, true - } - } - return "", false -} - func findNamedOpenAPISpecPath(fileName string) (string, bool) { candidates := []string{ filepath.Join("apigen", "openapi", fileName), @@ -237,18 +161,6 @@ func buildMergedOpenAPISpec() ([]byte, error) { return out, nil } -func loadSpecYAML(path string) (map[string]interface{}, error) { - raw, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var doc map[string]interface{} - if err := yaml.Unmarshal(raw, &doc); err != nil { - return nil, err - } - return doc, nil -} - func loadSpecYAMLByName(fileName string) (map[string]interface{}, error) { raw, err := loadSpecBytesByName(fileName) if err != nil { diff --git a/internal/api/internaldrs/bucket_helpers.go b/internal/api/internaldrs/bucket_helpers.go index 688ba24c..9d93e32f 100644 --- a/internal/api/internaldrs/bucket_helpers.go +++ b/internal/api/internaldrs/bucket_helpers.go @@ -46,16 +46,6 @@ func serviceResourceAllowed(ctx context.Context, resource, service string, metho return authz.HasAnyServiceMethodAccess(ctx, []string{resource}, service, methods...) } -func allowedBucketsForScopes(ctx context.Context, scopes []models.BucketScope, methods ...string) map[string]bool { - allowed := make(map[string]bool) - for _, scope := range scopes { - if bucketScopeAllowed(ctx, scope, methods...) { - allowed[scope.Bucket] = true - } - } - return allowed -} - func bucketsAllowedByNames(ctx context.Context, scopes []models.BucketScope, bucket string, methods ...string) bool { for _, scope := range scopes { if scope.Bucket != bucket { diff --git a/internal/api/internaldrs/bucket_policy_test.go b/internal/api/internaldrs/bucket_policy_test.go index 1db28fe7..94449f08 100644 --- a/internal/api/internaldrs/bucket_policy_test.go +++ b/internal/api/internaldrs/bucket_policy_test.go @@ -42,10 +42,6 @@ func TestBucketPolicyHelpers(t *testing.T) { resource: {"read": true}, }) - allowed := allowedBucketsForScopes(ctx, []models.BucketScope{scope}, "read") - if !allowed["bucket-a"] { - t.Fatal("expected bucket to be allowed") - } if !bucketsAllowedByNames(ctx, []models.BucketScope{scope}, "bucket-a", "read") { t.Fatal("expected bucket name match to be allowed") } diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index 20254def..6b995731 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -2,14 +2,11 @@ package internaldrs import ( "fmt" - "net/http" - "sort" "strconv" "strings" "time" "github.com/calypr/syfon/apigen/server/internalapi" - sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/api/apiutil" apimiddleware "github.com/calypr/syfon/internal/api/middleware" "github.com/calypr/syfon/internal/api/routeutil" @@ -41,9 +38,6 @@ func RegisterInternalRoutes(router fiber.Router, om *core.ObjectManager) { router.Post(common.RouteInternalBulkCreate, handleInternalBulkCreateFiber(om)) router.Post(common.RouteInternalBulkDocs, handleInternalBulkDocumentsFiber(om)) router.Post(common.RouteInternalBulkDeleteHashes, handleInternalBulkDeleteFiber(om)) - router.Post(common.RouteInternalRepairProjectDiff, handleInternalProjectDiffAuditFiber(om)) - router.Post(common.RouteInternalRepairCleanupAudit, handleInternalStorageCleanupAuditFiber(om)) - router.Post(common.RouteInternalRepairCleanupApply, handleInternalStorageCleanupApplyFiber(om)) router.Post(common.RouteInternalRepairScopeAudit, handleInternalScopeRepairAuditFiber(om)) router.Post(common.RouteInternalRepairScopeApply, handleInternalScopeRepairApplyFiber(om)) @@ -373,38 +367,6 @@ func parseScopeQueryParts(organization, program, project string) (string, string return "", "", false, nil } -func parseScopeQuery(r *http.Request) (string, string, bool, error) { - return parseScopeQueryParts( - r.URL.Query().Get("organization"), - r.URL.Query().Get("program"), - r.URL.Query().Get("project"), - ) -} - -func paginateInternalListIDsFiber(c fiber.Ctx, ids []string) ([]string, error) { - limit, start, offset, err := parseInternalListPaginationFiber(c) - if err != nil { - return nil, err - } - if limit == 0 || len(ids) == 0 { - return []string{}, nil - } - if start != "" { - offset = sort.SearchStrings(ids, start) - for offset < len(ids) && ids[offset] <= start { - offset++ - } - } - if offset >= len(ids) { - return []string{}, nil - } - end := offset + limit - if end > len(ids) { - end = len(ids) - } - return ids[offset:end], nil -} - func parseInternalListPaginationFiber(c fiber.Ctx) (int, string, int, error) { limit := defaultInternalListLimit rawLimit := strings.TrimSpace(c.Query("limit")) @@ -488,8 +450,3 @@ func normalizeNonEmptyBulkHashes(hashes []string) []string { } return normalized } - -func objectAuthzMatchesScope(obj models.InternalObject, org, project string) bool { - authzMap := sycommon.ControlledAccessToAuthzMap(core.ObjectAccessResources(&obj)) - return sycommon.AuthzMapMatchesScope(authzMap, org, project) -} diff --git a/internal/api/internaldrs/index_test.go b/internal/api/internaldrs/index_test.go index 3e20a024..a77c9146 100644 --- a/internal/api/internaldrs/index_test.go +++ b/internal/api/internaldrs/index_test.go @@ -30,30 +30,6 @@ func indexTestAuthContext(base context.Context, mode string, authHeader bool, pr return internalauth.WithSession(base, session) } -func TestParseScopeQuery(t *testing.T) { - t.Run("organization and project", func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/?organization=prog&project=study", nil) - org, project, ok, err := parseScopeQuery(req) - if err != nil { - t.Fatalf("parseScopeQuery returned error: %v", err) - } - if !ok || org != "prog" || project != "study" { - t.Fatalf("unexpected scope parse result: ok=%v org=%q project=%q", ok, org, project) - } - }) - - t.Run("project without organization fails", func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/?project=study", nil) - _, _, ok, err := parseScopeQuery(req) - if err == nil { - t.Fatal("expected validation error") - } - if ok { - t.Fatal("expected hasScope=false on validation error") - } - }) -} - func TestHandleInternalList_ScopeFilteringByReadPrivilege(t *testing.T) { now := time.Now().UTC() mockDB := &testutils.MockDatabase{ @@ -267,31 +243,6 @@ func TestHandleInternalList_PagePaginatesIDs(t *testing.T) { } } -func TestPaginateInternalListIDsFiberDefaultLimit(t *testing.T) { - ids := make([]string, defaultInternalListLimit+5) - for i := range ids { - ids[i] = "obj" - } - app := fiber.New() - app.Get("/", func(c fiber.Ctx) error { - paged, err := paginateInternalListIDsFiber(c, ids) - if err != nil { - return err - } - if len(paged) != defaultInternalListLimit { - t.Fatalf("expected default limit %d, got %d", defaultInternalListLimit, len(paged)) - } - return c.SendStatus(fiber.StatusOK) - }) - resp, err := app.Test(httptest.NewRequest(http.MethodGet, "/", nil)) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } -} - func TestHandleInternalList_LimitIsCappedAtTenThousand(t *testing.T) { now := time.Now().UTC() total := maxInternalListLimit + 1 diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go index 6c8adbda..127dfd0d 100644 --- a/internal/api/internaldrs/repair_cleanup.go +++ b/internal/api/internaldrs/repair_cleanup.go @@ -22,87 +22,6 @@ import ( "github.com/gofiber/fiber/v3" ) -func handleInternalStorageCleanupAuditFiber(om *core.ObjectManager) fiber.Handler { - svc := repair.NewStorageCleanupService(om) - return func(c fiber.Ctx) error { - if apimiddleware.MissingGen3AuthHeader(c.Context()) { - return c.SendStatus(fiber.StatusUnauthorized) - } - var req repair.StorageCleanupAuditRequest - if err := decodeStrictJSON(c.Body(), &req); err != nil { - return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) - } - req.Organization = strings.TrimSpace(req.Organization) - req.Project = strings.TrimSpace(req.Project) - req.PathPrefix = strings.TrimSpace(req.PathPrefix) - if req.Organization == "" || req.Project == "" { - return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") - } - if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read"); err != nil { - return apiutil.HandleError(c, err) - } - report, err := svc.Audit(c.Context(), req) - if err != nil { - return apiutil.HandleError(c, err) - } - return c.JSON(report) - } -} - -func handleInternalProjectDiffAuditFiber(om *core.ObjectManager) fiber.Handler { - svc := repair.NewProjectDiffService(om) - return func(c fiber.Ctx) error { - if apimiddleware.MissingGen3AuthHeader(c.Context()) { - return c.SendStatus(fiber.StatusUnauthorized) - } - var req repair.ProjectDiffAuditRequest - if err := decodeStrictJSON(c.Body(), &req); err != nil { - return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) - } - req.Organization = strings.TrimSpace(req.Organization) - req.Project = strings.TrimSpace(req.Project) - req.PathPrefix = strings.TrimSpace(req.PathPrefix) - if req.Organization == "" || req.Project == "" { - return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") - } - if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read"); err != nil { - return apiutil.HandleError(c, err) - } - report, err := svc.Audit(c.Context(), req) - if err != nil { - return apiutil.HandleError(c, err) - } - return c.JSON(report) - } -} - -func handleInternalStorageCleanupApplyFiber(om *core.ObjectManager) fiber.Handler { - svc := repair.NewStorageCleanupService(om) - return func(c fiber.Ctx) error { - if apimiddleware.MissingGen3AuthHeader(c.Context()) { - return c.SendStatus(fiber.StatusUnauthorized) - } - var req repair.StorageCleanupApplyRequest - if err := decodeStrictJSON(c.Body(), &req); err != nil { - return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) - } - req.Organization = strings.TrimSpace(req.Organization) - req.Project = strings.TrimSpace(req.Project) - req.PathPrefix = strings.TrimSpace(req.PathPrefix) - if req.Organization == "" || req.Project == "" { - return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") - } - if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read", "delete"); err != nil { - return apiutil.HandleError(c, err) - } - result, err := svc.Apply(c.Context(), req) - if err != nil { - return apiutil.HandleError(c, err) - } - return c.JSON(result) - } -} - func authorizeStorageCleanupScope(ctx context.Context, organization, project string, methods ...string) error { if !authz.IsAuthzEnforced(ctx) { return nil diff --git a/internal/api/lfs/lfs.go b/internal/api/lfs/lfs.go index 9a7a69ea..897ba128 100644 --- a/internal/api/lfs/lfs.go +++ b/internal/api/lfs/lfs.go @@ -55,10 +55,3 @@ func RegisterLFSRoutes(router fiber.Router, om *core.ObjectManager, opts ...Opti }) lfsapi.RegisterHandlers(router, strict) } - -func ResetLFSLimitersForTest() { - limitMu.Lock() - defer limitMu.Unlock() - requestWindowMap = map[string]windowCounter{} - bandwidthWindowMap = map[string]windowBytes{} -} diff --git a/internal/api/lfs/reset_test.go b/internal/api/lfs/reset_test.go new file mode 100644 index 00000000..c3e0c5d5 --- /dev/null +++ b/internal/api/lfs/reset_test.go @@ -0,0 +1,8 @@ +package lfs + +func ResetLFSLimitersForTest() { + limitMu.Lock() + defer limitMu.Unlock() + requestWindowMap = map[string]windowCounter{} + bandwidthWindowMap = map[string]windowBytes{} +} diff --git a/internal/api/metrics/transfer_ingest.go b/internal/api/metrics/transfer_ingest.go index 55e13edc..77d2d324 100644 --- a/internal/api/metrics/transfer_ingest.go +++ b/internal/api/metrics/transfer_ingest.go @@ -9,45 +9,12 @@ import ( "github.com/calypr/syfon/apigen/server/metricsapi" sycommon "github.com/calypr/syfon/common" - "github.com/calypr/syfon/internal/api/attribution" apimiddleware "github.com/calypr/syfon/internal/api/middleware" - internalauth "github.com/calypr/syfon/internal/auth" "github.com/calypr/syfon/internal/authz" intcommon "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" ) -type transferEventsRequest struct { - Events []transferEventPayload `json:"events"` -} - -type transferEventPayload struct { - EventID string `json:"event_id"` - EventType string `json:"event_type"` - Direction string `json:"direction"` - EventTime string `json:"event_time"` - RequestID string `json:"request_id"` - ObjectID string `json:"object_id"` - SHA256 string `json:"sha256"` - ObjectSize int64 `json:"object_size"` - Organization string `json:"organization"` - Project string `json:"project"` - AccessID string `json:"access_id"` - Provider string `json:"provider"` - Bucket string `json:"bucket"` - StorageURL string `json:"storage_url"` - RangeStart *int64 `json:"range_start"` - RangeEnd *int64 `json:"range_end"` - BytesRequested int64 `json:"bytes_requested"` - BytesCompleted int64 `json:"bytes_completed"` - ActorEmail string `json:"actor_email"` - ActorSubject string `json:"actor_subject"` - AuthMode string `json:"auth_mode"` - ClientName string `json:"client_name"` - ClientVersion string `json:"client_version"` - TransferSessionID string `json:"transfer_session_id"` -} - type providerTransferPayload struct { ProviderEventID string `json:"provider_event_id"` AccessGrantID string `json:"access_grant_id"` @@ -133,83 +100,6 @@ func providerTransferResource(organization, project string) (string, bool) { return resource, true } -func transferPayloadToModel(ctx context.Context, item transferEventPayload) (models.TransferAttributionEvent, error) { - eventType := strings.TrimSpace(item.EventType) - switch eventType { - case models.TransferEventAccessIssued: - default: - return models.TransferAttributionEvent{}, errors.New("invalid event_type") - } - when := time.Now().UTC() - if strings.TrimSpace(item.EventTime) != "" { - parsed, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(item.EventTime)) - if err != nil { - return models.TransferAttributionEvent{}, errors.New("invalid event_time") - } - when = parsed.UTC() - } - if item.BytesCompleted < 0 || item.BytesRequested < 0 { - return models.TransferAttributionEvent{}, errors.New("bytes cannot be negative") - } - if eventType != models.TransferEventAccessIssued && item.BytesCompleted <= 0 { - return models.TransferAttributionEvent{}, errors.New("bytes_completed is required") - } - ev := models.TransferAttributionEvent{ - EventID: strings.TrimSpace(item.EventID), - EventType: eventType, - Direction: normalizeTransferDirection(item.Direction), - EventTime: when, - RequestID: strings.TrimSpace(item.RequestID), - ObjectID: strings.TrimSpace(item.ObjectID), - SHA256: strings.TrimSpace(item.SHA256), - ObjectSize: item.ObjectSize, - Organization: strings.TrimSpace(item.Organization), - Project: strings.TrimSpace(item.Project), - AccessID: strings.TrimSpace(item.AccessID), - Provider: strings.TrimSpace(item.Provider), - Bucket: strings.TrimSpace(item.Bucket), - StorageURL: strings.TrimSpace(item.StorageURL), - RangeStart: item.RangeStart, - RangeEnd: item.RangeEnd, - BytesRequested: item.BytesRequested, - BytesCompleted: item.BytesCompleted, - ActorEmail: strings.TrimSpace(item.ActorEmail), - ActorSubject: strings.TrimSpace(item.ActorSubject), - AuthMode: strings.TrimSpace(item.AuthMode), - ClientName: strings.TrimSpace(item.ClientName), - ClientVersion: strings.TrimSpace(item.ClientVersion), - TransferSessionID: strings.TrimSpace(item.TransferSessionID), - } - if ev.RequestID == "" { - ev.RequestID = intcommon.GetRequestID(ctx) - } - if ev.ActorEmail == "" { - ev.ActorEmail = attribution.ActorEmail(ctx) - } - if ev.ActorSubject == "" { - ev.ActorSubject = attribution.ActorSubject(ctx) - } - if ev.AuthMode == "" { - ev.AuthMode = strings.TrimSpace(internalauth.FromContext(ctx).Mode) - } - if ev.EventID == "" { - ev.EventID = attribution.EventID(ev) - } - if ev.AccessGrantID == "" { - ev.AccessGrantID = attribution.AccessGrantID(ev) - } - return ev, nil -} - -func normalizeTransferDirection(direction string) string { - switch strings.ToLower(strings.TrimSpace(direction)) { - case models.ProviderTransferDirectionUpload: - return models.ProviderTransferDirectionUpload - default: - return models.ProviderTransferDirectionDownload - } -} - func providerTransferPayloadToModel(item providerTransferPayload) (models.ProviderTransferEvent, error) { direction := strings.ToLower(strings.TrimSpace(item.Direction)) switch direction { diff --git a/internal/api/metrics/transfer_ingest_test.go b/internal/api/metrics/transfer_ingest_test.go deleted file mode 100644 index f2d03a0b..00000000 --- a/internal/api/metrics/transfer_ingest_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package metrics - -import ( - "context" - "net/http" - "testing" - "time" - - "github.com/calypr/syfon/apigen/server/metricsapi" - internalauth "github.com/calypr/syfon/internal/auth" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" -) - -func ptr[T any](v T) *T { return &v } - -func TestNormalizeTransferDirection(t *testing.T) { - if got := normalizeTransferDirection("upload"); got != models.ProviderTransferDirectionUpload { - t.Fatalf("expected upload, got %q", got) - } - if got := normalizeTransferDirection("something-else"); got != models.ProviderTransferDirectionDownload { - t.Fatalf("expected default download, got %q", got) - } -} - -func TestTransferPayloadToModel(t *testing.T) { - ctx := common.WithRequestID(context.Background(), "req-1") - session := internalauth.NewSession("gen3") - session.SetSubject("user-sub") - ctx = internalauth.WithSession(ctx, session) - - payload := transferEventPayload{ - EventType: models.TransferEventAccessIssued, - Direction: "download", - EventTime: time.Now().UTC().Format(time.RFC3339Nano), - ObjectID: "did-1", - BytesRequested: 10, - } - got, err := transferPayloadToModel(ctx, payload) - if err != nil { - t.Fatalf("transferPayloadToModel error: %v", err) - } - if got.EventType != models.TransferEventAccessIssued || got.Direction != models.ProviderTransferDirectionDownload { - t.Fatalf("unexpected mapped event: %+v", got) - } - if got.RequestID != "req-1" { - t.Fatalf("expected request id fallback, got %q", got.RequestID) - } - if got.EventID == "" || got.AccessGrantID == "" { - t.Fatalf("expected generated ids, got event=%q grant=%q", got.EventID, got.AccessGrantID) - } - - payload.BytesRequested = -1 - if _, err := transferPayloadToModel(ctx, payload); err == nil { - t.Fatalf("expected negative-bytes validation error") - } -} - -func TestCheckProviderMetricsIngestAuth(t *testing.T) { - ctxNonGen3 := metricsTestContext(context.Background(), "local", true, true, nil) - body := &metricsapi.RecordProviderTransferEventsJSONRequestBody{ - Events: []metricsapi.ProviderTransferEvent{{ - Organization: ptr("org-a"), - Project: ptr("proj-a"), - }}, - } - if status, ok := checkProviderMetricsIngestAuth(ctxNonGen3, body); !ok || status != 0 { - t.Fatalf("expected non-gen3 auth allowed, got status=%d ok=%v", status, ok) - } - - ctxMissingHeader := metricsTestContext(context.Background(), "gen3", false, false, nil) - if status, ok := checkProviderMetricsIngestAuth(ctxMissingHeader, body); ok || status != http.StatusUnauthorized { - t.Fatalf("expected unauthorized for missing header, got status=%d ok=%v", status, ok) - } - - ctxForbidden := metricsTestContext(context.Background(), "gen3", true, true, map[string]map[string]bool{}) - if status, ok := checkProviderMetricsIngestAuth(ctxForbidden, body); ok || status != http.StatusForbidden { - t.Fatalf("expected forbidden without ingest privilege, got status=%d ok=%v", status, ok) - } - - ctxAllowed := metricsTestContext(context.Background(), "gen3", true, true, map[string]map[string]bool{ - "/programs/org-a/projects/proj-a": {"create": true}, - }) - if status, ok := checkProviderMetricsIngestAuth(ctxAllowed, body); !ok || status != 0 { - t.Fatalf("expected allowed with scoped create privilege, got status=%d ok=%v", status, ok) - } - - ctxOrgAllowed := metricsTestContext(context.Background(), "gen3", true, true, map[string]map[string]bool{ - "/programs/org-a": {"update": true}, - }) - orgBody := &metricsapi.RecordProviderTransferEventsJSONRequestBody{ - Events: []metricsapi.ProviderTransferEvent{{ - Organization: ptr("org-a"), - }}, - } - if status, ok := checkProviderMetricsIngestAuth(ctxOrgAllowed, orgBody); !ok || status != 0 { - t.Fatalf("expected allowed with org-scoped update privilege, got status=%d ok=%v", status, ok) - } -} - -func TestRecordProviderTransferEventsAuthResponse(t *testing.T) { - if _, ok := recordProviderTransferEventsAuthResponse(http.StatusUnauthorized).(metricsapi.RecordProviderTransferEvents401Response); !ok { - t.Fatalf("expected 401 response object") - } - if _, ok := recordProviderTransferEventsAuthResponse(http.StatusForbidden).(metricsapi.RecordProviderTransferEvents403Response); !ok { - t.Fatalf("expected 403 response object") - } - if _, ok := recordProviderTransferEventsAuthResponse(http.StatusBadRequest).(metricsapi.RecordProviderTransferEvents400Response); !ok { - t.Fatalf("expected 400 response object") - } -} diff --git a/internal/api/routeutil/routeutil.go b/internal/api/routeutil/routeutil.go index 02620cba..a85836fc 100644 --- a/internal/api/routeutil/routeutil.go +++ b/internal/api/routeutil/routeutil.go @@ -4,7 +4,6 @@ import ( "context" "net/http" "regexp" - "strings" "github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3/middleware/adaptor" @@ -31,18 +30,6 @@ func WithPathParams(r *http.Request, params map[string]string) *http.Request { return r.WithContext(context.WithValue(r.Context(), pathParamsKey{}, copyParams)) } -// PathParam reads a path parameter that was attached with WithPathParams. -func PathParam(r *http.Request, key string) string { - if r == nil { - return "" - } - params, ok := r.Context().Value(pathParamsKey{}).(map[string]string) - if !ok { - return "" - } - return strings.TrimSpace(params[key]) -} - // Handler adapts a standard net/http handler to fiber while preserving path params. func Handler(h any, paramNames ...string) fiber.Handler { return func(c fiber.Ctx) error { diff --git a/internal/common/browse_path.go b/internal/common/browse_path.go deleted file mode 100644 index 12b5e100..00000000 --- a/internal/common/browse_path.go +++ /dev/null @@ -1,96 +0,0 @@ -package common - -import ( - "fmt" - "strings" -) - -type BrowsePathInfo struct { - Normalized string - ParentPath string - EntryName string - Segments []string -} - -func NormalizeBrowsePath(raw string) (string, []string, error) { - trimmed := strings.TrimSpace(raw) - if trimmed == "" { - return "", nil, nil - } - trimmed = strings.ReplaceAll(trimmed, "\\", "/") - parts := strings.Split(trimmed, "/") - segments := make([]string, 0, len(parts)) - for _, part := range parts { - segment := strings.TrimSpace(part) - if segment == "" { - continue - } - if segment == "." || segment == ".." { - return "", nil, fmt.Errorf("path must not contain '.' or '..' segments") - } - segments = append(segments, segment) - } - return strings.Join(segments, "/"), segments, nil -} - -func BrowsePathInfoFromName(raw string) (BrowsePathInfo, bool, error) { - normalized, segments, err := NormalizeBrowsePath(raw) - if err != nil { - return BrowsePathInfo{}, false, err - } - if normalized == "" || len(segments) == 0 { - return BrowsePathInfo{}, false, nil - } - parentPath := "" - if len(segments) > 1 { - parentPath = strings.Join(segments[:len(segments)-1], "/") - } - return BrowsePathInfo{ - Normalized: normalized, - ParentPath: parentPath, - EntryName: segments[len(segments)-1], - Segments: segments, - }, true, nil -} - -func HasBrowsePathPrefix(candidate, prefix []string) bool { - if len(prefix) > len(candidate) { - return false - } - for i, segment := range prefix { - if candidate[i] != segment { - return false - } - } - return true -} - -func ImmediateBrowseDirectory(requestedPath, candidatePath string) (BrowsePathInfo, bool) { - requestedNormalized, requestedSegments, err := NormalizeBrowsePath(requestedPath) - if err != nil { - return BrowsePathInfo{}, false - } - candidateInfo, ok, err := BrowsePathInfoFromName(candidatePath) - if err != nil || !ok { - return BrowsePathInfo{}, false - } - if !HasBrowsePathPrefix(candidateInfo.Segments, requestedSegments) { - return BrowsePathInfo{}, false - } - remaining := candidateInfo.Segments[len(requestedSegments):] - if len(remaining) < 2 { - return BrowsePathInfo{}, false - } - dirSegments := append(append([]string{}, requestedSegments...), remaining[0]) - dirPath := strings.Join(dirSegments, "/") - parentPath := "" - if requestedNormalized != "" { - parentPath = requestedNormalized - } - return BrowsePathInfo{ - Normalized: dirPath, - ParentPath: parentPath, - EntryName: remaining[0], - Segments: dirSegments, - }, true -} diff --git a/internal/common/browse_path_test.go b/internal/common/browse_path_test.go deleted file mode 100644 index 88512595..00000000 --- a/internal/common/browse_path_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package common - -import "testing" - -func TestBrowsePathInfoFromName(t *testing.T) { - info, ok, err := BrowsePathInfoFromName(` nested\\dir//file.txt `) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !ok { - t.Fatal("expected valid browse path info") - } - if info.Normalized != "nested/dir/file.txt" { - t.Fatalf("unexpected normalized path: %q", info.Normalized) - } - if info.ParentPath != "nested/dir" { - t.Fatalf("unexpected parent path: %q", info.ParentPath) - } - if info.EntryName != "file.txt" { - t.Fatalf("unexpected entry name: %q", info.EntryName) - } -} - -func TestNormalizeBrowsePathRejectsDotSegments(t *testing.T) { - if _, _, err := NormalizeBrowsePath("../nested"); err == nil { - t.Fatal("expected invalid path error") - } -} - -func TestImmediateBrowseDirectory(t *testing.T) { - dir, ok := ImmediateBrowseDirectory("nested", "nested/deep/file.txt") - if !ok { - t.Fatal("expected directory to be derived") - } - if dir.Normalized != "nested/deep" { - t.Fatalf("unexpected directory path: %q", dir.Normalized) - } - if dir.EntryName != "deep" { - t.Fatalf("unexpected directory name: %q", dir.EntryName) - } -} diff --git a/internal/common/routes.go b/internal/common/routes.go index a7c94788..8b875b5c 100644 --- a/internal/common/routes.go +++ b/internal/common/routes.go @@ -1,24 +1,21 @@ package common const ( - RouteInternalDownload = "/data/download/{file_id}" - RouteInternalDownloadPart = "/data/download/{file_id}/part" - RouteInternalUpload = "/data/upload" - RouteInternalUploadURL = "/data/upload/{file_id}" - RouteInternalUploadBulk = "/data/upload/bulk" - RouteInternalMultipartInit = "/data/multipart/init" - RouteInternalMultipartUpload = "/data/multipart/upload" - RouteInternalMultipartComplete = "/data/multipart/complete" - RouteInternalInspectObject = "/data/inspect" - RouteInternalBuckets = "/data/buckets" - RouteInternalBucketDetail = "/data/buckets/{bucket}" - RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" - RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" - RouteInternalRepairCleanupAudit = "/data/repair/storage-cleanup/audit" - RouteInternalRepairCleanupApply = "/data/repair/storage-cleanup/apply" - RouteInternalRepairProjectDiff = "/data/repair/project-diff/audit" - RouteInternalRepairScopeAudit = "/data/repair/project-scope/audit" - RouteInternalRepairScopeApply = "/data/repair/project-scope/apply" + RouteInternalDownload = "/data/download/{file_id}" + RouteInternalDownloadPart = "/data/download/{file_id}/part" + RouteInternalUpload = "/data/upload" + RouteInternalUploadURL = "/data/upload/{file_id}" + RouteInternalUploadBulk = "/data/upload/bulk" + RouteInternalMultipartInit = "/data/multipart/init" + RouteInternalMultipartUpload = "/data/multipart/upload" + RouteInternalMultipartComplete = "/data/multipart/complete" + RouteInternalInspectObject = "/data/inspect" + RouteInternalBuckets = "/data/buckets" + RouteInternalBucketDetail = "/data/buckets/{bucket}" + RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" + RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" + RouteInternalRepairScopeAudit = "/data/repair/project-scope/audit" + RouteInternalRepairScopeApply = "/data/repair/project-scope/apply" RouteInternalIndex = "/index" RouteInternalIndexDetail = "/index/{id}" diff --git a/internal/converters/converters.go b/internal/converters/converters.go index b9604c7d..907dc8b0 100644 --- a/internal/converters/converters.go +++ b/internal/converters/converters.go @@ -2,6 +2,7 @@ package converters import ( "fmt" + "path/filepath" "strings" "time" @@ -41,7 +42,7 @@ func CandidateToInternalObject(c drs.DrsObjectCandidate, now time.Time) (models. obj.ControlledAccess = &controlled } if c.Name != nil { - obj.Name = c.Name + obj.Name = normalizedObjectNamePtr(c.Name) } obj.SelfUri = "drs://" + obj.Id if obj.Name == nil || strings.TrimSpace(*obj.Name) == "" { @@ -91,6 +92,22 @@ func CandidateToInternalObject(c drs.DrsObjectCandidate, now time.Time) (models. }, nil } +func normalizedObjectNamePtr(name *string) *string { + if name == nil { + return nil + } + trimmed := strings.TrimSpace(*name) + if trimmed == "" { + return nil + } + trimmed = strings.ReplaceAll(trimmed, "\\", "/") + base := filepath.Base(trimmed) + if base == "." || base == "/" || base == "" { + base = trimmed + } + return common.Ptr(base) +} + func LFSCandidateToDRS(in lfsapi.DrsObjectCandidate) drs.DrsObjectCandidate { var size int64 if in.Size != nil { diff --git a/internal/core/converters.go b/internal/core/converters.go index ee7177cd..ec8dee23 100644 --- a/internal/core/converters.go +++ b/internal/core/converters.go @@ -2,6 +2,7 @@ package core import ( "fmt" + "path/filepath" "strings" "time" @@ -158,7 +159,7 @@ func CandidateToInternalObject(c drs.DrsObjectCandidate, now time.Time) (models. obj.ControlledAccess = &controlled } if c.Name != nil { - obj.Name = c.Name + obj.Name = normalizedObjectNamePtr(c.Name) } if obj.Name == nil || strings.TrimSpace(*obj.Name) == "" { obj.Name = &oid @@ -202,7 +203,7 @@ func MergeInternalObjectUpdate(existing models.InternalObject, update models.Int } if update.DrsObject.Name != nil { - merged.DrsObject.Name = update.DrsObject.Name + merged.DrsObject.Name = normalizedObjectNamePtr(update.DrsObject.Name) } if update.DrsObject.Description != nil { merged.DrsObject.Description = update.DrsObject.Description @@ -250,7 +251,7 @@ func InternalRecordToInternalObject(r internalapi.InternalRecord, now time.Time) updatedTime := parseInternalRecordTime(r.UpdatedTime, obj.CreatedTime) obj.UpdatedTime = &updatedTime if r.Name != nil && strings.TrimSpace(*r.Name) != "" { - obj.Name = common.Ptr(strings.TrimSpace(*r.Name)) + obj.Name = normalizedObjectNamePtr(r.Name) } if v := r.Version; v != nil { obj.Version = v @@ -282,6 +283,22 @@ func InternalRecordToInternalObject(r internalapi.InternalRecord, now time.Time) return EnforceCanonicalProjectScope(internalObj, common.StringVal(r.Organization), common.StringVal(r.Project)) } +func normalizedObjectNamePtr(name *string) *string { + if name == nil { + return nil + } + trimmed := strings.TrimSpace(*name) + if trimmed == "" { + return nil + } + trimmed = strings.ReplaceAll(trimmed, "\\", "/") + base := filepath.Base(trimmed) + if base == "." || base == "/" || base == "" { + base = trimmed + } + return common.Ptr(base) +} + func parseInternalRecordTime(raw *string, fallback time.Time) time.Time { if raw == nil || strings.TrimSpace(*raw) == "" { return fallback.UTC() diff --git a/internal/core/object_reads.go b/internal/core/object_reads.go index ee69086e..8e520e42 100644 --- a/internal/core/object_reads.go +++ b/internal/core/object_reads.go @@ -238,30 +238,6 @@ func (m *ObjectManager) ListObjectIDsPageByScope(ctx context.Context, organizati return ids[offset:end], nil } -func (m *ObjectManager) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, requiredMethod, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { - if limit <= 0 { - return []string{}, []models.BrowseDirectory{}, nil - } - if strings.TrimSpace(requiredMethod) == objectMethodRead { - if authz.IsGen3Mode(ctx) && authz.IsAuthzEnforced(ctx) && !authz.HasAuthHeader(ctx) { - return []string{}, []models.BrowseDirectory{}, nil - } - if authz.IsAuthzEnforced(ctx) && !m.canPageScopeRead(ctx, organization, project) { - return []string{}, []models.BrowseDirectory{}, nil - } - } - - if pager, ok := m.db.(db.ObjectPathPageLister); ok { - return pager.ListObjectIDsPageByPath(ctx, organization, project, path, startAfter, limit, offset) - } - - ids, err := m.ListObjectIDsPageByScope(ctx, organization, project, requiredMethod, startAfter, limit, offset) - if err != nil { - return nil, nil, err - } - return ids, []models.BrowseDirectory{}, nil -} - func (m *ObjectManager) ListObjectIDsPageByURL(ctx context.Context, objectURL, organization, project, requiredMethod, startAfter string, limit, offset int) ([]string, error) { if limit <= 0 { return []string{}, nil diff --git a/internal/core/storage_cleanup.go b/internal/core/storage_cleanup.go deleted file mode 100644 index a0ae3e71..00000000 --- a/internal/core/storage_cleanup.go +++ /dev/null @@ -1,49 +0,0 @@ -package core - -import ( - "context" - "fmt" - "strings" - - sycommon "github.com/calypr/syfon/common" - "github.com/calypr/syfon/internal/authz" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/db" - "github.com/calypr/syfon/internal/models" -) - -func (m *ObjectManager) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - return m.listStorageCleanupRecords(ctx, organization, project, pathPrefix, false) -} - -func (m *ObjectManager) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - return m.listStorageCleanupRecords(ctx, organization, project, pathPrefix, true) -} - -func (m *ObjectManager) listStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string, duplicatesOnly bool) ([]models.StorageCleanupRecord, error) { - store, ok := m.db.(db.StorageMetricsStore) - if !ok { - return nil, fmt.Errorf("storage cleanup listing not supported by database") - } - organization = strings.TrimSpace(organization) - project = strings.TrimSpace(project) - if organization == "" || project == "" { - return nil, fmt.Errorf("organization and project are required") - } - if authz.IsAuthzEnforced(ctx) { - if authz.IsGen3Mode(ctx) && !authz.HasAuthHeader(ctx) { - return nil, common.ErrUnauthorized - } - resource, err := sycommon.ResourcePath(organization, project) - if err != nil { - return nil, err - } - if !authz.HasMethodAccess(ctx, objectMethodRead, []string{"/programs", "/data_file"}) && !authz.HasAnyMethodAccess(ctx, []string{resource}, objectMethodRead) { - return nil, common.ErrUnauthorized - } - } - if duplicatesOnly { - return store.ListDuplicateStorageCleanupRecords(ctx, organization, project, pathPrefix) - } - return store.ListStorageCleanupRecords(ctx, organization, project, pathPrefix) -} diff --git a/internal/db/interface.go b/internal/db/interface.go index 1670bf74..b7cf389e 100644 --- a/internal/db/interface.go +++ b/internal/db/interface.go @@ -41,10 +41,6 @@ type ObjectIDPageLister interface { ListObjectIDsPageByResources(ctx context.Context, resources []string, includeUnscoped bool, startAfter string, limit, offset int) ([]string, error) } -type ObjectPathPageLister interface { - ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) -} - type ObjectChecksumPageLister interface { ListObjectIDsPageByChecksum(ctx context.Context, checksum, checksumType, organization, project, startAfter string, limit, offset int, resources []string, includeUnscoped, restrictToResources bool) ([]string, error) } @@ -65,13 +61,6 @@ type FileUsageScopedLister interface { GetFileUsageSummaryByResources(ctx context.Context, resources []string, includeUnscoped bool, inactiveSince *time.Time) (models.FileUsageSummary, error) } -type StorageMetricsStore interface { - GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) - ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) - ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) - ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) -} - type TransferAttributionScopedStore interface { GetTransferAttributionSummaryByResources(ctx context.Context, filter models.TransferAttributionFilter, resources []string) (models.TransferAttributionSummary, error) GetTransferAttributionBreakdownByResources(ctx context.Context, filter models.TransferAttributionFilter, groupBy string, resources []string) ([]models.TransferAttributionBreakdown, error) @@ -141,10 +130,6 @@ type MetricsStore interface { ListFileUsageByObjectIDs(ctx context.Context, ids []string) ([]models.FileUsage, error) ListFileUsage(ctx context.Context, limit, offset int, inactiveSince *time.Time) ([]models.FileUsage, error) GetFileUsageSummary(ctx context.Context, inactiveSince *time.Time) (models.FileUsageSummary, error) - GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) - ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) - ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) - ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) } // LFSStore is the minimum storage surface needed by the LFS API. @@ -162,5 +147,4 @@ type DatabaseInterface interface { CredentialStore PendingLFSMetaStore UsageStore - StorageMetricsStore } diff --git a/internal/db/postgres/browse_index.go b/internal/db/postgres/browse_index.go deleted file mode 100644 index 041673ca..00000000 --- a/internal/db/postgres/browse_index.go +++ /dev/null @@ -1,163 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - "fmt" - "sort" - "strings" - - sycommon "github.com/calypr/syfon/common" - icommon "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" - "github.com/lib/pq" -) - -type browseIndexRow struct { - ObjectID string - Resource string - NormalizedPath string - ParentPath string - EntryName string -} - -func postgresBrowseRowsForObject(objectID, objectName string, resources []string) []browseIndexRow { - info, ok, err := icommon.BrowsePathInfoFromName(objectName) - if err != nil || !ok { - return nil - } - normalizedResources := sycommon.NormalizeAccessResources(resources) - rows := make([]browseIndexRow, 0, len(normalizedResources)) - for _, resource := range normalizedResources { - rows = append(rows, browseIndexRow{ - ObjectID: objectID, - Resource: resource, - NormalizedPath: info.Normalized, - ParentPath: info.ParentPath, - EntryName: info.EntryName, - }) - } - return rows -} - -func postgresDeleteBrowseRowsByIDsTx(ctx context.Context, tx *sql.Tx, ids []string) error { - if len(ids) == 0 { - return nil - } - _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = ANY($1)`, pq.Array(ids)) - return err -} - -func postgresDeleteBrowseRowTx(ctx context.Context, tx *sql.Tx, objectID, resource string) error { - _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = $1 AND resource = $2`, objectID, resource) - return err -} - -func postgresInsertBrowseRowsTx(ctx context.Context, tx *sql.Tx, rows []browseIndexRow) error { - if len(rows) == 0 { - return nil - } - objectIDs := make([]string, 0, len(rows)) - resources := make([]string, 0, len(rows)) - normalizedPaths := make([]string, 0, len(rows)) - parentPaths := make([]string, 0, len(rows)) - entryNames := make([]string, 0, len(rows)) - for _, row := range rows { - objectIDs = append(objectIDs, row.ObjectID) - resources = append(resources, row.Resource) - normalizedPaths = append(normalizedPaths, row.NormalizedPath) - parentPaths = append(parentPaths, row.ParentPath) - entryNames = append(entryNames, row.EntryName) - } - _, err := tx.ExecContext(ctx, ` - INSERT INTO drs_object_browse_index (object_id, resource, normalized_path, parent_path, entry_name) - SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::text[], $5::text[]) - ON CONFLICT (resource, object_id) DO UPDATE SET - normalized_path = EXCLUDED.normalized_path, - parent_path = EXCLUDED.parent_path, - entry_name = EXCLUDED.entry_name`, - pq.Array(objectIDs), pq.Array(resources), pq.Array(normalizedPaths), pq.Array(parentPaths), pq.Array(entryNames), - ) - return err -} - -func postgresRebuildBrowseRowsForObjectTx(ctx context.Context, tx *sql.Tx, objectID, objectName string, resources []string) error { - if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = $1`, objectID); err != nil { - return err - } - return postgresInsertBrowseRowsTx(ctx, tx, postgresBrowseRowsForObject(objectID, objectName, resources)) -} - -func (db *PostgresDB) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { - resource, err := sycommon.ResourcePath(organization, project) - if err != nil { - return nil, nil, err - } - normalizedPath, _, err := icommon.NormalizeBrowsePath(path) - if err != nil { - return nil, nil, err - } - if offset < 0 { - offset = 0 - } - - args := []any{resource, normalizedPath} - query := ` - SELECT object_id - FROM drs_object_browse_index - WHERE resource = $1 AND parent_path = $2` - if strings.TrimSpace(startAfter) != "" { - args = append(args, strings.TrimSpace(startAfter)) - query += fmt.Sprintf(" AND object_id > $%d", len(args)) - } - args = append(args, limit, offset) - query += fmt.Sprintf(" ORDER BY object_id LIMIT $%d OFFSET $%d", len(args)-1, len(args)) - rows, err := db.db.QueryContext(ctx, query, args...) - if err != nil { - return nil, nil, err - } - defer rows.Close() - ids, err := scanObjectIDs(rows) - if err != nil { - return nil, nil, err - } - - dirQuery := `SELECT normalized_path FROM drs_object_browse_index WHERE resource = $1 AND parent_path <> ''` - dirArgs := []any{resource} - if normalizedPath != "" { - dirQuery = `SELECT normalized_path FROM drs_object_browse_index WHERE resource = $1 AND normalized_path LIKE $2 AND parent_path <> $3` - dirArgs = append(dirArgs, normalizedPath+"/%", normalizedPath) - } - dirRows, err := db.db.QueryContext(ctx, dirQuery+" ORDER BY normalized_path", dirArgs...) - if err != nil { - return nil, nil, err - } - defer dirRows.Close() - - directoriesByPath := make(map[string]models.BrowseDirectory) - for dirRows.Next() { - var candidatePath string - if err := dirRows.Scan(&candidatePath); err != nil { - return nil, nil, err - } - dirInfo, ok := icommon.ImmediateBrowseDirectory(normalizedPath, candidatePath) - if !ok { - continue - } - directoriesByPath[dirInfo.Normalized] = models.BrowseDirectory{Name: dirInfo.EntryName, Path: dirInfo.Normalized} - } - if err := dirRows.Err(); err != nil { - return nil, nil, err - } - directories := make([]models.BrowseDirectory, 0, len(directoriesByPath)) - for _, directory := range directoriesByPath { - directories = append(directories, directory) - } - sort.Slice(directories, func(i, j int) bool { - if directories[i].Name == directories[j].Name { - return directories[i].Path < directories[j].Path - } - return directories[i].Name < directories[j].Name - }) - return ids, directories, nil -} diff --git a/internal/db/postgres/metrics.go b/internal/db/postgres/metrics.go index d5162f71..8e7138b0 100644 --- a/internal/db/postgres/metrics.go +++ b/internal/db/postgres/metrics.go @@ -12,7 +12,6 @@ import ( sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" - "github.com/calypr/syfon/internal/storagemetrics" "github.com/lib/pq" ) @@ -275,145 +274,6 @@ func (db *PostgresDB) GetFileUsage(ctx context.Context, objectID string) (*model return &usage, nil } -func (db *PostgresDB) GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) { - if err := db.flushObjectUsageEvents(ctx); err != nil { - return models.StoragePathSummary{}, err - } - rows, err := db.listStorageMetricRowsByScope(ctx, organization, project) - if err != nil { - return models.StoragePathSummary{}, err - } - return storagemetrics.AggregateStoragePathSummary(strings.TrimSpace(organization), strings.TrimSpace(project), path, rows) -} - -func (db *PostgresDB) ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) { - if err := db.flushObjectUsageEvents(ctx); err != nil { - return nil, err - } - rows, err := db.listStorageMetricRowsByScope(ctx, organization, project) - if err != nil { - return nil, err - } - return storagemetrics.AggregateStoragePathChildren(path, rows, limit, offset, sortBy, sortOrder) -} - -func (db *PostgresDB) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - if err := db.flushObjectUsageEvents(ctx); err != nil { - return nil, err - } - resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) - if err != nil { - return nil, err - } - where, args, err := postgresStorageCleanupScopeArgs(resource, pathPrefix, 1, "bi.normalized_path") - if err != nil { - return nil, err - } - rows, err := db.db.QueryContext(ctx, ` - SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, - COALESCE(u.download_count, 0), u.last_download_time - FROM drs_object_browse_index bi - JOIN drs_object o ON o.id = bi.object_id - LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE `+where+` - ORDER BY bi.normalized_path ASC, bi.object_id ASC - `, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - out := make([]models.StorageCleanupRecord, 0) - for rows.Next() { - var row models.StorageCleanupRecord - var lastDownload sql.NullTime - if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { - return nil, err - } - if lastDownload.Valid { - t := lastDownload.Time.UTC() - row.LastDownloadTime = &t - } - out = append(out, row) - } - if err := rows.Err(); err != nil { - return nil, err - } - return out, nil -} - -func (db *PostgresDB) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - if err := db.flushObjectUsageEvents(ctx); err != nil { - return nil, err - } - resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) - if err != nil { - return nil, err - } - cteWhere, cteArgs, err := postgresStorageCleanupScopeArgs(resource, pathPrefix, 1, "normalized_path") - if err != nil { - return nil, err - } - mainWhere, _, err := postgresStorageCleanupScopeArgs(resource, pathPrefix, 1, "bi.normalized_path") - if err != nil { - return nil, err - } - rows, err := db.db.QueryContext(ctx, ` - WITH duplicate_paths AS ( - SELECT normalized_path - FROM drs_object_browse_index - WHERE `+cteWhere+` - GROUP BY normalized_path - HAVING COUNT(*) > 1 - ) - SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, - COALESCE(u.download_count, 0), u.last_download_time - FROM drs_object_browse_index bi - JOIN duplicate_paths dp ON dp.normalized_path = bi.normalized_path - JOIN drs_object o ON o.id = bi.object_id - LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE `+mainWhere+` - ORDER BY bi.normalized_path ASC, bi.object_id ASC - `, cteArgs...) - if err != nil { - return nil, err - } - defer rows.Close() - - out := make([]models.StorageCleanupRecord, 0) - for rows.Next() { - var row models.StorageCleanupRecord - var lastDownload sql.NullTime - if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { - return nil, err - } - if lastDownload.Valid { - t := lastDownload.Time.UTC() - row.LastDownloadTime = &t - } - out = append(out, row) - } - if err := rows.Err(); err != nil { - return nil, err - } - return out, nil -} - -func postgresStorageCleanupScopeArgs(resource, pathPrefix string, argStart int, normalizedPathColumn string) (string, []any, error) { - where := []string{fmt.Sprintf("resource = $%d", argStart)} - args := []any{resource} - normalizedPrefix, _, err := common.NormalizeBrowsePath(pathPrefix) - if err != nil { - return "", nil, err - } - if normalizedPrefix == "" { - return strings.Join(where, " AND "), args, nil - } - where = append(where, fmt.Sprintf("(%s = $%d OR %s LIKE $%d)", normalizedPathColumn, argStart+1, normalizedPathColumn, argStart+2)) - args = append(args, normalizedPrefix, normalizedPrefix+"/%") - return strings.Join(where, " AND "), args, nil -} - func (db *PostgresDB) ListFileUsageByObjectIDs(ctx context.Context, ids []string) ([]models.FileUsage, error) { if len(ids) == 0 { return []models.FileUsage{}, nil @@ -537,44 +397,6 @@ func scanFileUsageRows(rows *sql.Rows, capacity int) ([]models.FileUsage, error) return out, nil } -func (db *PostgresDB) listStorageMetricRowsByScope(ctx context.Context, organization, project string) ([]models.DrsObjectRecord, error) { - resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) - if err != nil { - return nil, err - } - rows, err := db.db.QueryContext(ctx, ` - SELECT bi.object_id, o.size, o.updated_time, - COALESCE(u.download_count, 0), u.last_download_time, - bi.normalized_path, bi.normalized_path - FROM drs_object_browse_index bi - JOIN drs_object o ON o.id = bi.object_id - LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE bi.resource = $1 - ORDER BY bi.object_id - `, resource) - if err != nil { - return nil, err - } - defer rows.Close() - out := make([]models.DrsObjectRecord, 0) - for rows.Next() { - var row models.DrsObjectRecord - var lastDownload sql.NullTime - if err := rows.Scan(&row.ID, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload, &row.Name, &row.Path); err != nil { - return nil, err - } - if lastDownload.Valid { - t := lastDownload.Time.UTC() - row.LastDownloadTime = &t - } - out = append(out, row) - } - if err := rows.Err(); err != nil { - return nil, err - } - return out, nil -} - func (db *PostgresDB) listScopedFileUsagePage(ctx context.Context, resources []string, includeUnscoped bool, limit, offset int, inactiveSince *time.Time) ([]models.FileUsage, error) { if err := db.flushObjectUsageEvents(ctx); err != nil { return nil, err diff --git a/internal/db/postgres/metrics_sqlmock_test.go b/internal/db/postgres/metrics_sqlmock_test.go index 15e5dd3e..ca42a4fc 100644 --- a/internal/db/postgres/metrics_sqlmock_test.go +++ b/internal/db/postgres/metrics_sqlmock_test.go @@ -108,129 +108,3 @@ func TestRecordProviderTransferEvents_EmptyInput(t *testing.T) { t.Fatalf("expected nil for empty provider transfer events, got %v", err) } } - -func TestGetStoragePathSummary(t *testing.T) { - pg, mock, rawDB := newMockPostgresDB(t) - defer rawDB.Close() - - now := time.Now().UTC() - mock.ExpectBegin() - mock.ExpectQuery(regexp.QuoteMeta(` - SELECT DISTINCT e.object_id - FROM object_usage_event e - JOIN drs_object o ON o.id = e.object_id - `)).WillReturnRows(sqlmock.NewRows([]string{"object_id"})) - mock.ExpectCommit() - mock.ExpectQuery(regexp.QuoteMeta(` - SELECT bi.object_id, o.size, o.updated_time, - COALESCE(u.download_count, 0), u.last_download_time, - bi.normalized_path, bi.normalized_path - FROM drs_object_browse_index bi - JOIN drs_object o ON o.id = bi.object_id - LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE bi.resource = $1 - ORDER BY bi.object_id - `)). - WithArgs("/organization/org/project/proj"). - WillReturnRows(sqlmock.NewRows([]string{"object_id", "size", "updated_time", "download_count", "last_download_time", "normalized_path", "normalized_path"}). - AddRow("obj-1", int64(10), now.Add(-time.Hour), int64(2), now.Add(-3*time.Hour), "data/a.txt", "data/a.txt"). - AddRow("obj-2", int64(20), now, int64(4), now.Add(-30*time.Minute), "data/a.txt", "data/a.txt"). - AddRow("obj-3", int64(30), now.Add(-2*time.Hour), int64(1), now.Add(-90*time.Minute), "data/nested/b.txt", "data/nested/b.txt")) - - summary, err := pg.GetStoragePathSummary(context.Background(), "org", "proj", "data") - if err != nil { - t.Fatalf("GetStoragePathSummary returned error: %v", err) - } - if summary.RecordCount != 3 || summary.FileCount != 2 || summary.TotalBytes != 60 || summary.DuplicatePathCount != 1 || summary.DownloadCount != 7 { - t.Fatalf("unexpected summary: %+v", summary) - } - if summary.LastDownloadTime == nil || !summary.LastDownloadTime.Equal(now.Add(-30*time.Minute)) { - t.Fatalf("unexpected last download time: %+v", summary.LastDownloadTime) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations: %v", err) - } -} - -func TestListStorageCleanupRecordsUsesSQLPrefixFilter(t *testing.T) { - pg, mock, rawDB := newMockPostgresDB(t) - defer rawDB.Close() - - now := time.Now().UTC() - mock.ExpectBegin() - mock.ExpectQuery(regexp.QuoteMeta(` - SELECT DISTINCT e.object_id - FROM object_usage_event e - JOIN drs_object o ON o.id = e.object_id - `)).WillReturnRows(sqlmock.NewRows([]string{"object_id"})) - mock.ExpectCommit() - mock.ExpectQuery(regexp.QuoteMeta(` - SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, - COALESCE(u.download_count, 0), u.last_download_time - FROM drs_object_browse_index bi - JOIN drs_object o ON o.id = bi.object_id - LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE resource = $1 AND (bi.normalized_path = $2 OR bi.normalized_path LIKE $3) - ORDER BY bi.normalized_path ASC, bi.object_id ASC - `)). - WithArgs("/organization/org/project/proj", "data", "data/%"). - WillReturnRows(sqlmock.NewRows([]string{"object_id", "normalized_path", "size", "updated_time", "download_count", "last_download_time"}). - AddRow("obj-1", "data/a.txt", int64(10), now, int64(2), now)) - - rows, err := pg.ListStorageCleanupRecords(context.Background(), "org", "proj", "data") - if err != nil { - t.Fatalf("ListStorageCleanupRecords returned error: %v", err) - } - if len(rows) != 1 || rows[0].NormalizedPath != "data/a.txt" { - t.Fatalf("unexpected rows: %+v", rows) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations: %v", err) - } -} - -func TestListDuplicateStorageCleanupRecordsFiltersToDuplicatePaths(t *testing.T) { - pg, mock, rawDB := newMockPostgresDB(t) - defer rawDB.Close() - - now := time.Now().UTC() - mock.ExpectBegin() - mock.ExpectQuery(regexp.QuoteMeta(` - SELECT DISTINCT e.object_id - FROM object_usage_event e - JOIN drs_object o ON o.id = e.object_id - `)).WillReturnRows(sqlmock.NewRows([]string{"object_id"})) - mock.ExpectCommit() - mock.ExpectQuery(regexp.QuoteMeta(` - WITH duplicate_paths AS ( - SELECT normalized_path - FROM drs_object_browse_index - WHERE resource = $1 AND (normalized_path = $2 OR normalized_path LIKE $3) - GROUP BY normalized_path - HAVING COUNT(*) > 1 - ) - SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, - COALESCE(u.download_count, 0), u.last_download_time - FROM drs_object_browse_index bi - JOIN duplicate_paths dp ON dp.normalized_path = bi.normalized_path - JOIN drs_object o ON o.id = bi.object_id - LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE resource = $1 AND (bi.normalized_path = $2 OR bi.normalized_path LIKE $3) - ORDER BY bi.normalized_path ASC, bi.object_id ASC - `)). - WithArgs("/organization/org/project/proj", "data", "data/%"). - WillReturnRows(sqlmock.NewRows([]string{"object_id", "normalized_path", "size", "updated_time", "download_count", "last_download_time"}). - AddRow("obj-1", "data/a.txt", int64(10), now, int64(2), now). - AddRow("obj-2", "data/a.txt", int64(11), now, int64(3), now)) - - rows, err := pg.ListDuplicateStorageCleanupRecords(context.Background(), "org", "proj", "data") - if err != nil { - t.Fatalf("ListDuplicateStorageCleanupRecords returned error: %v", err) - } - if len(rows) != 2 { - t.Fatalf("expected duplicate rows, got %+v", rows) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations: %v", err) - } -} diff --git a/internal/db/postgres/object_schema.sql b/internal/db/postgres/object_schema.sql index c5eb31d1..339da7c9 100644 --- a/internal/db/postgres/object_schema.sql +++ b/internal/db/postgres/object_schema.sql @@ -4,13 +4,12 @@ CREATE TABLE IF NOT EXISTS drs_object ( created_time TIMESTAMPTZ, updated_time TIMESTAMPTZ, name TEXT, - file_name TEXT, version TEXT, description TEXT ); ALTER TABLE drs_object - ADD COLUMN IF NOT EXISTS file_name TEXT; + DROP COLUMN IF EXISTS file_name; CREATE TABLE IF NOT EXISTS drs_object_access_method ( object_id TEXT NOT NULL, @@ -38,15 +37,7 @@ CREATE TABLE IF NOT EXISTS drs_object_alias ( FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE ); -CREATE TABLE IF NOT EXISTS drs_object_browse_index ( - object_id TEXT NOT NULL, - resource TEXT NOT NULL, - normalized_path TEXT NOT NULL, - parent_path TEXT NOT NULL, - entry_name TEXT NOT NULL, - PRIMARY KEY (resource, object_id), - FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE -); +DROP TABLE IF EXISTS drs_object_browse_index; CREATE INDEX IF NOT EXISTS drs_object_access_method_object_id_idx ON drs_object_access_method(object_id); @@ -74,12 +65,3 @@ CREATE INDEX IF NOT EXISTS drs_object_controlled_access_object_id_resource_idx CREATE INDEX IF NOT EXISTS drs_object_alias_object_id_idx ON drs_object_alias(object_id); - -CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_object_id_idx - ON drs_object_browse_index(resource, parent_path, object_id); - -CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_entry_name_idx - ON drs_object_browse_index(resource, parent_path, entry_name); - -CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_normalized_path_idx - ON drs_object_browse_index(resource, normalized_path); diff --git a/internal/db/postgres/objects.go b/internal/db/postgres/objects.go index cd2ee6e4..2c9337eb 100644 --- a/internal/db/postgres/objects.go +++ b/internal/db/postgres/objects.go @@ -5,7 +5,6 @@ import ( "database/sql" "errors" "fmt" - "path" "sort" "strings" "time" @@ -17,37 +16,6 @@ import ( "github.com/lib/pq" ) -func postgresStoredPath(obj *models.InternalObject) string { - if obj != nil && obj.Properties != nil { - if raw, ok := obj.Properties["path"].(string); ok && strings.TrimSpace(raw) != "" { - return strings.TrimSpace(raw) - } - if raw, ok := obj.Properties["file_name"].(string); ok && strings.TrimSpace(raw) != "" { - return strings.TrimSpace(raw) - } - } - if obj == nil { - return "" - } - return common.StringVal(obj.Name) -} - -func postgresLoadedNames(name, fileName sql.NullString) (string, string) { - loadedName := strings.TrimSpace(name.String) - loadedPath := strings.TrimSpace(fileName.String) - if loadedPath == "" { - loadedPath = loadedName - loadedName = "" - } - if loadedName == "" && loadedPath != "" { - loadedName = path.Base(strings.Trim(loadedPath, "/")) - if loadedName == "." || loadedName == "/" || loadedName == "" { - loadedName = loadedPath - } - } - return loadedName, loadedPath -} - func (db *PostgresDB) DeleteObject(ctx context.Context, id string) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -142,11 +110,11 @@ func (db *PostgresDB) GetObject(ctx context.Context, id string) (*models.Interna retryLookup: // 1. Fetch main record var r models.DrsObjectRecord - var name, fileName, version, description sql.NullString + var name, version, description sql.NullString err := db.db.QueryRowContext(ctx, ` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = $1`, lookupID).Scan( - &r.ID, &r.Size, &r.CreatedTime, &r.UpdatedTime, &name, &fileName, &version, &description, + &r.ID, &r.Size, &r.CreatedTime, &r.UpdatedTime, &name, &version, &description, ) if err == sql.ErrNoRows { if !resolvedAlias { @@ -165,7 +133,7 @@ retryLookup: if err != nil { return nil, fmt.Errorf("failed to fetch record: %w", err) } - r.Name, r.Path = postgresLoadedNames(name, fileName) + r.Name = strings.TrimSpace(name.String) r.Version = version.String r.Description = description.String objectID := r.ID @@ -186,9 +154,6 @@ retryLookup: }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(r.Path) != "" { - obj.Properties["path"] = r.Path - } // 2. Fetch storage access methods. urlRows, err := db.db.QueryContext(ctx, "SELECT url, type FROM drs_object_access_method WHERE object_id = $1", lookupID) if err != nil { @@ -261,9 +226,9 @@ func (db *PostgresDB) CreateObject(ctx context.Context, obj *models.InternalObje // Insert main record _, err = tx.ExecContext(ctx, ` - INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), postgresStoredPath(obj), common.StringVal(obj.Version), common.StringVal(obj.Description), + INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), common.StringVal(obj.Version), common.StringVal(obj.Description), ) if err != nil { return fmt.Errorf("failed to insert drs_object: %w", err) @@ -272,9 +237,6 @@ func (db *PostgresDB) CreateObject(ctx context.Context, obj *models.InternalObje if err := insertControlledAccessTx(ctx, tx, obj.Id, objectAccessResources(obj)); err != nil { return err } - if err := postgresRebuildBrowseRowsForObjectTx(ctx, tx, obj.Id, postgresStoredPath(obj), objectAccessResources(obj)); err != nil { - return fmt.Errorf("failed to update browse index: %w", err) - } // Insert storage access methods. if obj.AccessMethods != nil { @@ -325,14 +287,12 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte names := make([]string, 0, len(objects)) versions := make([]string, 0, len(objects)) descriptions := make([]string, 0, len(objects)) - fileNames := make([]string, 0, len(objects)) accessObjectIDs := make([]string, 0) accessURLs := make([]string, 0) accessTypes := make([]string, 0) controlledObjectIDs := make([]string, 0) controlledResources := make([]string, 0) - browseRows := make([]browseIndexRow, 0) checksumObjectIDs := make([]string, 0) checksumTypes := make([]string, 0) @@ -344,7 +304,6 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte createdTimes = append(createdTimes, obj.CreatedTime) updatedTimes = append(updatedTimes, common.TimeVal(obj.UpdatedTime)) names = append(names, common.StringVal(obj.Name)) - fileNames = append(fileNames, postgresStoredPath(&obj)) versions = append(versions, common.StringVal(obj.Version)) descriptions = append(descriptions, common.StringVal(obj.Description)) @@ -353,7 +312,6 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte controlledObjectIDs = append(controlledObjectIDs, obj.Id) controlledResources = append(controlledResources, resource) } - browseRows = append(browseRows, postgresBrowseRowsForObject(obj.Id, postgresStoredPath(&obj), objectAccessResources(&obj))...) if obj.AccessMethods != nil { for _, am := range *obj.AccessMethods { if am.AccessUrl == nil || am.AccessUrl.Url == "" { @@ -384,18 +342,17 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte } if _, err := tx.ExecContext(ctx, ` - INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) - SELECT * FROM UNNEST($1::text[], $2::bigint[], $3::timestamp[], $4::timestamp[], $5::text[], $6::text[], $7::text[], $8::text[]) + INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) + SELECT * FROM UNNEST($1::text[], $2::bigint[], $3::timestamp[], $4::timestamp[], $5::text[], $6::text[], $7::text[]) ON CONFLICT (id) DO UPDATE SET size = EXCLUDED.size, created_time = EXCLUDED.created_time, updated_time = EXCLUDED.updated_time, name = EXCLUDED.name, - file_name = EXCLUDED.file_name, version = EXCLUDED.version, description = EXCLUDED.description`, pq.Array(ids), pq.Array(sizes), pq.Array(createdTimes), pq.Array(updatedTimes), - pq.Array(names), pq.Array(fileNames), pq.Array(versions), pq.Array(descriptions), + pq.Array(names), pq.Array(versions), pq.Array(descriptions), ); err != nil { return fmt.Errorf("failed bulk upsert drs_object: %w", err) } @@ -409,9 +366,6 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_checksum WHERE object_id = ANY($1)`, pq.Array(ids)); err != nil { return fmt.Errorf("failed bulk clear checksums: %w", err) } - if err := postgresDeleteBrowseRowsByIDsTx(ctx, tx, ids); err != nil { - return fmt.Errorf("failed bulk clear browse index: %w", err) - } if len(accessObjectIDs) > 0 { if _, err := tx.ExecContext(ctx, ` @@ -441,9 +395,6 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte return fmt.Errorf("failed bulk insert checksums: %w", err) } } - if err := postgresInsertBrowseRowsTx(ctx, tx, browseRows); err != nil { - return fmt.Errorf("failed bulk insert browse index: %w", err) - } if err := db.flushObjectUsageEventsForIDsTx(ctx, tx, ids); err != nil { return fmt.Errorf("failed to apply object usage events: %w", err) @@ -1143,7 +1094,6 @@ func (db *PostgresDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []st o.created_time, o.updated_time, o.name, - o.file_name, o.version, o.description FROM drs_object o @@ -1170,35 +1120,30 @@ func (db *PostgresDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []st for rows.Next() { var ( - id string - name, fileName sql.NullString - version, description sql.NullString - size int64 - createdTime, updatedTime time.Time + id string + name sql.NullString + version, description sql.NullString + size int64 + createdTime, updatedTime time.Time ) if err := rows.Scan( - &id, &size, &createdTime, &updatedTime, &name, &fileName, &version, &description, + &id, &size, &createdTime, &updatedTime, &name, &version, &description, ); err != nil { return nil, err } - - loadedName, loadedPath := postgresLoadedNames(name, fileName) objectsByID[id] = &models.InternalObject{ DrsObject: drs.DrsObject{ Id: id, Size: size, CreatedTime: createdTime, UpdatedTime: common.Ptr(updatedTime), - Name: common.Ptr(loadedName), + Name: common.Ptr(strings.TrimSpace(name.String)), Version: common.Ptr(version.String), Description: common.Ptr(description.String), SelfUri: "drs://" + id, }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(loadedPath) != "" { - objectsByID[id].Properties["path"] = loadedPath - } } if err := rows.Err(); err != nil { @@ -1356,9 +1301,6 @@ func (db *PostgresDB) RemoveObjectControlledAccess(ctx context.Context, objectID if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_controlled_access WHERE object_id = $1 AND resource = $2`, objectID, resource); err != nil { return err } - if err := postgresDeleteBrowseRowTx(ctx, tx, objectID, resource); err != nil { - return err - } return tx.Commit() } diff --git a/internal/db/postgres/objects_sqlmock_test.go b/internal/db/postgres/objects_sqlmock_test.go index 24935b44..505e156a 100644 --- a/internal/db/postgres/objects_sqlmock_test.go +++ b/internal/db/postgres/objects_sqlmock_test.go @@ -87,7 +87,7 @@ func TestGetObject_NotFound(t *testing.T) { defer rawDB.Close() mock.ExpectQuery(regexp.QuoteMeta(` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = $1`)). WithArgs("missing"). WillReturnError(sql.ErrNoRows) @@ -108,12 +108,12 @@ func TestGetObject_DeduplicatesAndPropagatesAuthz(t *testing.T) { created := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) updated := created.Add(2 * time.Hour) mock.ExpectQuery(regexp.QuoteMeta(` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = $1`)). WithArgs("obj-1"). WillReturnRows(sqlmock.NewRows([]string{ - "id", "size", "created_time", "updated_time", "name", "file_name", "version", "description", - }).AddRow("obj-1", int64(123), created, updated, "file.txt", "nested/file.txt", "v1", "desc")) + "id", "size", "created_time", "updated_time", "name", "version", "description", + }).AddRow("obj-1", int64(123), created, updated, "file.txt", "v1", "desc")) mock.ExpectQuery(regexp.QuoteMeta("SELECT url, type FROM drs_object_access_method WHERE object_id = $1")). WithArgs("obj-1"). @@ -162,12 +162,12 @@ func TestGetObject_IgnoresAuthContext(t *testing.T) { now := time.Date(2026, time.March, 1, 10, 0, 0, 0, time.UTC) mock.ExpectQuery(regexp.QuoteMeta(` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = $1`)). WithArgs("obj-2"). WillReturnRows(sqlmock.NewRows([]string{ - "id", "size", "created_time", "updated_time", "name", "file_name", "version", "description", - }).AddRow("obj-2", int64(1), now, now, "n", "nested/n", "v", "d")) + "id", "size", "created_time", "updated_time", "name", "version", "description", + }).AddRow("obj-2", int64(1), now, now, "n", "v", "d")) mock.ExpectQuery(regexp.QuoteMeta("SELECT url, type FROM drs_object_access_method WHERE object_id = $1")). WithArgs("obj-2"). WillReturnRows(sqlmock.NewRows([]string{"url", "type"}). @@ -191,41 +191,6 @@ func TestGetObject_IgnoresAuthContext(t *testing.T) { } } -func TestGetObject_LegacyNameFallsBackToFileName(t *testing.T) { - pg, mock, rawDB := newMockPostgresDB(t) - defer rawDB.Close() - - now := time.Date(2026, time.March, 1, 10, 0, 0, 0, time.UTC) - mock.ExpectQuery(regexp.QuoteMeta(` - SELECT id, size, created_time, updated_time, name, file_name, version, description - FROM drs_object WHERE id = $1`)). - WithArgs("obj-legacy"). - WillReturnRows(sqlmock.NewRows([]string{ - "id", "size", "created_time", "updated_time", "name", "file_name", "version", "description", - }).AddRow("obj-legacy", int64(1), now, now, "nested/dir/file.txt", nil, "v", "d")) - mock.ExpectQuery(regexp.QuoteMeta("SELECT url, type FROM drs_object_access_method WHERE object_id = $1")). - WithArgs("obj-legacy"). - WillReturnRows(sqlmock.NewRows([]string{"url", "type"}). - AddRow("s3://bucket/key", "s3")) - mock.ExpectQuery(regexp.QuoteMeta("SELECT resource FROM drs_object_controlled_access WHERE object_id = $1 ORDER BY resource")). - WithArgs("obj-legacy"). - WillReturnRows(sqlmock.NewRows([]string{"resource"})) - mock.ExpectQuery(regexp.QuoteMeta("SELECT type, checksum FROM drs_object_checksum WHERE object_id = $1")). - WithArgs("obj-legacy"). - WillReturnRows(sqlmock.NewRows([]string{"type", "checksum"})) - - obj, err := pg.GetObject(context.Background(), "obj-legacy") - if err != nil { - t.Fatalf("GetObject returned error: %v", err) - } - if got := common.StringVal(obj.Name); got != "file.txt" { - t.Fatalf("expected basename name, got %q", got) - } - if got, _ := obj.Properties["file_name"].(string); got != "nested/dir/file.txt" { - t.Fatalf("expected legacy path to populate file_name, got %#v", obj.Properties["file_name"]) - } -} - func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { pg, mock, rawDB := newMockPostgresDB(t) defer rawDB.Close() @@ -240,7 +205,6 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { o.created_time, o.updated_time, o.name, - o.file_name, o.version, o.description FROM drs_object o @@ -258,10 +222,10 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { )`)). WithArgs(pq.Array([]string{"obj-2", "obj-1", "obj-2"}), pq.Array([]string(nil))). WillReturnRows(sqlmock.NewRows([]string{ - "id", "size", "created_time", "updated_time", "name", "file_name", "version", "description", + "id", "size", "created_time", "updated_time", "name", "version", "description", }). - AddRow("obj-1", int64(10), created, updated, "file-1", "nested/file-1", "v1", "desc-1"). - AddRow("obj-2", int64(20), created, updated, "file-2", "nested/file-2", "v2", "desc-2")) + AddRow("obj-1", int64(10), created, updated, "file-1", "v1", "desc-1"). + AddRow("obj-2", int64(20), created, updated, "file-2", "v2", "desc-2")) mock.ExpectQuery(regexp.QuoteMeta(` SELECT object_id, url, type @@ -313,36 +277,6 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { } } -func TestListObjectIDsPageByPath(t *testing.T) { - pg, mock, rawDB := newMockPostgresDB(t) - defer rawDB.Close() - - mock.ExpectQuery(regexp.QuoteMeta(` - SELECT object_id - FROM drs_object_browse_index - WHERE resource = $1 AND parent_path = $2 AND object_id > $3 ORDER BY object_id LIMIT $4 OFFSET $5`)). - WithArgs("/organization/org/project/proj", "nested", "obj-a", 10, 0). - WillReturnRows(sqlmock.NewRows([]string{"object_id"}).AddRow("obj-c")) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT normalized_path FROM drs_object_browse_index WHERE resource = $1 AND normalized_path LIKE $2 AND parent_path <> $3 ORDER BY normalized_path`)). - WithArgs("/organization/org/project/proj", "nested/%", "nested"). - WillReturnRows(sqlmock.NewRows([]string{"normalized_path"}).AddRow("nested/deep/b.txt")) - - ids, directories, err := pg.ListObjectIDsPageByPath(context.Background(), "org", "proj", "nested", "obj-a", 10, 0) - if err != nil { - t.Fatalf("ListObjectIDsPageByPath returned error: %v", err) - } - if !slices.Equal(ids, []string{"obj-c"}) { - t.Fatalf("unexpected IDs: %v", ids) - } - if len(directories) != 1 || directories[0].Path != "nested/deep" { - t.Fatalf("unexpected directories: %+v", directories) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations: %v", err) - } -} - func TestListObjectIDsByScopeOrgIncludesProjectScopes(t *testing.T) { pg, mock, rawDB := newMockPostgresDB(t) defer rawDB.Close() diff --git a/internal/db/postgres/postgres_schema_test.go b/internal/db/postgres/postgres_schema_test.go index 0fa3bcf5..4e1cf8df 100644 --- a/internal/db/postgres/postgres_schema_test.go +++ b/internal/db/postgres/postgres_schema_test.go @@ -14,12 +14,12 @@ func TestSchemaEnsurers(t *testing.T) { defer rawDB.Close() mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object").WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("ALTER TABLE drs_object ADD COLUMN IF NOT EXISTS file_name TEXT")).WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta("ALTER TABLE drs_object DROP COLUMN IF EXISTS file_name")).WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_access_method").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_controlled_access").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_checksum").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_alias").WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_browse_index").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("DROP TABLE IF EXISTS drs_object_browse_index").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_access_method_object_id_idx ON drs_object_access_method(object_id)")). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_checksum_object_id_idx ON drs_object_checksum(object_id)")). @@ -38,12 +38,6 @@ func TestSchemaEnsurers(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_alias_object_id_idx ON drs_object_alias(object_id)")). WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_object_id_idx ON drs_object_browse_index(resource, parent_path, object_id)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_entry_name_idx ON drs_object_browse_index(resource, parent_path, entry_name)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_normalized_path_idx ON drs_object_browse_index(resource, normalized_path)")). - WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectQuery("information_schema\\.columns"). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) diff --git a/internal/db/sqlite/browse_index.go b/internal/db/sqlite/browse_index.go deleted file mode 100644 index 32d6b622..00000000 --- a/internal/db/sqlite/browse_index.go +++ /dev/null @@ -1,142 +0,0 @@ -package sqlite - -import ( - "context" - "database/sql" - "sort" - "strings" - - sycommon "github.com/calypr/syfon/common" - icommon "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" -) - -type browseIndexRow struct { - ObjectID string - Resource string - NormalizedPath string - ParentPath string - EntryName string -} - -func sqliteBrowseRowsForObject(objectID, objectName string, resources []string) []browseIndexRow { - info, ok, err := icommon.BrowsePathInfoFromName(objectName) - if err != nil || !ok { - return nil - } - normalizedResources := sycommon.NormalizeAccessResources(resources) - rows := make([]browseIndexRow, 0, len(normalizedResources)) - for _, resource := range normalizedResources { - rows = append(rows, browseIndexRow{ - ObjectID: objectID, - Resource: resource, - NormalizedPath: info.Normalized, - ParentPath: info.ParentPath, - EntryName: info.EntryName, - }) - } - return rows -} - -func sqliteDeleteBrowseRowsByIDsTx(tx *sql.Tx, ids []string) error { - return execSQLiteDeleteByIDs(tx, "drs_object_browse_index", ids) -} - -func sqliteInsertBrowseRowsTx(tx *sql.Tx, rows []browseIndexRow) error { - if len(rows) == 0 { - return nil - } - args := make([]interface{}, 0, len(rows)*5) - for _, row := range rows { - args = append(args, row.ObjectID, row.Resource, row.NormalizedPath, row.ParentPath, row.EntryName) - } - return execSQLiteBulkInsert( - tx, - "INSERT INTO drs_object_browse_index (object_id, resource, normalized_path, parent_path, entry_name) VALUES ", - "(?, ?, ?, ?, ?)", - 5, - args, - " ON CONFLICT(resource, object_id) DO UPDATE SET normalized_path=excluded.normalized_path, parent_path=excluded.parent_path, entry_name=excluded.entry_name", - ) -} - -func sqliteRebuildBrowseRowsForObjectTx(ctx context.Context, tx *sql.Tx, objectID, objectName string, resources []string) error { - if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = ?`, objectID); err != nil { - return err - } - return sqliteInsertBrowseRowsTx(tx, sqliteBrowseRowsForObject(objectID, objectName, resources)) -} - -func (db *SqliteDB) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { - resource, err := sycommon.ResourcePath(organization, project) - if err != nil { - return nil, nil, err - } - normalizedPath, _, err := icommon.NormalizeBrowsePath(path) - if err != nil { - return nil, nil, err - } - if offset < 0 { - offset = 0 - } - - args := []any{resource, normalizedPath} - query := ` - SELECT object_id - FROM drs_object_browse_index - WHERE resource = ? AND parent_path = ?` - if strings.TrimSpace(startAfter) != "" { - args = append(args, strings.TrimSpace(startAfter)) - query += " AND object_id > ?" - } - args = append(args, limit, offset) - query += " ORDER BY object_id LIMIT ? OFFSET ?" - rows, err := db.db.QueryContext(ctx, query, args...) - if err != nil { - return nil, nil, err - } - defer rows.Close() - ids, err := scanObjectIDs(rows) - if err != nil { - return nil, nil, err - } - - dirQuery := `SELECT normalized_path FROM drs_object_browse_index WHERE resource = ? AND parent_path <> '' ORDER BY normalized_path` - dirArgs := []any{resource} - if normalizedPath != "" { - dirQuery = `SELECT normalized_path FROM drs_object_browse_index WHERE resource = ? AND normalized_path LIKE ? AND parent_path <> ? ORDER BY normalized_path` - dirArgs = append(dirArgs, normalizedPath+"/%", normalizedPath) - } - dirRows, err := db.db.QueryContext(ctx, dirQuery, dirArgs...) - if err != nil { - return nil, nil, err - } - defer dirRows.Close() - - directoriesByPath := make(map[string]models.BrowseDirectory) - for dirRows.Next() { - var candidatePath string - if err := dirRows.Scan(&candidatePath); err != nil { - return nil, nil, err - } - dirInfo, ok := icommon.ImmediateBrowseDirectory(normalizedPath, candidatePath) - if !ok { - continue - } - directoriesByPath[dirInfo.Normalized] = models.BrowseDirectory{Name: dirInfo.EntryName, Path: dirInfo.Normalized} - } - if err := dirRows.Err(); err != nil { - return nil, nil, err - } - directories := make([]models.BrowseDirectory, 0, len(directoriesByPath)) - for _, directory := range directoriesByPath { - directories = append(directories, directory) - } - sort.Slice(directories, func(i, j int) bool { - if directories[i].Name == directories[j].Name { - return directories[i].Path < directories[j].Path - } - return directories[i].Name < directories[j].Name - }) - return ids, directories, nil -} diff --git a/internal/db/sqlite/metrics.go b/internal/db/sqlite/metrics.go index cedcd127..b331a43f 100644 --- a/internal/db/sqlite/metrics.go +++ b/internal/db/sqlite/metrics.go @@ -13,7 +13,6 @@ import ( sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" - "github.com/calypr/syfon/internal/storagemetrics" ) func (db *SqliteDB) flushObjectUsageEventsForIDsTx(ctx context.Context, tx *sql.Tx, ids []string) error { @@ -266,145 +265,6 @@ func (db *SqliteDB) GetTransferAttributionBreakdown(ctx context.Context, filter return scanTransferAttributionBreakdown(rows) } -func (db *SqliteDB) GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) { - if err := db.flushObjectUsageEvents(ctx); err != nil { - return models.StoragePathSummary{}, err - } - rows, err := db.listStorageMetricRowsByScope(ctx, organization, project) - if err != nil { - return models.StoragePathSummary{}, err - } - return storagemetrics.AggregateStoragePathSummary(strings.TrimSpace(organization), strings.TrimSpace(project), path, rows) -} - -func (db *SqliteDB) ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) { - if err := db.flushObjectUsageEvents(ctx); err != nil { - return nil, err - } - rows, err := db.listStorageMetricRowsByScope(ctx, organization, project) - if err != nil { - return nil, err - } - return storagemetrics.AggregateStoragePathChildren(path, rows, limit, offset, sortBy, sortOrder) -} - -func (db *SqliteDB) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - if err := db.flushObjectUsageEvents(ctx); err != nil { - return nil, err - } - resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) - if err != nil { - return nil, err - } - where, args, err := sqliteStorageCleanupScopeArgs(resource, pathPrefix, "bi.normalized_path") - if err != nil { - return nil, err - } - rows, err := db.db.QueryContext(ctx, ` - SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, - COALESCE(u.download_count, 0), u.last_download_time - FROM drs_object_browse_index bi - JOIN drs_object o ON o.id = bi.object_id - LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE `+where+` - ORDER BY bi.normalized_path ASC, bi.object_id ASC - `, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - out := make([]models.StorageCleanupRecord, 0) - for rows.Next() { - var row models.StorageCleanupRecord - var lastDownload sql.NullTime - if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { - return nil, err - } - if lastDownload.Valid { - t := lastDownload.Time.UTC() - row.LastDownloadTime = &t - } - out = append(out, row) - } - if err := rows.Err(); err != nil { - return nil, err - } - return out, nil -} - -func (db *SqliteDB) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - if err := db.flushObjectUsageEvents(ctx); err != nil { - return nil, err - } - resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) - if err != nil { - return nil, err - } - cteWhere, cteArgs, err := sqliteStorageCleanupScopeArgs(resource, pathPrefix, "normalized_path") - if err != nil { - return nil, err - } - mainWhere, _, err := sqliteStorageCleanupScopeArgs(resource, pathPrefix, "bi.normalized_path") - if err != nil { - return nil, err - } - rows, err := db.db.QueryContext(ctx, ` - WITH duplicate_paths AS ( - SELECT normalized_path - FROM drs_object_browse_index - WHERE `+cteWhere+` - GROUP BY normalized_path - HAVING COUNT(*) > 1 - ) - SELECT bi.object_id, bi.normalized_path, o.size, o.updated_time, - COALESCE(u.download_count, 0), u.last_download_time - FROM drs_object_browse_index bi - JOIN duplicate_paths dp ON dp.normalized_path = bi.normalized_path - JOIN drs_object o ON o.id = bi.object_id - LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE `+mainWhere+` - ORDER BY bi.normalized_path ASC, bi.object_id ASC - `, cteArgs...) - if err != nil { - return nil, err - } - defer rows.Close() - - out := make([]models.StorageCleanupRecord, 0) - for rows.Next() { - var row models.StorageCleanupRecord - var lastDownload sql.NullTime - if err := rows.Scan(&row.ObjectID, &row.NormalizedPath, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload); err != nil { - return nil, err - } - if lastDownload.Valid { - t := lastDownload.Time.UTC() - row.LastDownloadTime = &t - } - out = append(out, row) - } - if err := rows.Err(); err != nil { - return nil, err - } - return out, nil -} - -func sqliteStorageCleanupScopeArgs(resource, pathPrefix string, normalizedPathColumn string) (string, []any, error) { - where := []string{"resource = ?"} - args := []any{resource} - normalizedPrefix, _, err := common.NormalizeBrowsePath(pathPrefix) - if err != nil { - return "", nil, err - } - if normalizedPrefix == "" { - return strings.Join(where, " AND "), args, nil - } - where = append(where, fmt.Sprintf("(%s = ? OR %s LIKE ?)", normalizedPathColumn, normalizedPathColumn)) - args = append(args, normalizedPrefix, normalizedPrefix+"/%") - return strings.Join(where, " AND "), args, nil -} - func (db *SqliteDB) GetTransferAttributionSummaryByResources(ctx context.Context, filter models.TransferAttributionFilter, resources []string) (models.TransferAttributionSummary, error) { where, args := transferAttributionWhereByResources(filter, resources) var out models.TransferAttributionSummary @@ -636,44 +496,6 @@ func scanFileUsageRows(rows *sql.Rows, capacity int) ([]models.FileUsage, error) return out, nil } -func (db *SqliteDB) listStorageMetricRowsByScope(ctx context.Context, organization, project string) ([]models.DrsObjectRecord, error) { - resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) - if err != nil { - return nil, err - } - rows, err := db.db.QueryContext(ctx, ` - SELECT bi.object_id, o.size, o.updated_time, - COALESCE(u.download_count, 0), u.last_download_time, - bi.normalized_path, bi.normalized_path - FROM drs_object_browse_index bi - JOIN drs_object o ON o.id = bi.object_id - LEFT JOIN object_usage u ON u.object_id = bi.object_id - WHERE bi.resource = ? - ORDER BY bi.object_id - `, resource) - if err != nil { - return nil, err - } - defer rows.Close() - out := make([]models.DrsObjectRecord, 0) - for rows.Next() { - var row models.DrsObjectRecord - var lastDownload sql.NullTime - if err := rows.Scan(&row.ID, &row.Size, &row.UpdatedTime, &row.DownloadCount, &lastDownload, &row.Name, &row.Path); err != nil { - return nil, err - } - if lastDownload.Valid { - t := lastDownload.Time.UTC() - row.LastDownloadTime = &t - } - out = append(out, row) - } - if err := rows.Err(); err != nil { - return nil, err - } - return out, nil -} - func (db *SqliteDB) listScopedFileUsagePage(ctx context.Context, resources []string, includeUnscoped bool, limit, offset int, inactiveSince *time.Time) ([]models.FileUsage, error) { if err := db.flushObjectUsageEvents(ctx); err != nil { return nil, err diff --git a/internal/db/sqlite/objects.go b/internal/db/sqlite/objects.go index 2024c187..e3a30834 100644 --- a/internal/db/sqlite/objects.go +++ b/internal/db/sqlite/objects.go @@ -5,7 +5,6 @@ import ( "database/sql" "errors" "fmt" - "path" "sort" "strings" "time" @@ -16,37 +15,6 @@ import ( "github.com/calypr/syfon/internal/models" ) -func sqliteStoredPath(obj *models.InternalObject) string { - if obj != nil && obj.Properties != nil { - if raw, ok := obj.Properties["path"].(string); ok && strings.TrimSpace(raw) != "" { - return strings.TrimSpace(raw) - } - if raw, ok := obj.Properties["file_name"].(string); ok && strings.TrimSpace(raw) != "" { - return strings.TrimSpace(raw) - } - } - if obj == nil { - return "" - } - return common.StringVal(obj.Name) -} - -func sqliteLoadedNames(name, fileName sql.NullString) (string, string) { - loadedName := strings.TrimSpace(name.String) - loadedPath := strings.TrimSpace(fileName.String) - if loadedPath == "" { - loadedPath = loadedName - loadedName = "" - } - if loadedName == "" && loadedPath != "" { - loadedName = path.Base(strings.Trim(loadedPath, "/")) - if loadedName == "." || loadedName == "/" || loadedName == "" { - loadedName = loadedPath - } - } - return loadedName, loadedPath -} - func (db *SqliteDB) DeleteObject(ctx context.Context, id string) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -143,11 +111,11 @@ func (db *SqliteDB) GetObject(ctx context.Context, id string) (*models.InternalO retryLookup: // 1. Fetch main record var r models.DrsObjectRecord - var name, fileName, version, description sql.NullString + var name, version, description sql.NullString err := db.db.QueryRowContext(ctx, ` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = ?`, lookupID).Scan( - &r.ID, &r.Size, &r.CreatedTime, &r.UpdatedTime, &name, &fileName, &version, &description, + &r.ID, &r.Size, &r.CreatedTime, &r.UpdatedTime, &name, &version, &description, ) if err == sql.ErrNoRows { if !resolvedAlias { @@ -166,7 +134,7 @@ retryLookup: if err != nil { return nil, fmt.Errorf("failed to fetch record: %w", err) } - r.Name, r.Path = sqliteLoadedNames(name, fileName) + r.Name = strings.TrimSpace(name.String) r.Version = version.String r.Description = description.String objectID := r.ID @@ -187,9 +155,6 @@ retryLookup: }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(r.Path) != "" { - obj.Properties["path"] = r.Path - } // 2. Fetch storage access methods. urlRows, err := db.db.QueryContext(ctx, "SELECT url, type FROM drs_object_access_method WHERE object_id = ?", lookupID) @@ -263,9 +228,9 @@ func (db *SqliteDB) CreateObject(ctx context.Context, obj *models.InternalObject // Insert main record _, err = tx.ExecContext(ctx, ` - INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), sqliteStoredPath(obj), common.StringVal(obj.Version), common.StringVal(obj.Description), + INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), common.StringVal(obj.Version), common.StringVal(obj.Description), ) if err != nil { return fmt.Errorf("failed to insert drs_object: %w", err) @@ -274,9 +239,6 @@ func (db *SqliteDB) CreateObject(ctx context.Context, obj *models.InternalObject if err := insertControlledAccessTx(ctx, tx, obj.Id, objectAccessResources(obj)); err != nil { return err } - if err := sqliteRebuildBrowseRowsForObjectTx(ctx, tx, obj.Id, sqliteStoredPath(obj), objectAccessResources(obj)); err != nil { - return fmt.Errorf("failed to update browse index: %w", err) - } // Insert storage access methods. if obj.AccessMethods != nil { @@ -330,17 +292,15 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern accessArgs := make([]interface{}, 0) controlledArgs := make([]interface{}, 0) checksumArgs := make([]interface{}, 0) - browseRows := make([]browseIndexRow, 0) for _, obj := range objects { ids = append(ids, obj.Id) - mainArgs = append(mainArgs, obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), sqliteStoredPath(&obj), common.StringVal(obj.Version), common.StringVal(obj.Description)) + mainArgs = append(mainArgs, obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), common.StringVal(obj.Version), common.StringVal(obj.Description)) seenAccess := make(map[string]struct{}) for _, resource := range objectAccessResources(&obj) { controlledArgs = append(controlledArgs, obj.Id, resource) } - browseRows = append(browseRows, sqliteBrowseRowsForObject(obj.Id, sqliteStoredPath(&obj), objectAccessResources(&obj))...) if obj.AccessMethods != nil { for _, am := range *obj.AccessMethods { if am.AccessUrl == nil || am.AccessUrl.Url == "" { @@ -366,16 +326,15 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern } } - mainPrefix := `INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) VALUES ` + mainPrefix := `INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) VALUES ` mainSuffix := ` ON CONFLICT(id) DO UPDATE SET size=excluded.size, created_time=excluded.created_time, updated_time=excluded.updated_time, name=excluded.name, - file_name=excluded.file_name, version=excluded.version, description=excluded.description` - if err := execSQLiteBulkInsert(tx, mainPrefix, "(?, ?, ?, ?, ?, ?, ?, ?)", 8, mainArgs, mainSuffix); err != nil { + if err := execSQLiteBulkInsert(tx, mainPrefix, "(?, ?, ?, ?, ?, ?, ?)", 7, mainArgs, mainSuffix); err != nil { return fmt.Errorf("failed bulk upsert drs_object: %w", err) } @@ -388,9 +347,6 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern if err := execSQLiteDeleteByIDs(tx, "drs_object_checksum", ids); err != nil { return fmt.Errorf("failed bulk clear checksums: %w", err) } - if err := sqliteDeleteBrowseRowsByIDsTx(tx, ids); err != nil { - return fmt.Errorf("failed bulk clear browse index: %w", err) - } if len(accessArgs) > 0 { if err := execSQLiteBulkInsert( @@ -428,9 +384,6 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern return fmt.Errorf("failed bulk insert checksums: %w", err) } } - if err := sqliteInsertBrowseRowsTx(tx, browseRows); err != nil { - return fmt.Errorf("failed bulk insert browse index: %w", err) - } if err := db.flushObjectUsageEventsForIDsTx(ctx, tx, ids); err != nil { return fmt.Errorf("failed to apply object usage events: %w", err) } @@ -1166,7 +1119,6 @@ func (db *SqliteDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []stri o.created_time, o.updated_time, o.name, - o.file_name, o.version, o.description FROM drs_object o @@ -1182,35 +1134,30 @@ func (db *SqliteDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []stri for rows.Next() { var ( - id string - name, fileName sql.NullString - version, description sql.NullString - size int64 - createdTime, updatedTime time.Time + id string + name sql.NullString + version, description sql.NullString + size int64 + createdTime, updatedTime time.Time ) if err := rows.Scan( - &id, &size, &createdTime, &updatedTime, &name, &fileName, &version, &description, + &id, &size, &createdTime, &updatedTime, &name, &version, &description, ); err != nil { return nil, err } - - loadedName, loadedPath := sqliteLoadedNames(name, fileName) objectsByID[id] = &models.InternalObject{ DrsObject: drs.DrsObject{ Id: id, Size: size, CreatedTime: createdTime, UpdatedTime: common.Ptr(updatedTime), - Name: common.Ptr(loadedName), + Name: common.Ptr(strings.TrimSpace(name.String)), Version: common.Ptr(version.String), Description: common.Ptr(description.String), SelfUri: "drs://" + id, }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(loadedPath) != "" { - objectsByID[id].Properties["path"] = loadedPath - } } if err := rows.Err(); err != nil { diff --git a/internal/db/sqlite/sqlite.go b/internal/db/sqlite/sqlite.go index 0d7fe807..abe5e9f6 100644 --- a/internal/db/sqlite/sqlite.go +++ b/internal/db/sqlite/sqlite.go @@ -60,7 +60,6 @@ func (db *SqliteDB) initSchema() error { created_time TIMESTAMP, updated_time TIMESTAMP, name TEXT, - file_name TEXT, version TEXT, description TEXT )`, @@ -95,18 +94,6 @@ func (db *SqliteDB) initSchema() error { FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE )`, `CREATE INDEX IF NOT EXISTS idx_drs_object_alias_object_id ON drs_object_alias(object_id)`, - `CREATE TABLE IF NOT EXISTS drs_object_browse_index ( - object_id TEXT NOT NULL, - resource TEXT NOT NULL, - normalized_path TEXT NOT NULL, - parent_path TEXT NOT NULL, - entry_name TEXT NOT NULL, - PRIMARY KEY (resource, object_id), - FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE - )`, - `CREATE INDEX IF NOT EXISTS idx_drs_object_browse_index_resource_parent_object_id ON drs_object_browse_index(resource, parent_path, object_id)`, - `CREATE INDEX IF NOT EXISTS idx_drs_object_browse_index_resource_parent_entry_name ON drs_object_browse_index(resource, parent_path, entry_name)`, - `CREATE INDEX IF NOT EXISTS idx_drs_object_browse_index_resource_normalized_path ON drs_object_browse_index(resource, normalized_path)`, `CREATE TABLE IF NOT EXISTS s3_credential ( credential_id TEXT PRIMARY KEY, bucket TEXT NOT NULL, @@ -282,10 +269,11 @@ func (db *SqliteDB) initSchema() error { if err := db.ensureCredentialIdentitySchema(); err != nil { return err } - if _, err := db.db.Exec(`ALTER TABLE drs_object ADD COLUMN file_name TEXT NOT NULL DEFAULT ''`); err != nil { - if !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") { - return err - } + if err := db.ensureObjectTableShape(); err != nil { + return err + } + if _, err := db.db.Exec(`DROP TABLE IF EXISTS drs_object_browse_index`); err != nil { + return err } if _, err := db.db.Exec(`ALTER TABLE transfer_attribution_event ADD COLUMN access_grant_id TEXT NOT NULL DEFAULT ''`); err != nil { if !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") { @@ -306,6 +294,62 @@ func (db *SqliteDB) initSchema() error { return nil } +func (db *SqliteDB) ensureObjectTableShape() error { + rows, err := db.db.Query(`PRAGMA table_info(drs_object)`) + if err != nil { + return err + } + defer rows.Close() + + hasFileName := false + for rows.Next() { + var cid int + var name, typ string + var notNull, pk int + var dflt any + if err := rows.Scan(&cid, &name, &typ, ¬Null, &dflt, &pk); err != nil { + return err + } + if strings.EqualFold(strings.TrimSpace(name), "file_name") { + hasFileName = true + } + } + if err := rows.Err(); err != nil { + return err + } + if !hasFileName { + return nil + } + + tx, err := db.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + for _, stmt := range []string{ + `CREATE TABLE drs_object_new ( + id TEXT PRIMARY KEY, + size INTEGER, + created_time TIMESTAMP, + updated_time TIMESTAMP, + name TEXT, + version TEXT, + description TEXT + )`, + `INSERT INTO drs_object_new (id, size, created_time, updated_time, name, version, description) + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object`, + `DROP TABLE drs_object`, + `ALTER TABLE drs_object_new RENAME TO drs_object`, + } { + if _, err := tx.Exec(stmt); err != nil { + return err + } + } + + return tx.Commit() +} + func (db *SqliteDB) ensureCredentialIdentitySchema() error { rows, err := db.db.Query(`PRAGMA table_info(s3_credential)`) if err != nil { diff --git a/internal/db/sqlite/sqlite_test.go b/internal/db/sqlite/sqlite_test.go index 407bfc67..ca3d9c3d 100644 --- a/internal/db/sqlite/sqlite_test.go +++ b/internal/db/sqlite/sqlite_test.go @@ -243,7 +243,7 @@ func TestSqliteDB_GetObjectsByChecksum_WhenIDDiffers(t *testing.T) { } } -func TestSqliteDB_GetObject_LegacyNameFallsBackToFileName(t *testing.T) { +func TestSqliteDB_GetObjectPreservesStoredName(t *testing.T) { ctx := context.Background() db, err := NewSqliteDB(":memory:") if err != nil { @@ -252,11 +252,11 @@ func TestSqliteDB_GetObject_LegacyNameFallsBackToFileName(t *testing.T) { now := time.Now().UTC() if _, err := db.db.ExecContext(ctx, ` - INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - "legacy-1", int64(42), now, now, "nested/dir/file.txt", nil, "v1", "desc", + INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + "legacy-1", int64(42), now, now, "file.txt", "v1", "desc", ); err != nil { - t.Fatalf("insert legacy object: %v", err) + t.Fatalf("insert object: %v", err) } if _, err := db.db.ExecContext(ctx, ` INSERT INTO drs_object_access_method (object_id, url, type) @@ -271,10 +271,7 @@ func TestSqliteDB_GetObject_LegacyNameFallsBackToFileName(t *testing.T) { t.Fatalf("GetObject failed: %v", err) } if got := common.StringVal(obj.Name); got != "file.txt" { - t.Fatalf("expected basename name, got %q", got) - } - if got, _ := obj.Properties["file_name"].(string); got != "nested/dir/file.txt" { - t.Fatalf("expected legacy path to populate file_name, got %#v", obj.Properties["file_name"]) + t.Fatalf("expected stored name, got %q", got) } } @@ -1426,85 +1423,6 @@ func TestSqliteDB_TransferAttributionByResources(t *testing.T) { } } -func TestSqliteDB_StoragePathMetricsByScope(t *testing.T) { - ctx := context.Background() - db, err := NewSqliteDB(":memory:") - if err != nil { - t.Fatalf("failed to create db: %v", err) - } - now := time.Now().UTC() - for _, obj := range []models.InternalObject{ - { - Authorizations: map[string][]string{"org": {"p1"}}, - DrsObject: drs.DrsObject{ - Id: "obj-1", - Name: common.Ptr("data/a.txt"), - Size: 10, - CreatedTime: now, - UpdatedTime: &now, - }, - }, - { - Authorizations: map[string][]string{"org": {"p1"}}, - DrsObject: drs.DrsObject{ - Id: "obj-2", - Name: common.Ptr("data/a.txt"), - Size: 20, - CreatedTime: now, - UpdatedTime: &now, - }, - }, - { - Authorizations: map[string][]string{"org": {"p1"}}, - DrsObject: drs.DrsObject{ - Id: "obj-3", - Name: common.Ptr("data/nested/b.txt"), - Size: 30, - CreatedTime: now, - UpdatedTime: &now, - }, - }, - } { - if err := db.CreateObject(ctx, &obj); err != nil { - t.Fatalf("CreateObject failed: %v", err) - } - } - if err := db.RecordFileDownload(ctx, "obj-1"); err != nil { - t.Fatalf("RecordFileDownload failed: %v", err) - } - if err := db.RecordFileDownload(ctx, "obj-2"); err != nil { - t.Fatalf("RecordFileDownload failed: %v", err) - } - if err := db.RecordFileDownload(ctx, "obj-2"); err != nil { - t.Fatalf("RecordFileDownload failed: %v", err) - } - - summary, err := db.GetStoragePathSummary(ctx, "org", "p1", "data") - if err != nil { - t.Fatalf("GetStoragePathSummary failed: %v", err) - } - if summary.RecordCount != 3 || summary.FileCount != 2 || summary.TotalBytes != 60 || summary.DuplicatePathCount != 1 || summary.DownloadCount != 3 { - t.Fatalf("unexpected storage summary: %+v", summary) - } - if summary.LastDownloadTime == nil { - t.Fatalf("expected last download time in summary: %+v", summary) - } - - items, err := db.ListStoragePathChildren(ctx, "org", "p1", "data", 10, 0, "bytes", "desc") - if err != nil { - t.Fatalf("ListStoragePathChildren failed: %v", err) - } - if len(items) != 2 { - t.Fatalf("expected 2 storage children, got %+v", items) - } - if items[0].Name != "nested" || items[0].Type != "directory" || items[0].TotalBytes != 30 || items[0].DownloadCount != 0 { - t.Fatalf("unexpected first child: %+v", items[0]) - } - if items[1].Name != "a.txt" || items[1].DownloadCount != 3 || items[1].LastDownloadTime == nil { - t.Fatalf("unexpected second child: %+v", items[1]) - } -} - func TestSqliteDB_ListBucketVisibilityRows(t *testing.T) { ctx := context.Background() db, err := NewSqliteDB(":memory:") @@ -1947,54 +1865,3 @@ func TestSqliteDB_GetPendingLFSMeta(t *testing.T) { t.Fatalf("expected ErrNotFound for missing pending metadata, got: %v", err) } } - -func TestSqliteDB_ListObjectIDsPageByPath(t *testing.T) { - ctx := context.Background() - db, err := NewSqliteDB(":memory:") - if err != nil { - t.Fatalf("failed to create db: %v", err) - } - - now := time.Now().UTC() - for _, obj := range []models.InternalObject{ - { - DrsObject: drs.DrsObject{ - Id: "obj-a", - Name: common.Ptr("nested/a.txt"), - CreatedTime: now, - }, - Authorizations: map[string][]string{"org": {"proj"}}, - }, - { - DrsObject: drs.DrsObject{ - Id: "obj-b", - Name: common.Ptr("nested/deep/b.txt"), - CreatedTime: now, - }, - Authorizations: map[string][]string{"org": {"proj"}}, - }, - { - DrsObject: drs.DrsObject{ - Id: "obj-c", - Name: common.Ptr("nested/z.txt"), - CreatedTime: now, - }, - Authorizations: map[string][]string{"org": {"proj"}}, - }, - } { - if err := db.CreateObject(ctx, &obj); err != nil { - t.Fatalf("CreateObject failed: %v", err) - } - } - - ids, directories, err := db.ListObjectIDsPageByPath(ctx, "org", "proj", "nested", "obj-a", 10, 0) - if err != nil { - t.Fatalf("ListObjectIDsPageByPath failed: %v", err) - } - if !slices.Equal(ids, []string{"obj-c"}) { - t.Fatalf("unexpected IDs: %v", ids) - } - if len(directories) != 1 || directories[0].Path != "nested/deep" { - t.Fatalf("unexpected directories: %+v", directories) - } -} diff --git a/internal/models/object.go b/internal/models/object.go index c03d7a58..cfc07fe0 100644 --- a/internal/models/object.go +++ b/internal/models/object.go @@ -2,7 +2,7 @@ package models import ( "encoding/json" - "strings" + "fmt" "time" "github.com/calypr/syfon/apigen/server/drs" @@ -31,14 +31,18 @@ func (o *InternalObject) UnmarshalJSON(data []byte) error { if raw == nil { raw = map[string]interface{}{} } + if _, ok := raw["file_name"]; ok { + return fmt.Errorf("file_name is no longer supported") + } + if _, ok := raw["path"]; ok { + return fmt.Errorf("path is no longer supported") + } o.Properties = raw type wireObject struct { drs.DrsObject - Did string `json:"did,omitempty"` - Hashes map[string]string `json:"hashes,omitempty"` - Path *string `json:"path,omitempty"` - FileName *string `json:"file_name,omitempty"` + Did string `json:"did,omitempty"` + Hashes map[string]string `json:"hashes,omitempty"` } var wire wireObject @@ -71,11 +75,6 @@ func (o *InternalObject) UnmarshalJSON(data []byte) error { o.MimeType = wire.MimeType o.SelfUri = wire.SelfUri o.Version = wire.Version - if wire.Path != nil && strings.TrimSpace(*wire.Path) != "" { - o.Properties["path"] = strings.TrimSpace(*wire.Path) - } else if wire.FileName != nil && strings.TrimSpace(*wire.FileName) != "" { - o.Properties["path"] = strings.TrimSpace(*wire.FileName) - } if wire.ControlledAccess != nil { o.Authorizations = common.ControlledAccessToAuthzMap(*wire.ControlledAccess) @@ -142,7 +141,7 @@ func (o InternalObject) MarshalJSON() ([]byte, error) { func isRetiredInternalAuthField(key string) bool { switch key { - case "auth", "authz", "authorizations", "urls", "file_name", "path": + case "auth", "authz", "authorizations", "urls": return true default: return false diff --git a/internal/models/object_test.go b/internal/models/object_test.go index d3ecac5c..ac5dd428 100644 --- a/internal/models/object_test.go +++ b/internal/models/object_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func TestInternalObjectUnmarshalLegacyAndMarshalCompatibility(t *testing.T) { +func TestInternalObjectRejectsLegacyPathAliases(t *testing.T) { raw := []byte(`{ "did":"did-1", "file_name":"legacy-name.txt", @@ -14,41 +14,8 @@ func TestInternalObjectUnmarshalLegacyAndMarshalCompatibility(t *testing.T) { "unknown_field":"keep-me" }`) var obj InternalObject - if err := json.Unmarshal(raw, &obj); err != nil { - t.Fatalf("unmarshal failed: %v", err) - } - if obj.Id != "did-1" { - t.Fatalf("expected id from did, got %q", obj.Id) - } - if obj.Name != nil { - t.Fatalf("expected name to remain unset when only file_name is provided, got %+v", obj.Name) - } - if len(obj.Checksums) != 1 || obj.Checksums[0].Type != "sha256" || obj.Checksums[0].Checksum != "abc" { - t.Fatalf("unexpected checksums: %+v", obj.Checksums) - } - - encoded, err := json.Marshal(obj) - if err != nil { - t.Fatalf("marshal failed: %v", err) - } - var out map[string]interface{} - if err := json.Unmarshal(encoded, &out); err != nil { - t.Fatalf("decode marshaled payload: %v", err) - } - if out["did"] != "did-1" { - t.Fatalf("expected did in output, got %v", out["did"]) - } - if _, ok := out["name"]; ok { - t.Fatalf("did not expect name in output when only file_name was provided") - } - if _, ok := out["file_name"]; ok { - t.Fatalf("did not expect file_name in output") - } - if _, ok := out["unknown_field"]; !ok { - t.Fatalf("expected unknown field preservation in output") - } - if _, retired := out["urls"]; retired { - t.Fatalf("expected retired auth fields removed from output") + if err := json.Unmarshal(raw, &obj); err == nil { + t.Fatal("expected file_name payload to be rejected") } } diff --git a/internal/models/storage.go b/internal/models/storage.go index 5ffb1b44..fd193123 100644 --- a/internal/models/storage.go +++ b/internal/models/storage.go @@ -54,50 +54,6 @@ type FileUsageSummary struct { InactiveFileCount int64 } -type StoragePathSummary struct { - Organization string - Project string - Path string - FileCount int64 - RecordCount int64 - TotalBytes int64 - DownloadCount int64 - DirectChildCount int64 - DuplicatePathCount int64 - LatestUpdateTime *time.Time - LastDownloadTime *time.Time -} - -type StoragePathChild struct { - Name string - Path string - Type string - FileCount int64 - RecordCount int64 - TotalBytes int64 - LatestUpdateTime *time.Time - DownloadCount int64 - LastDownloadTime *time.Time -} - -type StoragePathChildrenResponse struct { - Organization string - Project string - Path string - Items []StoragePathChild -} - -// StorageCleanupRecord is the browse-index-backed projection used by storage -// cleanup audit/apply flows. -type StorageCleanupRecord struct { - ObjectID string - NormalizedPath string - Size int64 - UpdatedTime time.Time - DownloadCount int64 - LastDownloadTime *time.Time -} - // BucketVisibilityRow is the minimum storage projection needed to build bucket // visibility responses without hydrating full objects. type BucketVisibilityRow struct { @@ -106,11 +62,6 @@ type BucketVisibilityRow struct { Resource string } -type BrowseDirectory struct { - Name string - Path string -} - const ( TransferEventAccessIssued = "access_issued" @@ -265,7 +216,6 @@ type DrsObjectRecord struct { DownloadCount int64 LastDownloadTime *time.Time Name string - Path string Version string Description string } diff --git a/internal/repair/format.go b/internal/repair/format.go deleted file mode 100644 index 8dfe1486..00000000 --- a/internal/repair/format.go +++ /dev/null @@ -1,89 +0,0 @@ -package repair - -import ( - "encoding/json" - "fmt" - "io" -) - -func WriteAudit(w io.Writer, format string, report Report) error { - if format == "json" { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(report) - } - if _, err := fmt.Fprintf(w, "scanned=%d findings=%d\n", report.Scanned, len(report.Objects)); err != nil { - return err - } - for _, obj := range report.Objects { - if _, err := fmt.Fprintf(w, "%s sha256=%s scope=%s/%s autofix=%t\n", obj.ObjectID, obj.SHA256, obj.Organization, obj.Project, obj.AutoFixable); err != nil { - return err - } - for _, finding := range obj.Findings { - if _, err := fmt.Fprintf(w, " - %s [%s] %s\n", finding.Kind, finding.Severity, finding.Message); err != nil { - return err - } - } - } - return nil -} - -func WriteApply(w io.Writer, format string, result ApplyResult) error { - if format == "json" { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(result) - } - if err := WriteAudit(w, format, result.Report); err != nil { - return err - } - _, err := fmt.Fprintf(w, "mutated=%d skipped=%d auto_fixable=%d\n", result.Mutated, result.Skipped, result.AutoFixable) - return err -} - -func WriteStorageCleanupAudit(w io.Writer, format string, report StorageCleanupReport) error { - if format == "json" { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(report) - } - if _, err := fmt.Fprintf(w, "scope=%s/%s scanned=%d findings=%d\n", report.Organization, report.Project, report.Scanned, len(report.Findings)); err != nil { - return err - } - for _, finding := range report.Findings { - if _, err := fmt.Fprintf(w, "%s [%s] %s action=%s scope=%s\n", finding.NormalizedPath, finding.Kind, finding.Message, finding.RecommendedAction, finding.CleanupScope); err != nil { - return err - } - for _, record := range finding.Records { - if _, err := fmt.Fprintf(w, " - %s storage=%s downloads=%d scope=%s\n", record.ObjectID, record.StorageStatus, record.DownloadCount, record.CleanupScope); err != nil { - return err - } - for _, probe := range record.AccessProbes { - if _, err := fmt.Fprintf(w, " * %s storage=%s bucket=%s error=%s message=%s\n", probe.URL, probe.StorageStatus, probe.Bucket, probe.ErrorKind, probe.StorageMessage); err != nil { - return err - } - } - } - } - return nil -} - -func WriteStorageCleanupApply(w io.Writer, format string, result StorageCleanupApplyResult) error { - if format == "json" { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(result) - } - if err := WriteStorageCleanupAudit(w, format, result.Report); err != nil { - return err - } - if _, err := fmt.Fprintf(w, "dry_run=%t deleted=%d skipped=%d repo_delete_paths=%d\n", result.DryRun, len(result.DeletedRecordIDs), len(result.Skipped), len(result.RepoDeletePaths)); err != nil { - return err - } - for _, skipped := range result.Skipped { - if _, err := fmt.Fprintf(w, " - skipped %s %s: %s\n", skipped.Kind, skipped.ObjectID, skipped.Reason); err != nil { - return err - } - } - return nil -} diff --git a/internal/repair/project_diff_service.go b/internal/repair/project_diff_service.go deleted file mode 100644 index 2747277c..00000000 --- a/internal/repair/project_diff_service.go +++ /dev/null @@ -1,166 +0,0 @@ -package repair - -import ( - "context" - "sort" - "strings" - "time" - - "github.com/calypr/syfon/internal/models" -) - -type projectDiffManager interface { - ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) - ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) -} - -type ProjectDiffService struct { - om projectDiffManager -} - -func NewProjectDiffService(om projectDiffManager) *ProjectDiffService { - return &ProjectDiffService{om: om} -} - -type projectDiffAggregate struct { - path string - objectIDs map[string]bool - recordCount int - sizeBytes int64 - downloadCount int64 - lastDownloadTime *time.Time -} - -func (s *ProjectDiffService) Audit(ctx context.Context, req ProjectDiffAuditRequest) (ProjectDiffReport, error) { - report := ProjectDiffReport{ - Organization: strings.TrimSpace(req.Organization), - Project: strings.TrimSpace(req.Project), - PathPrefix: strings.TrimSpace(req.PathPrefix), - Summary: ProjectDiffSummary{ - CountsByKind: map[FindingKind]int{}, - }, - } - - expected, hasExpected, err := normalizeExpectedPaths(req.ExpectedPaths) - if err != nil { - return ProjectDiffReport{}, err - } - - rows, err := s.loadRows(ctx, report.Organization, report.Project, report.PathPrefix, hasExpected) - if err != nil { - return ProjectDiffReport{}, err - } - - grouped := make(map[string]*projectDiffAggregate) - for _, row := range rows { - state := grouped[row.NormalizedPath] - if state == nil { - state = &projectDiffAggregate{ - path: row.NormalizedPath, - objectIDs: map[string]bool{}, - } - grouped[row.NormalizedPath] = state - } - state.recordCount++ - state.sizeBytes += row.Size - state.downloadCount += row.DownloadCount - state.objectIDs[row.ObjectID] = true - if row.LastDownloadTime != nil { - candidate := row.LastDownloadTime.UTC() - if state.lastDownloadTime == nil || candidate.After(*state.lastDownloadTime) { - state.lastDownloadTime = &candidate - } - } - } - - report.Summary.ScannedRecordCount = len(rows) - report.Summary.IndexedPathCount = len(grouped) - report.Summary.ExpectedPathCount = len(expected) - report.Summary.IncludesRepoManifest = hasExpected - - paths := make([]string, 0, len(grouped)) - for path := range grouped { - paths = append(paths, path) - } - sort.Strings(paths) - - for _, path := range paths { - agg := grouped[path] - if agg == nil { - continue - } - if hasExpected && expected[path] { - report.Summary.MatchedPathCount++ - } - if agg.recordCount > 1 { - report.Findings = append(report.Findings, buildProjectDiffFinding( - FindingDuplicateSyfonPaths, - SeverityWarn, - agg, - "Review duplicate records before deleting anything.", - )) - report.Summary.CountsByKind[FindingDuplicateSyfonPaths]++ - } - if hasExpected && !expected[path] { - report.Findings = append(report.Findings, buildProjectDiffFinding( - FindingSyfonMissingInRepo, - SeverityWarn, - agg, - "Prepare delete to verify storage before removing Syfon-only records.", - )) - report.Summary.CountsByKind[FindingSyfonMissingInRepo]++ - } - } - - if hasExpected { - missing := make([]string, 0) - for path := range expected { - if _, ok := grouped[path]; ok { - continue - } - missing = append(missing, path) - } - sort.Strings(missing) - for _, path := range missing { - report.Findings = append(report.Findings, ProjectDiffFinding{ - Kind: FindingRepoMissingInSyfon, - Severity: SeverityInfo, - NormalizedPath: path, - RecommendedAction: "Review missing Syfon records for this Git-tracked path.", - }) - report.Summary.CountsByKind[FindingRepoMissingInSyfon]++ - } - } - - report.Summary.TotalFindings = len(report.Findings) - if len(report.Summary.CountsByKind) == 0 { - report.Summary.CountsByKind = nil - } - return report, nil -} - -func (s *ProjectDiffService) loadRows(ctx context.Context, organization, project, pathPrefix string, hasExpected bool) ([]models.StorageCleanupRecord, error) { - if hasExpected { - return s.om.ListStorageCleanupRecords(ctx, organization, project, pathPrefix) - } - return s.om.ListDuplicateStorageCleanupRecords(ctx, organization, project, pathPrefix) -} - -func buildProjectDiffFinding(kind FindingKind, severity Severity, agg *projectDiffAggregate, recommendedAction string) ProjectDiffFinding { - objectIDs := make([]string, 0, len(agg.objectIDs)) - for objectID := range agg.objectIDs { - objectIDs = append(objectIDs, objectID) - } - sort.Strings(objectIDs) - return ProjectDiffFinding{ - Kind: kind, - Severity: severity, - NormalizedPath: agg.path, - ObjectIDs: objectIDs, - RecordCount: agg.recordCount, - SizeBytes: agg.sizeBytes, - DownloadCount: agg.downloadCount, - LastDownloadTime: agg.lastDownloadTime, - RecommendedAction: recommendedAction, - } -} diff --git a/internal/repair/project_diff_service_test.go b/internal/repair/project_diff_service_test.go deleted file mode 100644 index 5060d0d2..00000000 --- a/internal/repair/project_diff_service_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package repair - -import ( - "context" - "testing" - "time" - - "github.com/calypr/syfon/internal/models" -) - -func TestProjectDiffAuditWithManifest(t *testing.T) { - lastDownload := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC) - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "dup-1", NormalizedPath: "data/a.tsv", Size: 10, DownloadCount: 2, LastDownloadTime: &lastDownload}, - {ObjectID: "dup-2", NormalizedPath: "data/a.tsv", Size: 10, DownloadCount: 3}, - {ObjectID: "syfon-only", NormalizedPath: "data/b.tsv", Size: 7, DownloadCount: 1}, - }, - } - - report, err := NewProjectDiffService(manager).Audit(context.Background(), ProjectDiffAuditRequest{ - Organization: "org", - Project: "proj", - PathPrefix: "data", - ExpectedPaths: []string{"data/a.tsv", "data/c.tsv"}, - }) - if err != nil { - t.Fatalf("Audit returned error: %v", err) - } - - if manager.listCalls != 1 { - t.Fatalf("expected full subtree listing once, got %d", manager.listCalls) - } - if manager.duplicateListCalls != 0 { - t.Fatalf("expected duplicate-only listing to be skipped, got %d", manager.duplicateListCalls) - } - if got := report.Summary.TotalFindings; got != 3 { - t.Fatalf("expected 3 findings, got %d", got) - } - if got := report.Summary.CountsByKind[FindingDuplicateSyfonPaths]; got != 1 { - t.Fatalf("expected 1 duplicate finding, got %d", got) - } - if got := report.Summary.CountsByKind[FindingSyfonMissingInRepo]; got != 1 { - t.Fatalf("expected 1 syfon-only finding, got %d", got) - } - if got := report.Summary.CountsByKind[FindingRepoMissingInSyfon]; got != 1 { - t.Fatalf("expected 1 repo-only finding, got %d", got) - } - if got := report.Summary.MatchedPathCount; got != 1 { - t.Fatalf("expected 1 matched path, got %d", got) - } - if got := report.Summary.IndexedPathCount; got != 2 { - t.Fatalf("expected 2 indexed paths, got %d", got) - } - if got := report.Summary.ScannedRecordCount; got != 3 { - t.Fatalf("expected 3 scanned records, got %d", got) - } -} - -func TestProjectDiffAuditWithoutManifestUsesDuplicateListing(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "dup-1", NormalizedPath: "data/a.tsv", Size: 10}, - {ObjectID: "dup-2", NormalizedPath: "data/a.tsv", Size: 10}, - {ObjectID: "single", NormalizedPath: "data/b.tsv", Size: 7}, - }, - } - - report, err := NewProjectDiffService(manager).Audit(context.Background(), ProjectDiffAuditRequest{ - Organization: "org", - Project: "proj", - PathPrefix: "data", - }) - if err != nil { - t.Fatalf("Audit returned error: %v", err) - } - - if manager.listCalls != 0 { - t.Fatalf("expected full subtree listing to be skipped, got %d", manager.listCalls) - } - if manager.duplicateListCalls != 1 { - t.Fatalf("expected duplicate-only listing once, got %d", manager.duplicateListCalls) - } - if got := report.Summary.TotalFindings; got != 1 { - t.Fatalf("expected 1 finding, got %d", got) - } - if got := report.Summary.IndexedPathCount; got != 1 { - t.Fatalf("expected 1 indexed path in duplicate mode, got %d", got) - } -} diff --git a/internal/repair/project_diff_types.go b/internal/repair/project_diff_types.go deleted file mode 100644 index 962848f4..00000000 --- a/internal/repair/project_diff_types.go +++ /dev/null @@ -1,46 +0,0 @@ -package repair - -import "time" - -const ( - FindingDuplicateSyfonPaths FindingKind = "duplicate_syfon_paths" - FindingSyfonMissingInRepo FindingKind = "syfon_missing_in_repo" - FindingRepoMissingInSyfon FindingKind = "repo_missing_in_syfon" -) - -type ProjectDiffAuditRequest struct { - Organization string `json:"organization"` - Project string `json:"project"` - PathPrefix string `json:"path_prefix,omitempty"` - ExpectedPaths []string `json:"expected_paths,omitempty"` -} - -type ProjectDiffFinding struct { - Kind FindingKind `json:"kind"` - Severity Severity `json:"severity"` - NormalizedPath string `json:"normalized_path"` - ObjectIDs []string `json:"object_ids,omitempty"` - RecordCount int `json:"record_count,omitempty"` - SizeBytes int64 `json:"size_bytes,omitempty"` - DownloadCount int64 `json:"download_count,omitempty"` - LastDownloadTime *time.Time `json:"last_download_time,omitempty"` - RecommendedAction string `json:"recommended_action,omitempty"` -} - -type ProjectDiffSummary struct { - CountsByKind map[FindingKind]int `json:"counts_by_kind,omitempty"` - TotalFindings int `json:"total_findings"` - IndexedPathCount int `json:"indexed_path_count"` - ExpectedPathCount int `json:"expected_path_count"` - MatchedPathCount int `json:"matched_path_count"` - IncludesRepoManifest bool `json:"includes_repo_manifest"` - ScannedRecordCount int `json:"scanned_record_count"` -} - -type ProjectDiffReport struct { - Organization string `json:"organization"` - Project string `json:"project"` - PathPrefix string `json:"path_prefix,omitempty"` - Summary ProjectDiffSummary `json:"summary"` - Findings []ProjectDiffFinding `json:"findings,omitempty"` -} diff --git a/internal/repair/storage_cleanup_service.go b/internal/repair/storage_cleanup_service.go deleted file mode 100644 index 647e298a..00000000 --- a/internal/repair/storage_cleanup_service.go +++ /dev/null @@ -1,971 +0,0 @@ -package repair - -import ( - "context" - "errors" - "fmt" - "sort" - "strings" - "sync" - "time" - - drsapi "github.com/calypr/syfon/apigen/server/drs" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/core" - "github.com/calypr/syfon/internal/models" - "github.com/google/uuid" -) - -type storageCleanupManager interface { - ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) - ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) - GetBulkObjects(ctx context.Context, ids []string, requiredMethod string) ([]models.InternalObject, error) - InspectStorageObject(ctx context.Context, req core.InspectStorageRequest) (*core.StorageObjectMetadata, error) - DeleteObjectWithOptions(ctx context.Context, id string, opts core.DeleteOptions) error -} - -type StorageCleanupService struct { - om storageCleanupManager -} - -func NewStorageCleanupService(om storageCleanupManager) *StorageCleanupService { - return &StorageCleanupService{om: om} -} - -type storageCleanupPathState struct { - Path string - Records []StorageCleanupRecordAudit - Analysis cleanupStructuralAnalysis -} - -type cleanupURLTemplate string - -const ( - cleanupURLTemplateUnknown cleanupURLTemplate = "unknown" - cleanupURLTemplateLegacyDID cleanupURLTemplate = "legacy_did_keyed" - cleanupURLTemplateProjectScope cleanupURLTemplate = "project_scoped" - cleanupURLTemplateChecksumRoot cleanupURLTemplate = "checksum_root" - cleanupURLTemplateOther cleanupURLTemplate = "other" -) - -type cleanupStructuralAnalysis struct { - ChecksumCount int - SizeCount int - StructuralReason string - LegacyURLTemplateDetected bool - StructurallyEquivalent bool - MixedContentCollision bool -} - -const defaultStorageCleanupProbeConcurrency = 8 - -func (s *StorageCleanupService) Audit(ctx context.Context, req StorageCleanupAuditRequest) (StorageCleanupReport, error) { - checkStorage := true - if req.CheckStorage != nil { - checkStorage = *req.CheckStorage - } - report := StorageCleanupReport{ - Organization: strings.TrimSpace(req.Organization), - Project: strings.TrimSpace(req.Project), - PathPrefix: strings.TrimSpace(req.PathPrefix), - Summary: map[FindingKind]int{}, - } - expected, hasExpected, err := normalizeExpectedPaths(req.ExpectedPaths) - if err != nil { - return StorageCleanupReport{}, err - } - selectedPaths, hasSelectedPaths, err := normalizeExpectedPaths(req.SelectedPaths) - if err != nil { - return StorageCleanupReport{}, err - } - rows, err := s.loadCleanupRows(ctx, report.Organization, report.Project, report.PathPrefix, hasExpected) - if err != nil { - return StorageCleanupReport{}, err - } - if hasSelectedPaths { - filtered := make([]models.StorageCleanupRecord, 0, len(rows)) - for _, row := range rows { - if selectedPaths[row.NormalizedPath] { - filtered = append(filtered, row) - } - } - rows = filtered - } - report.Scanned = len(rows) - - ids := uniqueCleanupIDs(rows) - objects, err := s.om.GetBulkObjects(ctx, ids, "read") - if err != nil { - return StorageCleanupReport{}, err - } - objectByID := make(map[string]models.InternalObject, len(objects)) - for _, obj := range objects { - objectByID[strings.TrimSpace(obj.Id)] = obj - } - - grouped := make(map[string]*storageCleanupPathState) - for _, row := range rows { - state := grouped[row.NormalizedPath] - if state == nil { - state = &storageCleanupPathState{Path: row.NormalizedPath} - grouped[row.NormalizedPath] = state - } - record := StorageCleanupRecordAudit{ - ObjectID: row.ObjectID, - NormalizedPath: row.NormalizedPath, - Size: row.Size, - DownloadCount: row.DownloadCount, - LastDownloadTime: utcTimePtr(row.LastDownloadTime), - UpdatedTime: utcTimeValue(row.UpdatedTime), - } - if obj, ok := objectByID[row.ObjectID]; ok && obj.AccessMethods != nil { - record.CurrentAccessURLs = storageCleanupAccessMethodURLs(*obj.AccessMethods) - } - record.StorageStatus = StorageProbeStatusUnknown - state.Records = append(state.Records, record) - } - - if checkStorage { - if err := s.populateProbeStates(ctx, report.Organization, report.Project, grouped); err != nil { - return StorageCleanupReport{}, err - } - } - - paths := make([]string, 0, len(grouped)) - for path := range grouped { - paths = append(paths, path) - } - sort.Strings(paths) - report.ScannedPaths = len(paths) - - for _, path := range paths { - state := grouped[path] - sort.Slice(state.Records, func(i, j int) bool { - return state.Records[i].ObjectID < state.Records[j].ObjectID - }) - state.Analysis = analyzeCleanupStructuralDuplicates(state, objectByID) - findings := classifyStorageCleanupPath(state, hasExpected, expected[state.Path]) - if len(findings) == 0 && len(state.Records) > 1 { - findings = append(findings, newDuplicateStorageProbeErrorFinding(state)) - } - if len(findings) == 0 { - report.UnclassifiedPaths++ - continue - } - report.ClassifiedPaths++ - for _, finding := range findings { - report.Summary[finding.Kind]++ - report.Findings = append(report.Findings, finding) - } - } - if len(report.Summary) == 0 { - report.Summary = nil - } - return report, nil -} - -func (s *StorageCleanupService) loadCleanupRows(ctx context.Context, organization, project, pathPrefix string, hasExpected bool) ([]models.StorageCleanupRecord, error) { - if hasExpected { - return s.om.ListStorageCleanupRecords(ctx, organization, project, pathPrefix) - } - return s.om.ListDuplicateStorageCleanupRecords(ctx, organization, project, pathPrefix) -} - -func (s *StorageCleanupService) Apply(ctx context.Context, req StorageCleanupApplyRequest) (StorageCleanupApplyResult, error) { - checkStorage := req.CheckStorage - report, err := s.Audit(ctx, StorageCleanupAuditRequest{ - Organization: req.Organization, - Project: req.Project, - PathPrefix: req.PathPrefix, - ExpectedPaths: req.ExpectedPaths, - SelectedPaths: req.SelectedPaths, - CheckStorage: checkStorage, - }) - if err != nil { - return StorageCleanupApplyResult{}, err - } - result := StorageCleanupApplyResult{Report: report, DryRun: req.DryRun} - actions, repoPaths := buildStorageCleanupActions(report, req) - result.RepoDeletePaths = repoPaths - if req.DryRun { - return result, nil - } - ids := make([]string, 0, len(actions)) - for id := range actions { - ids = append(ids, id) - } - sort.Strings(ids) - for _, objectID := range ids { - action := actions[objectID] - if err := s.om.DeleteObjectWithOptions(ctx, objectID, core.DeleteOptions{DeleteStorageData: action.DeleteStorageData}); err != nil { - result.Skipped = append(result.Skipped, StorageCleanupSkipped{ - NormalizedPath: action.Path, - ObjectID: objectID, - Kind: action.Kind, - Reason: err.Error(), - }) - if action.DeleteStorageData { - result.StoragePurgeResults = append(result.StoragePurgeResults, StorageCleanupStoragePurgeResult{ - ObjectID: objectID, - Purged: false, - Message: err.Error(), - }) - } - continue - } - result.DeletedRecordIDs = append(result.DeletedRecordIDs, objectID) - if action.DeleteStorageData { - result.StoragePurgeResults = append(result.StoragePurgeResults, StorageCleanupStoragePurgeResult{ - ObjectID: objectID, - Purged: true, - }) - } - } - return result, nil -} - -type cleanupProbeResult struct { - Status StorageProbeStatus - Message string - CleanupScope CleanupScope - HasBrokenAccess bool - Probes []StorageCleanupAccessProbe -} - -type storageCleanupProbeTask struct { - recordIdx int - urlIdx int - rawURL string -} - -type storageCleanupProbeResponse struct { - recordIdx int - urlIdx int - probe StorageCleanupAccessProbe -} - -type storageCleanupRecordSlot struct { - state *storageCleanupPathState - index int -} - -type storageCleanupAuditContext struct { - om storageCleanupManager - organization string - project string - - mu sync.Mutex - urlCache map[string]StorageCleanupAccessProbe - bucketErrCache map[string]StorageCleanupAccessProbe - bucketInFlight map[string]chan struct{} -} - -func (s *StorageCleanupService) populateProbeStates(ctx context.Context, organization, project string, grouped map[string]*storageCleanupPathState) error { - ctx = core.WithStorageInspectCache(ctx) - recordSlots := make([]storageCleanupRecordSlot, 0) - tasks := make([]storageCleanupProbeTask, 0) - for _, path := range sortedCleanupPaths(grouped) { - state := grouped[path] - for i := range state.Records { - recordIdx := len(recordSlots) - recordSlots = append(recordSlots, storageCleanupRecordSlot{state: state, index: i}) - urls := state.Records[i].CurrentAccessURLs - for urlIdx, raw := range urls { - raw = strings.TrimSpace(raw) - if raw == "" { - continue - } - tasks = append(tasks, storageCleanupProbeTask{recordIdx: recordIdx, urlIdx: urlIdx, rawURL: raw}) - } - } - } - if len(recordSlots) == 0 { - return nil - } - accumulators := make([][]StorageCleanupAccessProbe, len(recordSlots)) - for i, slot := range recordSlots { - accumulators[i] = make([]StorageCleanupAccessProbe, len(slot.state.Records[slot.index].CurrentAccessURLs)) - } - auditCtx := &storageCleanupAuditContext{ - om: s.om, - organization: organization, - project: project, - urlCache: map[string]StorageCleanupAccessProbe{}, - bucketErrCache: map[string]StorageCleanupAccessProbe{}, - bucketInFlight: map[string]chan struct{}{}, - } - if err := auditCtx.runProbes(ctx, tasks, accumulators); err != nil { - return err - } - for i, slot := range recordSlots { - record := &slot.state.Records[slot.index] - probeState := deriveCleanupProbeResult(accumulators[i]) - record.AccessProbes = probeState.Probes - record.StorageStatus = probeState.Status - record.StorageMessage = probeState.Message - record.CleanupScope = probeState.CleanupScope - } - return nil -} - -func (a *storageCleanupAuditContext) runProbes(ctx context.Context, tasks []storageCleanupProbeTask, accumulators [][]StorageCleanupAccessProbe) error { - if len(tasks) == 0 { - return nil - } - workerCount := defaultStorageCleanupProbeConcurrency - if workerCount > len(tasks) { - workerCount = len(tasks) - } - jobs := make(chan storageCleanupProbeTask) - results := make(chan storageCleanupProbeResponse, len(tasks)) - errCh := make(chan error, 1) - var wg sync.WaitGroup - - for i := 0; i < workerCount; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - case <-ctx.Done(): - select { - case errCh <- ctx.Err(): - default: - } - return - case task, ok := <-jobs: - if !ok { - return - } - probe, err := a.probeURL(ctx, task.rawURL) - if err != nil { - select { - case errCh <- err: - default: - } - return - } - select { - case <-ctx.Done(): - select { - case errCh <- ctx.Err(): - default: - } - return - case results <- storageCleanupProbeResponse{recordIdx: task.recordIdx, urlIdx: task.urlIdx, probe: probe}: - } - } - } - }() - } - - go func() { - defer close(jobs) - for _, task := range tasks { - select { - case <-ctx.Done(): - return - case jobs <- task: - } - } - }() - - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) - }() - - received := 0 - for received < len(tasks) { - select { - case err := <-errCh: - if err != nil { - <-done - return err - } - case <-ctx.Done(): - <-done - return ctx.Err() - case res := <-results: - accumulators[res.recordIdx][res.urlIdx] = res.probe - received++ - } - } - <-done - return nil -} - -func (a *storageCleanupAuditContext) probeURL(ctx context.Context, raw string) (StorageCleanupAccessProbe, error) { - raw = strings.TrimSpace(raw) - probe := StorageCleanupAccessProbe{URL: raw} - if raw == "" { - return probe, nil - } - if bucket, _, ok := common.ParseS3URL(raw); ok { - probe.Bucket = bucket - } - - var ( - bucketLeader bool - releaseWait func() - ) - for { - a.mu.Lock() - if cached, ok := a.urlCache[raw]; ok { - a.mu.Unlock() - return cached, nil - } - if probe.Bucket != "" { - if cached, ok := a.bucketErrCache[probe.Bucket]; ok { - cached.URL = raw - cached.Bucket = probe.Bucket - a.urlCache[raw] = cached - a.mu.Unlock() - return cached, nil - } - if !bucketLeader { - if waiter, ok := a.bucketInFlight[probe.Bucket]; ok { - a.mu.Unlock() - select { - case <-ctx.Done(): - return StorageCleanupAccessProbe{}, ctx.Err() - case <-waiter: - continue - } - } - waiter := make(chan struct{}) - a.bucketInFlight[probe.Bucket] = waiter - bucketLeader = true - releaseWait = func() { - a.mu.Lock() - delete(a.bucketInFlight, probe.Bucket) - close(waiter) - a.mu.Unlock() - } - } - } - a.mu.Unlock() - break - } - if releaseWait != nil { - defer releaseWait() - } - - if err := ctx.Err(); err != nil { - return StorageCleanupAccessProbe{}, err - } - - _, err := a.om.InspectStorageObject(ctx, core.InspectStorageRequest{ - Organization: a.organization, - Project: a.project, - ObjectURL: raw, - }) - if err == nil { - probe.StorageStatus = StorageProbeStatusLive - a.cacheProbe(raw, probe) - return probe, nil - } - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return StorageCleanupAccessProbe{}, err - } - if inspectErr, ok := err.(*core.StorageInspectError); ok { - probe.ErrorKind = string(inspectErr.Kind) - probe.StorageMessage = inspectErr.Error() - switch inspectErr.Kind { - case core.StorageInspectObjectNotFound: - probe.StorageStatus = StorageProbeStatusMissing - default: - probe.StorageStatus = StorageProbeStatusError - if probe.Bucket != "" && cacheableBucketInspectKind(inspectErr.Kind) { - a.cacheBucketError(probe.Bucket, probe) - } - } - a.cacheProbe(raw, probe) - return probe, nil - } - probe.StorageStatus = StorageProbeStatusError - probe.StorageMessage = err.Error() - a.cacheProbe(raw, probe) - return probe, nil -} - -func (a *storageCleanupAuditContext) cacheProbe(raw string, probe StorageCleanupAccessProbe) { - a.mu.Lock() - defer a.mu.Unlock() - a.urlCache[raw] = probe -} - -func (a *storageCleanupAuditContext) cacheBucketError(bucket string, probe StorageCleanupAccessProbe) { - a.mu.Lock() - defer a.mu.Unlock() - a.bucketErrCache[bucket] = probe -} - -func cacheableBucketInspectKind(kind core.StorageInspectErrorKind) bool { - switch kind { - case core.StorageInspectCredentialMissing, core.StorageInspectPermissionDenied, core.StorageInspectUnsupported: - return true - default: - return false - } -} - -func deriveCleanupProbeResult(probes []StorageCleanupAccessProbe) cleanupProbeResult { - if len(probes) == 0 { - return cleanupProbeResult{ - Status: StorageProbeStatusUnknown, - Message: "record has no access URLs", - CleanupScope: CleanupScopeRecord, - } - } - - var ( - liveCount int - missingCount int - errorCount int - usableProbeSeen bool - lastMessage string - filtered = make([]StorageCleanupAccessProbe, 0, len(probes)) - ) - for _, probe := range probes { - if strings.TrimSpace(probe.URL) == "" { - continue - } - filtered = append(filtered, probe) - switch probe.StorageStatus { - case StorageProbeStatusLive: - liveCount++ - usableProbeSeen = true - case StorageProbeStatusMissing: - missingCount++ - usableProbeSeen = true - lastMessage = probe.StorageMessage - case StorageProbeStatusError: - errorCount++ - usableProbeSeen = true - lastMessage = probe.StorageMessage - } - } - if len(filtered) == 0 { - return cleanupProbeResult{ - Status: StorageProbeStatusUnknown, - Message: "record has no usable access URLs", - CleanupScope: CleanupScopeRecord, - } - } - - result := cleanupProbeResult{Probes: filtered} - switch { - case liveCount > 0: - result.Status = StorageProbeStatusLive - if errorCount > 0 { - result.CleanupScope = CleanupScopeAccessURL - result.Message = "record has one or more broken access URLs" - result.HasBrokenAccess = true - } - case missingCount > 0 && errorCount == 0: - result.Status = StorageProbeStatusMissing - result.CleanupScope = CleanupScopeRecord - result.Message = "storage object not found" - case errorCount > 0: - result.Status = StorageProbeStatusError - result.CleanupScope = CleanupScopeRecord - result.Message = lastMessage - result.HasBrokenAccess = true - case usableProbeSeen: - result.Status = StorageProbeStatusUnknown - result.CleanupScope = CleanupScopeRecord - result.Message = "record could not be classified from storage probes" - default: - result.Status = StorageProbeStatusUnknown - result.CleanupScope = CleanupScopeRecord - result.Message = "record has no usable access URLs" - } - return result -} - -func classifyStorageCleanupPath(state *storageCleanupPathState, hasExpected, inExpectedSet bool) []StorageCleanupFinding { - liveCount, missingCount, errorCount, unknownCount := 0, 0, 0, 0 - hasBrokenAccess := false - for _, rec := range state.Records { - switch rec.StorageStatus { - case StorageProbeStatusLive: - liveCount++ - case StorageProbeStatusMissing: - missingCount++ - case StorageProbeStatusError: - errorCount++ - case StorageProbeStatusUnknown: - unknownCount++ - } - if recordHasBrokenAccess(rec) { - hasBrokenAccess = true - } - } - findings := make([]StorageCleanupFinding, 0, 3) - if len(state.Records) > 1 { - switch { - case liveCount > 1: - findings = append(findings, withStructuralContext(StorageCleanupFinding{ - Kind: FindingLiveDuplicateConflict, - Severity: SeverityWarn, - NormalizedPath: state.Path, - Message: fmt.Sprintf("multiple records for %s still resolve in storage", state.Path), - RecommendedAction: "manual_review", - Records: cloneCleanupRecords(state.Records), - }, state.Analysis)) - case liveCount >= 1 && missingCount >= 1: - findings = append(findings, withStructuralContext(StorageCleanupFinding{ - Kind: FindingStaleDuplicateRecord, - Severity: SeverityWarn, - NormalizedPath: state.Path, - Message: fmt.Sprintf("duplicate path %s has stale records that no longer resolve", state.Path), - RecommendedAction: "delete_stale_duplicates", - Records: cloneCleanupRecords(state.Records), - }, state.Analysis)) - case errorCount > 0 && !hasBrokenAccess: - findings = append(findings, newDuplicateStorageProbeErrorFinding(state)) - case unknownCount > 0 || (errorCount > 0 && hasBrokenAccess) || (liveCount > 0 && unknownCount > 0) || (missingCount > 0 && unknownCount > 0): - findings = append(findings, newDuplicateStorageProbeErrorFinding(state)) - } - } - if hasBrokenAccess { - findings = append(findings, withStructuralContext(StorageCleanupFinding{ - Kind: FindingBrokenAccessURLError, - Severity: SeverityWarn, - NormalizedPath: state.Path, - Message: fmt.Sprintf("path %s includes one or more broken access URLs", state.Path), - RecommendedAction: "manual_review", - CleanupScope: cleanupScopeForFinding(state.Records), - Records: cloneCleanupRecords(filterCleanupRecordsWithBrokenAccess(state.Records)), - }, state.Analysis)) - } - if hasExpected && !inExpectedSet { - switch { - case liveCount > 0: - findings = append(findings, withStructuralContext(StorageCleanupFinding{ - Kind: FindingRepoOrphanLiveObject, - Severity: SeverityWarn, - NormalizedPath: state.Path, - Message: fmt.Sprintf("path %s is absent from expected repo contents but still resolves in storage", state.Path), - RecommendedAction: "delete_repo_orphans", - RepoDeleteCandidate: true, - Records: cloneCleanupRecords(state.Records), - }, state.Analysis)) - case missingCount > 0 && errorCount == 0: - findings = append(findings, withStructuralContext(StorageCleanupFinding{ - Kind: FindingRepoOrphanStaleRecord, - Severity: SeverityWarn, - NormalizedPath: state.Path, - Message: fmt.Sprintf("path %s is absent from expected repo contents and storage is already missing", state.Path), - RecommendedAction: "delete_repo_orphans", - RepoDeleteCandidate: true, - Records: cloneCleanupRecords(state.Records), - }, state.Analysis)) - case errorCount > 0: - findings = append(findings, withStructuralContext(StorageCleanupFinding{ - Kind: FindingStorageProbeError, - Severity: SeverityWarn, - NormalizedPath: state.Path, - Message: fmt.Sprintf("path %s is absent from expected repo contents but storage verification failed", state.Path), - RecommendedAction: "manual_review", - RepoDeleteCandidate: true, - CleanupScope: cleanupScopeForFinding(state.Records), - Records: cloneCleanupRecords(state.Records), - }, state.Analysis)) - } - } - return findings -} - -type storageCleanupAction struct { - Path string - Kind FindingKind - DeleteStorageData bool -} - -func buildStorageCleanupActions(report StorageCleanupReport, req StorageCleanupApplyRequest) (map[string]storageCleanupAction, []string) { - pathFilter := make(map[string]bool, len(req.SelectedPaths)) - for _, path := range req.SelectedPaths { - if normalized, _, err := common.NormalizeBrowsePath(path); err == nil { - pathFilter[normalized] = true - } - } - objectFilter := make(map[string]bool, len(req.SelectedObjectIDs)) - for _, objectID := range req.SelectedObjectIDs { - objectID = strings.TrimSpace(objectID) - if objectID != "" { - objectFilter[objectID] = true - } - } - kindFilter := make(map[FindingKind]bool, len(req.SelectedFindingKinds)) - for _, kind := range req.SelectedFindingKinds { - kindFilter[kind] = true - } - repoPaths := make(map[string]bool) - actions := map[string]storageCleanupAction{} - for _, finding := range report.Findings { - if len(pathFilter) > 0 && !pathFilter[finding.NormalizedPath] { - continue - } - if len(kindFilter) > 0 && !kindFilter[finding.Kind] { - continue - } - switch finding.Kind { - case FindingStaleDuplicateRecord: - if !req.DeleteStaleDuplicates { - continue - } - for _, rec := range finding.Records { - if rec.StorageStatus != StorageProbeStatusMissing { - continue - } - if len(objectFilter) > 0 && !objectFilter[rec.ObjectID] { - continue - } - if rec.CleanupScope == CleanupScopeAccessURL { - continue - } - actions[rec.ObjectID] = storageCleanupAction{Path: finding.NormalizedPath, Kind: finding.Kind} - } - case FindingRepoOrphanLiveObject, FindingRepoOrphanStaleRecord: - if !req.DeleteRepoOrphans { - continue - } - repoPaths[finding.NormalizedPath] = true - for _, rec := range finding.Records { - if len(objectFilter) > 0 && !objectFilter[rec.ObjectID] { - continue - } - if rec.CleanupScope == CleanupScopeAccessURL { - continue - } - switch rec.StorageStatus { - case StorageProbeStatusLive: - actions[rec.ObjectID] = storageCleanupAction{Path: finding.NormalizedPath, Kind: finding.Kind, DeleteStorageData: true} - case StorageProbeStatusMissing: - if current, ok := actions[rec.ObjectID]; !ok || !current.DeleteStorageData { - actions[rec.ObjectID] = storageCleanupAction{Path: finding.NormalizedPath, Kind: finding.Kind} - } - } - } - } - } - outPaths := make([]string, 0, len(repoPaths)) - for path := range repoPaths { - outPaths = append(outPaths, path) - } - sort.Strings(outPaths) - return actions, outPaths -} - -func normalizeExpectedPaths(values []string) (map[string]bool, bool, error) { - if len(values) == 0 { - return nil, false, nil - } - out := make(map[string]bool, len(values)) - for _, raw := range values { - normalized, _, err := common.NormalizeBrowsePath(raw) - if err != nil { - return nil, false, err - } - out[normalized] = true - } - return out, true, nil -} - -func uniqueCleanupIDs(rows []models.StorageCleanupRecord) []string { - seen := make(map[string]bool, len(rows)) - out := make([]string, 0, len(rows)) - for _, row := range rows { - if seen[row.ObjectID] { - continue - } - seen[row.ObjectID] = true - out = append(out, row.ObjectID) - } - sort.Strings(out) - return out -} - -func sortedCleanupPaths(grouped map[string]*storageCleanupPathState) []string { - paths := make([]string, 0, len(grouped)) - for path := range grouped { - paths = append(paths, path) - } - sort.Strings(paths) - return paths -} - -func storageCleanupAccessMethodURLs(methods []drsapi.AccessMethod) []string { - out := make([]string, 0, len(methods)) - for _, method := range methods { - if method.AccessUrl == nil { - continue - } - raw := strings.TrimSpace(method.AccessUrl.Url) - if raw != "" { - out = append(out, raw) - } - } - return out -} - -func analyzeCleanupStructuralDuplicates(state *storageCleanupPathState, objectByID map[string]models.InternalObject) cleanupStructuralAnalysis { - checksumSet := map[string]bool{} - sizeSet := map[int64]bool{} - templateSet := map[cleanupURLTemplate]bool{} - legacyDetected := false - for _, record := range state.Records { - sizeSet[record.Size] = true - if obj, ok := objectByID[record.ObjectID]; ok { - if checksum, ok := common.CanonicalSHA256(obj.Checksums); ok && checksum != "" { - checksumSet[strings.ToLower(strings.TrimSpace(checksum))] = true - for _, rawURL := range record.CurrentAccessURLs { - tpl := detectCleanupURLTemplate(rawURL, checksum) - templateSet[tpl] = true - if tpl == cleanupURLTemplateLegacyDID { - legacyDetected = true - } - } - continue - } - } - for _, rawURL := range record.CurrentAccessURLs { - tpl := detectCleanupURLTemplate(rawURL, "") - templateSet[tpl] = true - if tpl == cleanupURLTemplateLegacyDID { - legacyDetected = true - } - } - } - analysis := cleanupStructuralAnalysis{ - ChecksumCount: len(checksumSet), - SizeCount: len(sizeSet), - LegacyURLTemplateDetected: legacyDetected, - } - sameChecksum := len(checksumSet) == 1 && len(state.Records) > 1 - sameSize := len(sizeSet) == 1 && len(state.Records) > 1 - multipleTemplates := len(templateSet) > 1 - switch { - case sameChecksum && sameSize && legacyDetected && multipleTemplates: - analysis.StructurallyEquivalent = true - analysis.StructuralReason = "same_checksum_same_size_legacy_url_mismatch" - case sameChecksum && sameSize && multipleTemplates: - analysis.StructurallyEquivalent = true - analysis.StructuralReason = "same_checksum_same_size_url_template_mismatch" - case sameChecksum && sameSize: - analysis.StructurallyEquivalent = true - analysis.StructuralReason = "same_checksum_same_size" - case len(checksumSet) > 1 || len(sizeSet) > 1: - analysis.MixedContentCollision = true - analysis.StructuralReason = "mixed_checksum_or_size_duplicates" - case len(checksumSet) == 0: - analysis.StructuralReason = "duplicate_path_no_checksum_evidence" - default: - analysis.StructuralReason = "duplicate_path_structural_mismatch" - } - return analysis -} - -func detectCleanupURLTemplate(rawURL string, checksum string) cleanupURLTemplate { - _, key, ok := common.ParseS3URL(strings.TrimSpace(rawURL)) - if !ok { - return cleanupURLTemplateUnknown - } - segments := make([]string, 0) - for _, segment := range strings.Split(strings.Trim(key, "/"), "/") { - segment = strings.TrimSpace(segment) - if segment != "" { - segments = append(segments, segment) - } - } - if len(segments) == 0 { - return cleanupURLTemplateUnknown - } - last := strings.TrimSpace(segments[len(segments)-1]) - if checksum != "" && !strings.EqualFold(last, checksum) { - return cleanupURLTemplateOther - } - if len(segments) >= 2 { - if _, err := uuid.Parse(strings.TrimSpace(segments[0])); err == nil { - return cleanupURLTemplateLegacyDID - } - return cleanupURLTemplateProjectScope - } - if checksum != "" && strings.EqualFold(segments[0], checksum) { - return cleanupURLTemplateChecksumRoot - } - return cleanupURLTemplateOther -} - -func withStructuralContext(finding StorageCleanupFinding, analysis cleanupStructuralAnalysis) StorageCleanupFinding { - finding.StructuralReason = analysis.StructuralReason - finding.ChecksumCount = analysis.ChecksumCount - finding.SizeCount = analysis.SizeCount - finding.LegacyURLTemplateDetected = analysis.LegacyURLTemplateDetected - return finding -} - -func newDuplicateStorageProbeErrorFinding(state *storageCleanupPathState) StorageCleanupFinding { - return withStructuralContext(StorageCleanupFinding{ - Kind: FindingStorageProbeError, - Severity: SeverityWarn, - NormalizedPath: state.Path, - Message: fmt.Sprintf("duplicate path %s could not be fully classified after storage verification", state.Path), - RecommendedAction: "manual_review", - CleanupScope: cleanupScopeForFinding(state.Records), - Records: cloneCleanupRecords(state.Records), - }, state.Analysis) -} - -func cloneCleanupRecords(in []StorageCleanupRecordAudit) []StorageCleanupRecordAudit { - out := make([]StorageCleanupRecordAudit, len(in)) - copy(out, in) - return out -} - -func filterCleanupRecordsWithBrokenAccess(in []StorageCleanupRecordAudit) []StorageCleanupRecordAudit { - out := make([]StorageCleanupRecordAudit, 0, len(in)) - for _, record := range in { - if recordHasBrokenAccess(record) { - out = append(out, record) - } - } - return out -} - -func recordHasBrokenAccess(record StorageCleanupRecordAudit) bool { - for _, probe := range record.AccessProbes { - if probe.StorageStatus == StorageProbeStatusError { - return true - } - } - return false -} - -func cleanupScopeForFinding(records []StorageCleanupRecordAudit) CleanupScope { - for _, record := range records { - if record.CleanupScope == CleanupScopeAccessURL { - return CleanupScopeAccessURL - } - } - return CleanupScopeRecord -} - -func utcTimeValue(v time.Time) *time.Time { - if v.IsZero() { - return nil - } - t := v.UTC() - return &t -} - -func utcTimePtr(v *time.Time) *time.Time { - if v == nil { - return nil - } - t := v.UTC() - return &t -} diff --git a/internal/repair/storage_cleanup_service_test.go b/internal/repair/storage_cleanup_service_test.go deleted file mode 100644 index 8d3789b5..00000000 --- a/internal/repair/storage_cleanup_service_test.go +++ /dev/null @@ -1,663 +0,0 @@ -package repair - -import ( - "context" - "testing" - - drsapi "github.com/calypr/syfon/apigen/server/drs" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/core" - "github.com/calypr/syfon/internal/models" -) - -func TestStorageCleanupAuditStaleDuplicateRecord(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-live", NormalizedPath: "data/file.tsv", Size: 10}, - {ObjectID: "obj-missing", NormalizedPath: "data/file.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "obj-live": cleanupObjectWithChecksum( - "obj-live", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "s3://bucket/project/path/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - ), - "obj-missing": cleanupObjectWithChecksum( - "obj-missing", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "s3://bucket/11111111-1111-1111-1111-111111111111/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - ), - }, - inspectByURL: map[string]error{ - "s3://bucket/project/path/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": nil, - "s3://bucket/11111111-1111-1111-1111-111111111111/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, - }, - } - - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - ExpectedPaths: []string{ - "data/file.tsv", - }, - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if len(report.Findings) != 1 { - t.Fatalf("expected one finding, got %d", len(report.Findings)) - } - if report.Findings[0].Kind != FindingStaleDuplicateRecord { - t.Fatalf("expected stale duplicate finding, got %s", report.Findings[0].Kind) - } - if report.Findings[0].StructuralReason != "same_checksum_same_size_legacy_url_mismatch" { - t.Fatalf("unexpected structural reason: %q", report.Findings[0].StructuralReason) - } - if !report.Findings[0].LegacyURLTemplateDetected { - t.Fatal("expected legacy url template detection") - } -} - -func TestStorageCleanupAuditLiveDuplicateConflict(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-1", NormalizedPath: "data/file.tsv", Size: 10}, - {ObjectID: "obj-2", NormalizedPath: "data/file.tsv", Size: 11}, - }, - objects: map[string]models.InternalObject{ - "obj-1": cleanupObject("obj-1", "s3://bucket/object-1/file.tsv"), - "obj-2": cleanupObject("obj-2", "s3://bucket/object-2/file.tsv"), - }, - } - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{Organization: "org", Project: "proj"}) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if len(report.Findings) != 1 || report.Findings[0].Kind != FindingLiveDuplicateConflict { - t.Fatalf("expected live duplicate conflict, got %+v", report.Findings) - } -} - -func TestStorageCleanupAuditDuplicateWithoutAccessMethodsEmitsProbeError(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-1", NormalizedPath: "data/file.tsv", Size: 10}, - {ObjectID: "obj-2", NormalizedPath: "data/file.tsv", Size: 11}, - }, - objects: map[string]models.InternalObject{ - "obj-1": cleanupObject("obj-1"), - "obj-2": cleanupObject("obj-2"), - }, - } - - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if report.Scanned != 2 || report.ScannedPaths != 1 || report.ClassifiedPaths != 1 || report.UnclassifiedPaths != 0 { - t.Fatalf("unexpected counters: %+v", report) - } - if len(report.Findings) != 1 || report.Findings[0].Kind != FindingStorageProbeError { - t.Fatalf("expected one storage probe error, got %+v", report.Findings) - } - for _, record := range report.Findings[0].Records { - if record.StorageStatus != StorageProbeStatusUnknown { - t.Fatalf("expected unknown status, got %s", record.StorageStatus) - } - if record.StorageMessage != "record has no access URLs" { - t.Fatalf("unexpected storage message: %q", record.StorageMessage) - } - } -} - -func TestStorageCleanupAuditDuplicateWithOneLiveAndOneUnknownEmitsProbeError(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-live", NormalizedPath: "data/file.tsv", Size: 10}, - {ObjectID: "obj-unknown", NormalizedPath: "data/file.tsv", Size: 11}, - }, - objects: map[string]models.InternalObject{ - "obj-live": cleanupObject("obj-live", "s3://bucket/live.tsv"), - "obj-unknown": cleanupObject("obj-unknown"), - }, - inspectByURL: map[string]error{ - "s3://bucket/live.tsv": nil, - }, - } - - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if len(report.Findings) != 1 || report.Findings[0].Kind != FindingStorageProbeError { - t.Fatalf("expected one storage probe error, got %+v", report.Findings) - } -} - -func TestStorageCleanupAuditDuplicateWithEmptyAccessURLsEmitsProbeError(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-1", NormalizedPath: "data/file.tsv", Size: 10}, - {ObjectID: "obj-2", NormalizedPath: "data/file.tsv", Size: 11}, - }, - objects: map[string]models.InternalObject{ - "obj-1": cleanupObject("obj-1", ""), - "obj-2": cleanupObject("obj-2", " "), - }, - } - - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if len(report.Findings) != 1 || report.Findings[0].Kind != FindingStorageProbeError { - t.Fatalf("expected one storage probe error, got %+v", report.Findings) - } - for _, record := range report.Findings[0].Records { - if record.StorageStatus != StorageProbeStatusUnknown { - t.Fatalf("expected unknown status, got %s", record.StorageStatus) - } - if record.StorageMessage != "record has no access URLs" { - t.Fatalf("unexpected storage message: %q", record.StorageMessage) - } - } -} - -func TestStorageCleanupAuditBrokenAccessURLDuplicateDoesNotAbort(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-live", NormalizedPath: "data/file.tsv", Size: 10}, - {ObjectID: "obj-broken", NormalizedPath: "data/file.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "obj-live": cleanupObjectWithChecksum( - "obj-live", - "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "s3://bucket/project/path/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - ), - "obj-broken": cleanupObjectWithChecksum( - "obj-broken", - "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "s3://old-bucket/22222222-2222-2222-2222-222222222222/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - ), - }, - inspectByURL: map[string]error{ - "s3://bucket/project/path/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": nil, - "s3://old-bucket/22222222-2222-2222-2222-222222222222/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": &core.StorageInspectError{ - Kind: core.StorageInspectCredentialMissing, - Message: "no stored bucket credential found for bucket \"old-bucket\"", - }, - }, - } - - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - ExpectedPaths: []string{"data/file.tsv"}, - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if len(report.Findings) != 2 { - t.Fatalf("expected two findings, got %+v", report.Findings) - } - if report.Findings[0].Kind != FindingStorageProbeError || report.Findings[1].Kind != FindingBrokenAccessURLError { - t.Fatalf("unexpected finding kinds: %+v", report.Findings) - } - finding := report.Findings[1] - if finding.Kind != FindingBrokenAccessURLError { - t.Fatalf("expected broken access url finding, got %s", finding.Kind) - } - if len(finding.Records) != 1 || finding.Records[0].ObjectID != "obj-broken" { - t.Fatalf("expected only broken record in finding, got %+v", finding.Records) - } - if finding.CleanupScope != CleanupScopeRecord { - t.Fatalf("expected record cleanup scope, got %s", finding.CleanupScope) - } - if finding.StructuralReason != "same_checksum_same_size_legacy_url_mismatch" { - t.Fatalf("unexpected structural reason: %q", finding.StructuralReason) - } -} - -func TestStorageCleanupAuditMixedMissingAndBrokenDuplicateEmitsBothFindings(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-live", NormalizedPath: "data/file.tsv", Size: 10}, - {ObjectID: "obj-missing", NormalizedPath: "data/file.tsv", Size: 10}, - {ObjectID: "obj-broken", NormalizedPath: "data/file.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "obj-live": cleanupObject("obj-live", "s3://bucket/live/file.tsv"), - "obj-missing": cleanupObject("obj-missing", "s3://bucket/missing/file.tsv"), - "obj-broken": cleanupObject("obj-broken", "s3://old-bucket/file.tsv"), - }, - inspectByURL: map[string]error{ - "s3://bucket/live/file.tsv": nil, - "s3://bucket/missing/file.tsv": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, - "s3://old-bucket/file.tsv": &core.StorageInspectError{ - Kind: core.StorageInspectCredentialMissing, - Message: "no stored bucket credential found for bucket \"old-bucket\"", - }, - }, - } - - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - ExpectedPaths: []string{"data/file.tsv"}, - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if len(report.Findings) != 2 { - t.Fatalf("expected two findings, got %+v", report.Findings) - } - if report.Findings[0].Kind != FindingStaleDuplicateRecord || report.Findings[1].Kind != FindingBrokenAccessURLError { - t.Fatalf("unexpected finding kinds: %+v", report.Findings) - } -} - -func TestStorageCleanupAuditRepoOrphanStatuses(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-live", NormalizedPath: "data/live.tsv", Size: 10}, - {ObjectID: "obj-stale", NormalizedPath: "data/stale.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "obj-live": cleanupObject("obj-live", "s3://bucket/data/live.tsv"), - "obj-stale": cleanupObject("obj-stale", "s3://bucket/data/stale.tsv"), - }, - inspectByURL: map[string]error{ - "s3://bucket/data/live.tsv": nil, - "s3://bucket/data/stale.tsv": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, - }, - } - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - ExpectedPaths: []string{"data/expected.tsv"}, - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if len(report.Findings) != 2 { - t.Fatalf("expected two orphan findings, got %d", len(report.Findings)) - } - if report.Findings[0].Kind != FindingRepoOrphanLiveObject || report.Findings[1].Kind != FindingRepoOrphanStaleRecord { - t.Fatalf("unexpected findings: %+v", report.Findings) - } -} - -func TestStorageCleanupAuditPartialBrokenAccessURLIsAuditOnly(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-partial", NormalizedPath: "data/file.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "obj-partial": cleanupObject("obj-partial", - "s3://broken-bucket/file.tsv", - "s3://bucket/live/file.tsv", - ), - }, - inspectByURL: map[string]error{ - "s3://broken-bucket/file.tsv": &core.StorageInspectError{ - Kind: core.StorageInspectCredentialMissing, - Message: "no stored bucket credential found for bucket \"broken-bucket\"", - }, - "s3://bucket/live/file.tsv": nil, - }, - } - - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - ExpectedPaths: []string{"data/file.tsv"}, - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if len(report.Findings) != 1 { - t.Fatalf("expected one finding, got %+v", report.Findings) - } - finding := report.Findings[0] - if finding.Kind != FindingBrokenAccessURLError { - t.Fatalf("expected broken access url finding, got %s", finding.Kind) - } - if finding.CleanupScope != CleanupScopeAccessURL { - t.Fatalf("expected access_url scope, got %s", finding.CleanupScope) - } - record := finding.Records[0] - if record.StorageStatus != StorageProbeStatusLive { - t.Fatalf("expected live derived status, got %s", record.StorageStatus) - } - if record.CleanupScope != CleanupScopeAccessURL { - t.Fatalf("expected record access_url scope, got %s", record.CleanupScope) - } -} - -func TestStorageCleanupApplyDeletesExpectedRecords(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "dup-live", NormalizedPath: "data/dup.tsv", Size: 10}, - {ObjectID: "dup-missing", NormalizedPath: "data/dup.tsv", Size: 10}, - {ObjectID: "orphan-live", NormalizedPath: "data/orphan.tsv", Size: 10}, - {ObjectID: "orphan-stale", NormalizedPath: "data/stale.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "dup-live": cleanupObject("dup-live", "s3://bucket/dup/live.tsv"), - "dup-missing": cleanupObject("dup-missing", "s3://bucket/dup/missing.tsv"), - "orphan-live": cleanupObject("orphan-live", "s3://bucket/data/orphan.tsv"), - "orphan-stale": cleanupObject("orphan-stale", "s3://bucket/data/stale.tsv"), - }, - inspectByURL: map[string]error{ - "s3://bucket/dup/live.tsv": nil, - "s3://bucket/dup/missing.tsv": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, - "s3://bucket/data/orphan.tsv": nil, - "s3://bucket/data/stale.tsv": &core.StorageInspectError{Kind: core.StorageInspectObjectNotFound, Message: "missing"}, - }, - } - svc := NewStorageCleanupService(manager) - result, err := svc.Apply(context.Background(), StorageCleanupApplyRequest{ - Organization: "org", - Project: "proj", - ExpectedPaths: []string{"data/dup.tsv", "data/keep.tsv"}, - DeleteStaleDuplicates: true, - DeleteRepoOrphans: true, - }) - if err != nil { - t.Fatalf("Apply failed: %v", err) - } - if len(result.DeletedRecordIDs) != 3 { - t.Fatalf("expected three deleted records, got %d", len(result.DeletedRecordIDs)) - } - if got := manager.deleteOpts["dup-missing"]; got.DeleteStorageData { - t.Fatal("stale duplicate delete should not purge storage") - } - if got := manager.deleteOpts["orphan-live"]; !got.DeleteStorageData { - t.Fatal("live orphan delete should purge storage") - } - if got := manager.deleteOpts["orphan-stale"]; got.DeleteStorageData { - t.Fatal("stale orphan delete should not purge storage") - } -} - -func TestStorageCleanupApplySkipsPartialBrokenAccessRecord(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-partial", NormalizedPath: "data/file.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "obj-partial": cleanupObject("obj-partial", - "s3://broken-bucket/file.tsv", - "s3://bucket/live/file.tsv", - ), - }, - inspectByURL: map[string]error{ - "s3://broken-bucket/file.tsv": &core.StorageInspectError{ - Kind: core.StorageInspectCredentialMissing, - Message: "no stored bucket credential found for bucket \"broken-bucket\"", - }, - "s3://bucket/live/file.tsv": nil, - }, - } - - svc := NewStorageCleanupService(manager) - result, err := svc.Apply(context.Background(), StorageCleanupApplyRequest{ - Organization: "org", - Project: "proj", - ExpectedPaths: []string{"data/file.tsv"}, - DeleteStaleDuplicates: true, - DeleteRepoOrphans: true, - }) - if err != nil { - t.Fatalf("Apply failed: %v", err) - } - if len(result.DeletedRecordIDs) != 0 { - t.Fatalf("expected no deleted records, got %+v", result.DeletedRecordIDs) - } -} - -func TestStorageCleanupAuditUsesBulkObjectFetchOnce(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-1", NormalizedPath: "data/a.tsv", Size: 10}, - {ObjectID: "obj-1", NormalizedPath: "data/a.tsv", Size: 10}, - {ObjectID: "obj-2", NormalizedPath: "data/b.tsv", Size: 11}, - }, - objects: map[string]models.InternalObject{ - "obj-1": cleanupObject("obj-1", "s3://bucket/a.tsv"), - "obj-2": cleanupObject("obj-2", "s3://bucket/b.tsv"), - }, - } - - svc := NewStorageCleanupService(manager) - if _, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - }); err != nil { - t.Fatalf("Audit failed: %v", err) - } - if manager.bulkCalls != 1 { - t.Fatalf("expected one bulk object fetch, got %d", manager.bulkCalls) - } -} - -func TestStorageCleanupAuditUsesDuplicateListingWithoutExpectedPaths(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "dup-1", NormalizedPath: "data/a.tsv", Size: 10}, - {ObjectID: "dup-2", NormalizedPath: "data/a.tsv", Size: 10}, - {ObjectID: "single", NormalizedPath: "data/b.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "dup-1": cleanupObject("dup-1", "s3://bucket/data/a.tsv"), - "dup-2": cleanupObject("dup-2", "s3://bucket/data/a-copy.tsv"), - "single": cleanupObject("single", "s3://bucket/data/b.tsv"), - }, - } - - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if report.Scanned != 2 { - t.Fatalf("expected duplicate-only scan count 2, got %d", report.Scanned) - } - if report.ScannedPaths != 1 { - t.Fatalf("expected one scanned path, got %d", report.ScannedPaths) - } - if manager.duplicateListCalls != 1 || manager.listCalls != 0 { - t.Fatalf("expected duplicate listing only, got list=%d duplicate=%d", manager.listCalls, manager.duplicateListCalls) - } -} - -func TestStorageCleanupAuditUsesFullListingForOrphanAudit(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-1", NormalizedPath: "data/a.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "obj-1": cleanupObject("obj-1", "s3://bucket/data/a.tsv"), - }, - } - - svc := NewStorageCleanupService(manager) - if _, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - ExpectedPaths: []string{"data/other.tsv"}, - }); err != nil { - t.Fatalf("Audit failed: %v", err) - } - if manager.listCalls != 1 || manager.duplicateListCalls != 0 { - t.Fatalf("expected full listing only, got list=%d duplicate=%d", manager.listCalls, manager.duplicateListCalls) - } -} - -func TestStorageCleanupAuditCachesBrokenBucketFailures(t *testing.T) { - manager := &fakeCleanupManager{ - rows: []models.StorageCleanupRecord{ - {ObjectID: "obj-1", NormalizedPath: "data/a.tsv", Size: 10}, - {ObjectID: "obj-2", NormalizedPath: "data/a.tsv", Size: 10}, - }, - objects: map[string]models.InternalObject{ - "obj-1": cleanupObject("obj-1", "s3://old-bucket/data/a.tsv"), - "obj-2": cleanupObject("obj-2", "s3://old-bucket/data/b.tsv"), - }, - inspectByURL: map[string]error{ - "s3://old-bucket/data/a.tsv": &core.StorageInspectError{ - Kind: core.StorageInspectCredentialMissing, - Message: "no stored bucket credential found for bucket \"old-bucket\"", - }, - "s3://old-bucket/data/b.tsv": &core.StorageInspectError{ - Kind: core.StorageInspectCredentialMissing, - Message: "no stored bucket credential found for bucket \"old-bucket\"", - }, - }, - } - - svc := NewStorageCleanupService(manager) - report, err := svc.Audit(context.Background(), StorageCleanupAuditRequest{ - Organization: "org", - Project: "proj", - }) - if err != nil { - t.Fatalf("Audit failed: %v", err) - } - if len(report.Findings) == 0 { - t.Fatalf("expected findings for broken duplicate path") - } - if manager.inspectCalls != 1 { - t.Fatalf("expected one inspect call after broken bucket cache, got %d", manager.inspectCalls) - } -} - -type fakeCleanupManager struct { - rows []models.StorageCleanupRecord - objects map[string]models.InternalObject - inspectByURL map[string]error - deleteOpts map[string]core.DeleteOptions - bulkCalls int - listCalls int - duplicateListCalls int - inspectCalls int -} - -func (f *fakeCleanupManager) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - f.listCalls++ - return append([]models.StorageCleanupRecord(nil), filterCleanupRowsByPrefix(f.rows, pathPrefix)...), nil -} - -func (f *fakeCleanupManager) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - f.duplicateListCalls++ - rows := filterCleanupRowsByPrefix(f.rows, pathPrefix) - counts := make(map[string]int, len(rows)) - for _, row := range rows { - counts[row.NormalizedPath]++ - } - out := make([]models.StorageCleanupRecord, 0, len(rows)) - for _, row := range rows { - if counts[row.NormalizedPath] > 1 { - out = append(out, row) - } - } - return out, nil -} - -func (f *fakeCleanupManager) GetBulkObjects(ctx context.Context, ids []string, requiredMethod string) ([]models.InternalObject, error) { - f.bulkCalls++ - out := make([]models.InternalObject, 0, len(ids)) - for _, id := range ids { - if obj, ok := f.objects[id]; ok { - out = append(out, obj) - } - } - return out, nil -} - -func (f *fakeCleanupManager) InspectStorageObject(ctx context.Context, req core.InspectStorageRequest) (*core.StorageObjectMetadata, error) { - f.inspectCalls++ - if err, ok := f.inspectByURL[req.ObjectURL]; ok { - if err != nil { - return nil, err - } - } - return &core.StorageObjectMetadata{ObjectURL: req.ObjectURL}, nil -} - -func (f *fakeCleanupManager) DeleteObjectWithOptions(ctx context.Context, id string, opts core.DeleteOptions) error { - if f.deleteOpts == nil { - f.deleteOpts = map[string]core.DeleteOptions{} - } - f.deleteOpts[id] = opts - return nil -} - -func cleanupObject(id string, urls ...string) models.InternalObject { - methods := make([]drsapi.AccessMethod, 0, len(urls)) - for _, url := range urls { - methods = append(methods, drsapi.AccessMethod{ - Type: drsapi.AccessMethodTypeS3, - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: url}, - }) - } - return models.InternalObject{ - DrsObject: drsapi.DrsObject{ - Id: id, - AccessMethods: &methods, - }, - } -} - -func cleanupObjectWithChecksum(id string, checksum string, urls ...string) models.InternalObject { - obj := cleanupObject(id, urls...) - obj.Checksums = []drsapi.Checksum{{ - Type: "sha256", - Checksum: checksum, - }} - return obj -} - -func filterCleanupRowsByPrefix(rows []models.StorageCleanupRecord, pathPrefix string) []models.StorageCleanupRecord { - normalizedPrefix, prefixSegments, err := common.NormalizeBrowsePath(pathPrefix) - if err != nil { - return nil - } - if normalizedPrefix == "" { - return append([]models.StorageCleanupRecord(nil), rows...) - } - out := make([]models.StorageCleanupRecord, 0, len(rows)) - for _, row := range rows { - info, ok, err := common.BrowsePathInfoFromName(row.NormalizedPath) - if err != nil || !ok || !common.HasBrowsePathPrefix(info.Segments, prefixSegments) { - continue - } - out = append(out, row) - } - return out -} diff --git a/internal/repair/storage_cleanup_types.go b/internal/repair/storage_cleanup_types.go deleted file mode 100644 index 8a897911..00000000 --- a/internal/repair/storage_cleanup_types.go +++ /dev/null @@ -1,121 +0,0 @@ -package repair - -import "time" - -type StorageProbeStatus string - -const ( - StorageProbeStatusUnknown StorageProbeStatus = "unknown" - StorageProbeStatusLive StorageProbeStatus = "live" - StorageProbeStatusMissing StorageProbeStatus = "missing" - StorageProbeStatusError StorageProbeStatus = "error" -) - -const ( - FindingStaleDuplicateRecord FindingKind = "stale_duplicate_record" - FindingLiveDuplicateConflict FindingKind = "live_duplicate_conflict" - FindingBrokenAccessURLError FindingKind = "broken_access_url_error" - FindingRepoOrphanLiveObject FindingKind = "repo_orphan_live_object" - FindingRepoOrphanStaleRecord FindingKind = "repo_orphan_stale_record" -) - -type CleanupScope string - -const ( - CleanupScopeRecord CleanupScope = "record" - CleanupScopeAccessURL CleanupScope = "access_url" -) - -type StorageCleanupAccessProbe struct { - URL string `json:"url"` - Bucket string `json:"bucket,omitempty"` - StorageStatus StorageProbeStatus `json:"storage_status"` - StorageMessage string `json:"storage_message,omitempty"` - ErrorKind string `json:"error_kind,omitempty"` -} - -type StorageCleanupRecordAudit struct { - ObjectID string `json:"object_id"` - NormalizedPath string `json:"normalized_path"` - Size int64 `json:"size"` - UpdatedTime *time.Time `json:"updated_time,omitempty"` - DownloadCount int64 `json:"download_count"` - LastDownloadTime *time.Time `json:"last_download_time,omitempty"` - CurrentAccessURLs []string `json:"current_access_urls,omitempty"` - AccessProbes []StorageCleanupAccessProbe `json:"access_probes,omitempty"` - CleanupScope CleanupScope `json:"cleanup_scope,omitempty"` - StorageStatus StorageProbeStatus `json:"storage_status"` - StorageMessage string `json:"storage_message,omitempty"` -} - -type StorageCleanupFinding struct { - Kind FindingKind `json:"kind"` - Severity Severity `json:"severity"` - NormalizedPath string `json:"normalized_path"` - Message string `json:"message"` - RecommendedAction string `json:"recommended_action,omitempty"` - RepoDeleteCandidate bool `json:"repo_delete_candidate"` - CleanupScope CleanupScope `json:"cleanup_scope,omitempty"` - StructuralReason string `json:"structural_reason,omitempty"` - ChecksumCount int `json:"checksum_count,omitempty"` - SizeCount int `json:"size_count,omitempty"` - LegacyURLTemplateDetected bool `json:"legacy_url_template_detected,omitempty"` - Records []StorageCleanupRecordAudit `json:"records,omitempty"` -} - -type StorageCleanupReport struct { - Organization string `json:"organization"` - Project string `json:"project"` - PathPrefix string `json:"path_prefix,omitempty"` - Scanned int `json:"scanned"` - ScannedPaths int `json:"scanned_paths,omitempty"` - ClassifiedPaths int `json:"classified_paths,omitempty"` - UnclassifiedPaths int `json:"unclassified_paths,omitempty"` - Summary map[FindingKind]int `json:"summary,omitempty"` - Findings []StorageCleanupFinding `json:"findings,omitempty"` -} - -type StorageCleanupAuditRequest struct { - Organization string `json:"organization"` - Project string `json:"project"` - PathPrefix string `json:"path_prefix,omitempty"` - ExpectedPaths []string `json:"expected_paths,omitempty"` - SelectedPaths []string `json:"selected_paths,omitempty"` - CheckStorage *bool `json:"check_storage,omitempty"` -} - -type StorageCleanupApplyRequest struct { - Organization string `json:"organization"` - Project string `json:"project"` - PathPrefix string `json:"path_prefix,omitempty"` - ExpectedPaths []string `json:"expected_paths,omitempty"` - DeleteStaleDuplicates bool `json:"delete_stale_duplicates"` - DeleteRepoOrphans bool `json:"delete_repo_orphans"` - DryRun bool `json:"dry_run"` - SelectedPaths []string `json:"selected_paths,omitempty"` - SelectedObjectIDs []string `json:"selected_object_ids,omitempty"` - SelectedFindingKinds []FindingKind `json:"selected_finding_kinds,omitempty"` - CheckStorage *bool `json:"check_storage,omitempty"` -} - -type StorageCleanupSkipped struct { - NormalizedPath string `json:"normalized_path"` - ObjectID string `json:"object_id,omitempty"` - Kind FindingKind `json:"kind"` - Reason string `json:"reason"` -} - -type StorageCleanupStoragePurgeResult struct { - ObjectID string `json:"object_id"` - Purged bool `json:"purged"` - Message string `json:"message,omitempty"` -} - -type StorageCleanupApplyResult struct { - Report StorageCleanupReport `json:"report"` - DryRun bool `json:"dry_run"` - DeletedRecordIDs []string `json:"deleted_record_ids,omitempty"` - StoragePurgeResults []StorageCleanupStoragePurgeResult `json:"storage_purge_results,omitempty"` - RepoDeletePaths []string `json:"repo_delete_paths,omitempty"` - Skipped []StorageCleanupSkipped `json:"skipped,omitempty"` -} diff --git a/internal/storagemetrics/storage_metrics.go b/internal/storagemetrics/storage_metrics.go deleted file mode 100644 index 6c67e2f5..00000000 --- a/internal/storagemetrics/storage_metrics.go +++ /dev/null @@ -1,248 +0,0 @@ -package storagemetrics - -import ( - "fmt" - "sort" - "strings" - "time" - - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" -) - -const ( - StoragePathChildTypeFile = "file" - StoragePathChildTypeDirectory = "directory" -) - -func AggregateStoragePathSummary(organization, project, path string, rows []models.DrsObjectRecord) (models.StoragePathSummary, error) { - normalizedPath, requestedSegments, err := common.NormalizeBrowsePath(path) - if err != nil { - return models.StoragePathSummary{}, err - } - summary := models.StoragePathSummary{ - Organization: organization, - Project: project, - Path: normalizedPath, - } - distinctPaths := make(map[string]struct{}) - directChildren := make(map[string]struct{}) - recordsByPath := make(map[string]int64) - for _, row := range rows { - info, ok := storageMetricPathInfo(row) - if !ok || !common.HasBrowsePathPrefix(info.Segments, requestedSegments) { - continue - } - if len(info.Segments) > len(requestedSegments) { - childPath := strings.Join(info.Segments[:len(requestedSegments)+1], "/") - directChildren[childPath] = struct{}{} - } - summary.RecordCount++ - summary.TotalBytes += row.Size - summary.DownloadCount += row.DownloadCount - distinctPaths[info.Normalized] = struct{}{} - recordsByPath[info.Normalized]++ - if row.LastDownloadTime != nil && (summary.LastDownloadTime == nil || row.LastDownloadTime.After(*summary.LastDownloadTime)) { - t := row.LastDownloadTime.UTC() - summary.LastDownloadTime = &t - } - if row.UpdatedTime.IsZero() { - continue - } - if summary.LatestUpdateTime == nil || row.UpdatedTime.After(*summary.LatestUpdateTime) { - t := row.UpdatedTime.UTC() - summary.LatestUpdateTime = &t - } - } - summary.FileCount = int64(len(distinctPaths)) - summary.DirectChildCount = int64(len(directChildren)) - for _, count := range recordsByPath { - if count > 1 { - summary.DuplicatePathCount++ - } - } - return summary, nil -} - -func AggregateStoragePathChildren(path string, rows []models.DrsObjectRecord, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) { - _, requestedSegments, err := common.NormalizeBrowsePath(path) - if err != nil { - return nil, err - } - type childAggregate struct { - item models.StoragePathChild - distinctPaths map[string]struct{} - isDirectory bool - } - children := make(map[string]*childAggregate) - for _, row := range rows { - info, ok := storageMetricPathInfo(row) - if !ok || !common.HasBrowsePathPrefix(info.Segments, requestedSegments) || len(info.Segments) <= len(requestedSegments) { - continue - } - childSegments := info.Segments[:len(requestedSegments)+1] - childPath := strings.Join(childSegments, "/") - agg, exists := children[childPath] - if !exists { - agg = &childAggregate{ - item: models.StoragePathChild{ - Name: childSegments[len(childSegments)-1], - Path: childPath, - Type: StoragePathChildTypeFile, - }, - distinctPaths: make(map[string]struct{}), - } - children[childPath] = agg - } - if len(info.Segments) > len(requestedSegments)+1 { - agg.isDirectory = true - agg.item.Type = StoragePathChildTypeDirectory - } - agg.item.RecordCount++ - agg.item.TotalBytes += row.Size - agg.item.DownloadCount += row.DownloadCount - agg.distinctPaths[info.Normalized] = struct{}{} - if row.LastDownloadTime != nil && (agg.item.LastDownloadTime == nil || row.LastDownloadTime.After(*agg.item.LastDownloadTime)) { - t := row.LastDownloadTime.UTC() - agg.item.LastDownloadTime = &t - } - if row.UpdatedTime.IsZero() { - continue - } - if agg.item.LatestUpdateTime == nil || row.UpdatedTime.After(*agg.item.LatestUpdateTime) { - t := row.UpdatedTime.UTC() - agg.item.LatestUpdateTime = &t - } - } - items := make([]models.StoragePathChild, 0, len(children)) - for _, agg := range children { - agg.item.FileCount = int64(len(agg.distinctPaths)) - if agg.isDirectory { - agg.item.Type = StoragePathChildTypeDirectory - } - items = append(items, agg.item) - } - sortStoragePathChildren(items, sortBy, sortOrder) - if offset < 0 { - offset = 0 - } - if offset >= len(items) { - return []models.StoragePathChild{}, nil - } - if limit <= 0 { - return items[offset:], nil - } - end := offset + limit - if end > len(items) { - end = len(items) - } - return items[offset:end], nil -} - -func storageMetricPathInfo(row models.DrsObjectRecord) (common.BrowsePathInfo, bool) { - raw := strings.TrimSpace(row.Path) - if raw == "" { - raw = strings.TrimSpace(row.Name) - } - info, ok, err := common.BrowsePathInfoFromName(raw) - if err != nil || !ok { - return common.BrowsePathInfo{}, false - } - return info, true -} - -func sortStoragePathChildren(items []models.StoragePathChild, sortBy, sortOrder string) { - desc := strings.EqualFold(strings.TrimSpace(sortOrder), "desc") - key := strings.TrimSpace(sortBy) - if key == "" { - key = "name" - } - sort.Slice(items, func(i, j int) bool { - left := items[i] - right := items[j] - less := false - switch key { - case "bytes": - less = left.TotalBytes < right.TotalBytes - case "updated_time": - less = compareTimePtr(left.LatestUpdateTime, right.LatestUpdateTime) - case "records": - less = left.RecordCount < right.RecordCount - case "name": - fallthrough - default: - less = strings.ToLower(left.Name) < strings.ToLower(right.Name) - } - if equivalentSortValue(left, right, key) { - if strings.EqualFold(left.Name, right.Name) { - if desc { - return left.Path > right.Path - } - return left.Path < right.Path - } - if desc { - return strings.ToLower(left.Name) > strings.ToLower(right.Name) - } - return strings.ToLower(left.Name) < strings.ToLower(right.Name) - } - if desc { - return !less - } - return less - }) -} - -func equivalentSortValue(left, right models.StoragePathChild, key string) bool { - switch key { - case "bytes": - return left.TotalBytes == right.TotalBytes - case "updated_time": - return sameTimePtr(left.LatestUpdateTime, right.LatestUpdateTime) - case "records": - return left.RecordCount == right.RecordCount - default: - return strings.EqualFold(left.Name, right.Name) - } -} - -func compareTimePtr(left, right *time.Time) bool { - if left == nil && right == nil { - return false - } - if left == nil { - return true - } - if right == nil { - return false - } - return left.Before(*right) -} - -func sameTimePtr(left, right *time.Time) bool { - if left == nil || right == nil { - return left == right - } - return left.Equal(*right) -} - -func NormalizeStorageChildrenSort(sortBy, sortOrder string) (string, string, error) { - sortBy = strings.TrimSpace(sortBy) - if sortBy == "" { - sortBy = "name" - } - switch sortBy { - case "bytes", "name", "updated_time", "records": - default: - return "", "", fmt.Errorf("invalid sort_by") - } - sortOrder = strings.TrimSpace(sortOrder) - if sortOrder == "" { - sortOrder = "asc" - } - switch sortOrder { - case "asc", "desc": - default: - return "", "", fmt.Errorf("invalid sort_order") - } - return sortBy, sortOrder, nil -} diff --git a/internal/storagemetrics/storage_metrics_test.go b/internal/storagemetrics/storage_metrics_test.go deleted file mode 100644 index c986143f..00000000 --- a/internal/storagemetrics/storage_metrics_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package storagemetrics - -import ( - "testing" - "time" - - "github.com/calypr/syfon/internal/models" -) - -func TestAggregateStoragePathSummary(t *testing.T) { - now := time.Now().UTC() - rows := []models.DrsObjectRecord{ - {Name: "data/a.txt", Size: 10, UpdatedTime: now.Add(-time.Hour), DownloadCount: 2, LastDownloadTime: timePtr(now.Add(-3 * time.Hour))}, - {Path: "data/a.txt", Size: 20, UpdatedTime: now, DownloadCount: 4, LastDownloadTime: timePtr(now.Add(-30 * time.Minute))}, - {Name: "data/nested/b.txt", Size: 30, UpdatedTime: now.Add(-2 * time.Hour), DownloadCount: 1, LastDownloadTime: timePtr(now.Add(-90 * time.Minute))}, - {Name: "other/c.txt", Size: 40, UpdatedTime: now.Add(-3 * time.Hour)}, - {Name: "bad/../path", Size: 50, UpdatedTime: now}, - } - - summary, err := AggregateStoragePathSummary("org", "proj", "data", rows) - if err != nil { - t.Fatalf("AggregateStoragePathSummary returned error: %v", err) - } - if summary.Path != "data" { - t.Fatalf("expected normalized path data, got %q", summary.Path) - } - if summary.FileCount != 2 || summary.RecordCount != 3 || summary.TotalBytes != 60 { - t.Fatalf("unexpected counts: %+v", summary) - } - if summary.DirectChildCount != 2 { - t.Fatalf("expected 2 direct children, got %+v", summary) - } - if summary.DuplicatePathCount != 1 { - t.Fatalf("expected one duplicate path, got %+v", summary) - } - if summary.DownloadCount != 7 { - t.Fatalf("unexpected download count: %+v", summary) - } - if summary.LastDownloadTime == nil || !summary.LastDownloadTime.Equal(now.Add(-30*time.Minute)) { - t.Fatalf("unexpected last download time: %+v", summary.LastDownloadTime) - } - if summary.LatestUpdateTime == nil || !summary.LatestUpdateTime.Equal(now) { - t.Fatalf("unexpected latest update time: %+v", summary.LatestUpdateTime) - } -} - -func TestAggregateStoragePathChildren(t *testing.T) { - now := time.Now().UTC() - rows := []models.DrsObjectRecord{ - {Name: ` data\\alpha.txt `, Size: 10, UpdatedTime: now.Add(-time.Hour), DownloadCount: 2, LastDownloadTime: timePtr(now.Add(-10 * time.Hour))}, - {Name: "data/dir/one.txt", Size: 15, UpdatedTime: now.Add(-2 * time.Hour), DownloadCount: 3, LastDownloadTime: timePtr(now.Add(-4 * time.Hour))}, - {Name: "data/dir/two.txt", Size: 25, UpdatedTime: now, DownloadCount: 5, LastDownloadTime: timePtr(now.Add(-15 * time.Minute))}, - {Name: "data/alpha.txt", Size: 5, UpdatedTime: now.Add(-30 * time.Minute), DownloadCount: 7, LastDownloadTime: timePtr(now.Add(-2 * time.Hour))}, - {Name: "skip/elsewhere.txt", Size: 99, UpdatedTime: now}, - } - - items, err := AggregateStoragePathChildren("data", rows, 10, 0, "bytes", "desc") - if err != nil { - t.Fatalf("AggregateStoragePathChildren returned error: %v", err) - } - if len(items) != 2 { - t.Fatalf("expected 2 items, got %+v", items) - } - if items[0].Name != "dir" || items[0].Type != StoragePathChildTypeDirectory || items[0].FileCount != 2 || items[0].RecordCount != 2 || items[0].TotalBytes != 40 || items[0].DownloadCount != 8 { - t.Fatalf("unexpected directory aggregate: %+v", items[0]) - } - if items[0].LastDownloadTime == nil || !items[0].LastDownloadTime.Equal(now.Add(-15*time.Minute)) { - t.Fatalf("unexpected directory last download time: %+v", items[0].LastDownloadTime) - } - if items[1].Name != "alpha.txt" || items[1].Type != StoragePathChildTypeFile || items[1].FileCount != 1 || items[1].RecordCount != 2 || items[1].TotalBytes != 15 || items[1].DownloadCount != 9 { - t.Fatalf("unexpected file aggregate: %+v", items[1]) - } - if items[1].LastDownloadTime == nil || !items[1].LastDownloadTime.Equal(now.Add(-2*time.Hour)) { - t.Fatalf("unexpected file last download time: %+v", items[1].LastDownloadTime) - } -} - -func TestNormalizeStorageChildrenSort(t *testing.T) { - sortBy, sortOrder, err := NormalizeStorageChildrenSort("", "") - if err != nil { - t.Fatalf("NormalizeStorageChildrenSort returned error: %v", err) - } - if sortBy != "name" || sortOrder != "asc" { - t.Fatalf("unexpected defaults: %q %q", sortBy, sortOrder) - } - if _, _, err := NormalizeStorageChildrenSort("nope", "asc"); err == nil { - t.Fatal("expected invalid sort_by error") - } - if _, _, err := NormalizeStorageChildrenSort("name", "down"); err == nil { - t.Fatal("expected invalid sort_order error") - } -} - -func timePtr(t time.Time) *time.Time { - return &t -} diff --git a/internal/testutils/mocks.go b/internal/testutils/mocks.go index 5e681dc8..29b21465 100644 --- a/internal/testutils/mocks.go +++ b/internal/testutils/mocks.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "net/http" - "sort" "strings" "time" @@ -12,7 +11,6 @@ import ( sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" - "github.com/calypr/syfon/internal/storagemetrics" "github.com/calypr/syfon/internal/urlmanager" ) @@ -152,224 +150,6 @@ func (m *MockDatabase) ListObjectIDsByScope(ctx context.Context, organization, p return ids, nil } -func (m *MockDatabase) GetStoragePathSummary(ctx context.Context, organization, project, path string) (models.StoragePathSummary, error) { - return storagemetrics.AggregateStoragePathSummary(strings.TrimSpace(organization), strings.TrimSpace(project), path, m.storageMetricRowsForScope(strings.TrimSpace(organization), strings.TrimSpace(project))) -} - -func (m *MockDatabase) ListStoragePathChildren(ctx context.Context, organization, project, path string, limit, offset int, sortBy, sortOrder string) ([]models.StoragePathChild, error) { - return storagemetrics.AggregateStoragePathChildren(path, m.storageMetricRowsForScope(strings.TrimSpace(organization), strings.TrimSpace(project)), limit, offset, sortBy, sortOrder) -} - -func (m *MockDatabase) ListStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - _, prefixSegments, err := common.NormalizeBrowsePath(pathPrefix) - if err != nil { - return nil, err - } - rows := make([]models.StorageCleanupRecord, 0, len(m.Objects)) - for id, obj := range m.Objects { - authz := map[string][]string{} - if m.ObjectAuthz != nil { - authz = m.ObjectAuthz[id] - } - projects, ok := authz[organization] - if !ok { - continue - } - matchesProject := strings.TrimSpace(project) == "" - for _, candidate := range projects { - if candidate == project { - matchesProject = true - break - } - } - if !matchesProject || obj.Name == nil { - continue - } - info, ok, err := common.BrowsePathInfoFromName(*obj.Name) - if err != nil || !ok || !common.HasBrowsePathPrefix(info.Segments, prefixSegments) { - continue - } - row := models.StorageCleanupRecord{ - ObjectID: id, - NormalizedPath: info.Normalized, - Size: obj.Size, - } - if obj.UpdatedTime != nil { - row.UpdatedTime = obj.UpdatedTime.UTC() - } - if u, ok := m.Usage[id]; ok { - row.DownloadCount = u.DownloadCount - row.LastDownloadTime = u.LastDownloadTime - } - rows = append(rows, row) - } - return rows, nil -} - -func (m *MockDatabase) ListDuplicateStorageCleanupRecords(ctx context.Context, organization, project, pathPrefix string) ([]models.StorageCleanupRecord, error) { - rows, err := m.ListStorageCleanupRecords(ctx, organization, project, pathPrefix) - if err != nil { - return nil, err - } - counts := make(map[string]int, len(rows)) - for _, row := range rows { - counts[row.NormalizedPath]++ - } - out := make([]models.StorageCleanupRecord, 0, len(rows)) - for _, row := range rows { - if counts[row.NormalizedPath] > 1 { - out = append(out, row) - } - } - return out, nil -} - -func (m *MockDatabase) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { - normalizedPath, pathSegments, err := common.NormalizeBrowsePath(path) - if err != nil { - return nil, nil, err - } - ids := make([]string, 0) - directoriesByPath := make(map[string]models.BrowseDirectory) - for id, obj := range m.Objects { - authz := map[string][]string{} - if m.ObjectAuthz != nil { - if v, ok := m.ObjectAuthz[id]; ok { - authz = v - } - } - projects, ok := authz[organization] - if !ok { - continue - } - matchesProject := false - for _, candidateProject := range projects { - if candidateProject == project { - matchesProject = true - break - } - } - if !matchesProject { - continue - } - if obj.Name == nil { - continue - } - info, ok, err := common.BrowsePathInfoFromName(*obj.Name) - if err != nil || !ok { - continue - } - if !common.HasBrowsePathPrefix(info.Segments, pathSegments) { - continue - } - remaining := info.Segments[len(pathSegments):] - if len(remaining) == 0 { - continue - } - if len(remaining) == 1 { - if info.ParentPath == normalizedPath { - ids = append(ids, id) - } - continue - } - dirInfo, ok := common.ImmediateBrowseDirectory(normalizedPath, info.Normalized) - if !ok { - continue - } - directoriesByPath[dirInfo.Normalized] = models.BrowseDirectory{Name: dirInfo.EntryName, Path: dirInfo.Normalized} - } - sort.Strings(ids) - if startAfter != "" { - offset = sort.SearchStrings(ids, startAfter) - for offset < len(ids) && ids[offset] <= startAfter { - offset++ - } - } - if offset < 0 { - offset = 0 - } - if offset > len(ids) { - offset = len(ids) - } - end := offset + limit - if end > len(ids) { - end = len(ids) - } - directories := make([]models.BrowseDirectory, 0, len(directoriesByPath)) - for _, directory := range directoriesByPath { - directories = append(directories, directory) - } - sort.Slice(directories, func(i, j int) bool { - if directories[i].Name == directories[j].Name { - return directories[i].Path < directories[j].Path - } - return directories[i].Name < directories[j].Name - }) - return ids[offset:end], directories, nil -} - -func (m *MockDatabase) storageMetricRows() []models.DrsObjectRecord { - rows := make([]models.DrsObjectRecord, 0, len(m.Objects)) - for id, obj := range m.Objects { - row := models.DrsObjectRecord{ - ID: id, - Size: obj.Size, - } - if obj.Name != nil { - row.Name = *obj.Name - } - if obj.UpdatedTime != nil { - row.UpdatedTime = obj.UpdatedTime.UTC() - } - if u, ok := m.Usage[id]; ok { - row.DownloadCount = u.DownloadCount - row.LastDownloadTime = u.LastDownloadTime - } - rows = append(rows, row) - } - return rows -} - -func (m *MockDatabase) storageMetricRowsForScope(organization, project string) []models.DrsObjectRecord { - rows := make([]models.DrsObjectRecord, 0, len(m.Objects)) - for id, obj := range m.Objects { - authz := map[string][]string{} - if m.ObjectAuthz != nil { - authz = m.ObjectAuthz[id] - } - projects, ok := authz[organization] - if !ok { - continue - } - matchesProject := strings.TrimSpace(project) == "" - for _, candidate := range projects { - if candidate == project { - matchesProject = true - break - } - } - if !matchesProject { - continue - } - row := models.DrsObjectRecord{ - ID: id, - Size: obj.Size, - } - if obj.Name != nil { - row.Name = *obj.Name - } - if obj.UpdatedTime != nil { - row.UpdatedTime = obj.UpdatedTime.UTC() - } - if u, ok := m.Usage[id]; ok { - row.DownloadCount = u.DownloadCount - row.LastDownloadTime = u.LastDownloadTime - } - rows = append(rows, row) - } - return rows -} - func (m *MockDatabase) ListObjectIDsByResources(ctx context.Context, resources []string, includeUnscoped bool) ([]string, error) { allowed := map[string]struct{}{} for _, resource := range resources { diff --git a/internal/version/version.go b/internal/version/version.go index 884fe763..f908be39 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -22,14 +22,3 @@ version: %s func String() string { return fmt.Sprintf("Syfon — a Modern DRS Server ⚡️\n"+tpl, GitCommit, GitBranch, GitUpstream, BuildDate, Version) } - -// LogFields logs build and version information to the given logger. -func LogFields() []interface{} { - return []interface{}{ - "GitCommit", GitCommit, - "GitBranch", GitBranch, - "GitUpstream", GitUpstream, - "BuildDate", BuildDate, - "Version", Version, - } -} diff --git a/migrate/client.go b/migrate/client.go deleted file mode 100644 index e067d406..00000000 --- a/migrate/client.go +++ /dev/null @@ -1,379 +0,0 @@ -package migrate - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/url" - "strings" - "time" - - "github.com/calypr/syfon/apigen/server/drs" -) - -const maxHTTPAttempts = 3 - -type BasicAuth struct { - Username string - Password string -} - -type AuthConfig struct { - BearerToken string - Basic *BasicAuth -} - -type HTTPClient struct { - baseURL string - httpClient *http.Client - auth AuthConfig - userAgent string -} - -func NewHTTPClient(baseURL string, auth AuthConfig, httpClient *http.Client) (*HTTPClient, error) { - baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") - if baseURL == "" { - return nil, fmt.Errorf("base URL is required") - } - if httpClient == nil { - httpClient = &http.Client{Timeout: 60 * time.Second} - } - return &HTTPClient{ - baseURL: baseURL, - httpClient: httpClient, - auth: auth, - userAgent: "syfon-migrate", - }, nil -} - -func (c *HTTPClient) ListPage(ctx context.Context, limit int, start string) ([]IndexdRecord, error) { - q := url.Values{} - if limit > 0 { - q.Set("limit", fmt.Sprintf("%d", limit)) - } - if strings.TrimSpace(start) != "" { - q.Set("start", start) - } - endpoint := c.endpoint("/index") - if encoded := q.Encode(); encoded != "" { - endpoint += "?" + encoded - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return nil, err - } - c.applyHeaders(req) - resp, err := c.do(req) - if err != nil { - return nil, fmt.Errorf("GET %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return nil, fmt.Errorf("GET %s: read error response body: %w", endpoint, err) - } - return nil, fmt.Errorf("GET %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - - var page IndexdPage - if err := json.NewDecoder(resp.Body).Decode(&page); err != nil { - return nil, fmt.Errorf("decode %s: %w", endpoint, err) - } - if len(page.Records) > 0 { - return page.Records, nil - } - if len(page.IDs) > 0 { - return c.fetchDocuments(ctx, page.IDs) - } - return nil, nil -} - -func (c *HTTPClient) fetchDocuments(ctx context.Context, ids []string) ([]IndexdRecord, error) { - records, err := c.bulkDocuments(ctx, ids) - if err == nil { - return records, nil - } - - out := make([]IndexdRecord, 0, len(ids)) - for _, id := range ids { - rec, getErr := c.getRecord(ctx, id) - if getErr != nil { - return nil, fmt.Errorf("bulk documents failed: %v; detail fetch %s failed: %w", err, id, getErr) - } - out = append(out, rec) - } - return out, nil -} - -func (c *HTTPClient) bulkDocuments(ctx context.Context, ids []string) ([]IndexdRecord, error) { - body, err := json.Marshal(ids) - if err != nil { - return nil, err - } - endpoint := c.endpoint("/index/bulk/documents") - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if err != nil { - return nil, err - } - c.applyHeaders(req) - req.Header.Set("Content-Type", "application/json") - resp, err := c.do(req) - if err != nil { - return nil, fmt.Errorf("POST %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return nil, fmt.Errorf("POST %s: read error response body: %w", endpoint, err) - } - return nil, fmt.Errorf("POST %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - var records []IndexdRecord - if err := json.NewDecoder(resp.Body).Decode(&records); err != nil { - return nil, fmt.Errorf("decode %s: %w", endpoint, err) - } - return records, nil -} - -func (c *HTTPClient) getRecord(ctx context.Context, id string) (IndexdRecord, error) { - endpoint := c.endpoint("/index/") + url.PathEscape(id) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return IndexdRecord{}, err - } - c.applyHeaders(req) - resp, err := c.do(req) - if err != nil { - return IndexdRecord{}, fmt.Errorf("GET %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return IndexdRecord{}, fmt.Errorf("GET %s: read error response body: %w", endpoint, err) - } - return IndexdRecord{}, fmt.Errorf("GET %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - var record IndexdRecord - if err := json.NewDecoder(resp.Body).Decode(&record); err != nil { - return IndexdRecord{}, fmt.Errorf("decode %s: %w", endpoint, err) - } - return record, nil -} - -func (c *HTTPClient) LoadBatch(ctx context.Context, records []MigrationRecord) error { - body, err := json.Marshal(bulkCreateRequestFromMigration(records)) - if err != nil { - return err - } - endpoint := c.endpoint("/index/bulk") - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if err != nil { - return err - } - c.applyHeaders(req) - req.Header.Set("Content-Type", "application/json") - resp, err := c.do(req) - if err != nil { - return fmt.Errorf("POST %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return fmt.Errorf("POST %s: read error response body: %w", endpoint, err) - } - return fmt.Errorf("POST %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - return nil -} - -func (c *HTTPClient) UserPrivileges(ctx context.Context) (map[string]map[string]bool, error) { - endpoint := c.userEndpoint("/user/user") - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return nil, err - } - c.applyHeaders(req) - resp, err := c.do(req) - if err != nil { - return nil, fmt.Errorf("GET %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return nil, fmt.Errorf("GET %s: read error response body: %w", endpoint, err) - } - return nil, fmt.Errorf("GET %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - - var data map[string]any - if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { - return nil, fmt.Errorf("decode %s: %w", endpoint, err) - } - resourceAccess, ok := data["authz"].(map[string]any) - if !ok || len(resourceAccess) == 0 { - resourceAccess, ok = data["project_access"].(map[string]any) - if !ok { - return nil, fmt.Errorf("no authz/project_access found in user response") - } - } - return extractUserPrivileges(resourceAccess), nil -} - -type bulkCreateRequest struct { - Records []bulkInternalRecord `json:"records"` -} - -type bulkInternalRecord struct { - AccessMethods []drs.AccessMethod `json:"access_methods,omitempty"` - ControlledAccess []string `json:"controlled_access,omitempty"` - CreatedTime *string `json:"created_time,omitempty"` - Description *string `json:"description,omitempty"` - Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` - Hashes *map[string]string `json:"hashes,omitempty"` - Size *int64 `json:"size,omitempty"` - UpdatedTime *string `json:"updated_time,omitempty"` - Version *string `json:"version,omitempty"` -} - -func bulkCreateRequestFromMigration(records []MigrationRecord) bulkCreateRequest { - out := bulkCreateRequest{Records: make([]bulkInternalRecord, 0, len(records))} - for _, record := range records { - size := record.Size - created := record.CreatedTime.Format(time.RFC3339Nano) - var updated *string - if record.UpdatedTime != nil { - value := record.UpdatedTime.Format(time.RFC3339Nano) - updated = &value - } - hashes := make(map[string]string, len(record.Checksums)) - for _, checksum := range record.Checksums { - if checksum.Type != "" && checksum.Checksum != "" { - hashes[checksum.Type] = checksum.Checksum - } - } - out.Records = append(out.Records, bulkInternalRecord{ - Did: record.ID, - Size: &size, - FileName: record.Name, - Version: record.Version, - Description: record.Description, - CreatedTime: &created, - UpdatedTime: updated, - Hashes: &hashes, - AccessMethods: record.AccessMethods, - ControlledAccess: migrationRecordControlledAccess(record), - }) - } - return out -} - -func migrationRecordControlledAccess(record MigrationRecord) []string { - if len(record.ControlledAccess) > 0 { - return record.ControlledAccess - } - return record.Authz -} - -func (c *HTTPClient) endpoint(path string) string { - path = "/" + strings.TrimLeft(path, "/") - if strings.HasSuffix(c.baseURL, "/index") { - if path == "/index" { - return c.baseURL - } - return c.baseURL + strings.TrimPrefix(path, "/index") - } - return c.baseURL + path -} - -func (c *HTTPClient) userEndpoint(path string) string { - path = "/" + strings.TrimLeft(path, "/") - base := c.baseURL - if strings.HasSuffix(base, "/index") { - base = strings.TrimSuffix(base, "/index") - } - return base + path -} - -func (c *HTTPClient) do(req *http.Request) (*http.Response, error) { - var lastErr error - for attempt := 1; attempt <= maxHTTPAttempts; attempt++ { - if attempt > 1 && req.GetBody != nil { - body, err := req.GetBody() - if err != nil { - return nil, err - } - req.Body = body - } - - resp, err := c.httpClient.Do(req) - if !shouldRetryHTTP(resp, err) || attempt == maxHTTPAttempts { - return resp, err - } - if err != nil { - lastErr = err - } - if resp != nil && resp.Body != nil { - if _, copyErr := io.Copy(io.Discard, resp.Body); copyErr != nil && lastErr == nil { - lastErr = fmt.Errorf("discard retry response body: %w", copyErr) - } - if closeErr := resp.Body.Close(); closeErr != nil && lastErr == nil { - lastErr = fmt.Errorf("close retry response body: %w", closeErr) - } - } - - timer := time.NewTimer(time.Duration(attempt) * 750 * time.Millisecond) - select { - case <-req.Context().Done(): - timer.Stop() - if lastErr != nil { - return nil, lastErr - } - return nil, req.Context().Err() - case <-timer.C: - } - } - return nil, lastErr -} - -func readTrimmedBody(r io.Reader) (string, error) { - body, err := io.ReadAll(r) - if err != nil { - return "", err - } - return strings.TrimSpace(string(body)), nil -} - -func shouldRetryHTTP(resp *http.Response, err error) bool { - if err != nil { - var netErr net.Error - return errors.As(err, &netErr) && netErr.Timeout() - } - if resp == nil { - return false - } - return resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= http.StatusInternalServerError -} - -func (c *HTTPClient) applyHeaders(req *http.Request) { - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", c.userAgent) - if strings.TrimSpace(c.auth.BearerToken) != "" { - req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(c.auth.BearerToken)) - return - } - if c.auth.Basic != nil && c.auth.Basic.Username != "" { - req.SetBasicAuth(c.auth.Basic.Username, c.auth.Basic.Password) - } -} diff --git a/migrate/dump.go b/migrate/dump.go deleted file mode 100644 index d471e2a8..00000000 --- a/migrate/dump.go +++ /dev/null @@ -1,155 +0,0 @@ -package migrate - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - _ "github.com/mattn/go-sqlite3" -) - -type SQLiteDump struct { - db *sql.DB -} - -func OpenSQLiteDump(path string) (*SQLiteDump, error) { - if path == "" { - return nil, fmt.Errorf("dump path is required") - } - if dir := filepath.Dir(path); dir != "." && dir != "" { - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, err - } - } - db, err := sql.Open("sqlite3", path) - if err != nil { - return nil, err - } - dump := &SQLiteDump{db: db} - if err := dump.init(); err != nil { - db.Close() - return nil, err - } - return dump, nil -} - -func OpenExistingSQLiteDump(path string) (*SQLiteDump, error) { - if path == "" { - return nil, fmt.Errorf("dump path is required") - } - if _, err := os.Stat(path); err != nil { - return nil, err - } - return OpenSQLiteDump(path) -} - -func (d *SQLiteDump) Close() error { - if d == nil || d.db == nil { - return nil - } - return d.db.Close() -} - -func (d *SQLiteDump) init() error { - _, err := d.db.Exec(` -CREATE TABLE IF NOT EXISTS migration_records ( - id TEXT PRIMARY KEY, - record_json TEXT NOT NULL, - exported_at TEXT NOT NULL -); -CREATE TABLE IF NOT EXISTS migration_metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL -); -`) - return err -} - -func (d *SQLiteDump) LoadBatch(ctx context.Context, records []MigrationRecord) error { - tx, err := d.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() - stmt, err := tx.PrepareContext(ctx, ` -INSERT INTO migration_records (id, record_json, exported_at) -VALUES (?, ?, ?) -ON CONFLICT(id) DO UPDATE SET - record_json=excluded.record_json, - exported_at=excluded.exported_at -`) - if err != nil { - return err - } - defer stmt.Close() - - now := time.Now().UTC().Format(time.RFC3339Nano) - for _, record := range records { - body, err := json.Marshal(record) - if err != nil { - return err - } - if _, err := stmt.ExecContext(ctx, record.ID, string(body), now); err != nil { - return err - } - } - return tx.Commit() -} - -func (d *SQLiteDump) ReadBatches(ctx context.Context, batchSize int, fn func([]MigrationRecord) error) error { - if batchSize <= 0 { - batchSize = 500 - } - offset := 0 - for { - rows, err := d.db.QueryContext(ctx, ` -SELECT record_json FROM migration_records -ORDER BY id -LIMIT ? OFFSET ? -`, batchSize, offset) - if err != nil { - return err - } - - batch := make([]MigrationRecord, 0, batchSize) - for rows.Next() { - var raw string - if err := rows.Scan(&raw); err != nil { - rows.Close() - return err - } - var record MigrationRecord - if err := json.Unmarshal([]byte(raw), &record); err != nil { - rows.Close() - return err - } - batch = append(batch, record) - } - if err := rows.Err(); err != nil { - rows.Close() - return err - } - rows.Close() - - if len(batch) == 0 { - return nil - } - if err := fn(batch); err != nil { - return err - } - if len(batch) < batchSize { - return nil - } - offset += len(batch) - } -} - -func (d *SQLiteDump) Count(ctx context.Context) (int, error) { - var count int - err := d.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM migration_records`).Scan(&count) - return count, err -} diff --git a/migrate/dump_test.go b/migrate/dump_test.go deleted file mode 100644 index 2d47e10e..00000000 --- a/migrate/dump_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package migrate - -import ( - "context" - "path/filepath" - "testing" - "time" - - "github.com/calypr/syfon/apigen/server/drs" -) - -func TestSQLiteDumpRoundTrip(t *testing.T) { - path := filepath.Join(t.TempDir(), "records.sqlite") - dump, err := OpenSQLiteDump(path) - if err != nil { - t.Fatalf("OpenSQLiteDump: %v", err) - } - defer dump.Close() - - record := MigrationRecord{ - ID: "dg.test/roundtrip", - Size: 1, - CreatedTime: time.Now().UTC(), - Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha"}}, - } - if err := dump.LoadBatch(context.Background(), []MigrationRecord{record}); err != nil { - t.Fatalf("LoadBatch: %v", err) - } - count, err := dump.Count(context.Background()) - if err != nil { - t.Fatalf("Count: %v", err) - } - if count != 1 { - t.Fatalf("expected 1 record, got %d", count) - } - - var got []MigrationRecord - if err := dump.ReadBatches(context.Background(), 10, func(batch []MigrationRecord) error { - got = append(got, batch...) - return nil - }); err != nil { - t.Fatalf("ReadBatches: %v", err) - } - if len(got) != 1 || got[0].ID != record.ID { - t.Fatalf("unexpected roundtrip records: %+v", got) - } -} diff --git a/migrate/preflight.go b/migrate/preflight.go deleted file mode 100644 index 83c5e8f4..00000000 --- a/migrate/preflight.go +++ /dev/null @@ -1,205 +0,0 @@ -package migrate - -import ( - "context" - "fmt" - "sort" - "strings" - - syfoncommon "github.com/calypr/syfon/common" -) - -const maxPreflightMissingScopes = 10 - -type ImportPreflightReport struct { - Records int - RequiredResources []string - MissingResources []string - MissingRecords int - FirstDeniedRecord string -} - -type ImportPreflightError struct { - Report ImportPreflightReport -} - -func (e *ImportPreflightError) Error() string { - report := e.Report - scopes := FormatPreflightScopes(report.MissingResources) - if len(scopes) > maxPreflightMissingScopes { - truncated := len(scopes) - maxPreflightMissingScopes - scopes = scopes[:maxPreflightMissingScopes] - scopes = append(scopes, fmt.Sprintf("(and %d more)", truncated)) - } - - var b strings.Builder - fmt.Fprintf(&b, "missing create access for %d/%d records", report.MissingRecords, report.Records) - if report.FirstDeniedRecord != "" { - fmt.Fprintf(&b, "; first denied record=%q", report.FirstDeniedRecord) - } - if len(scopes) > 0 { - fmt.Fprintf(&b, "; missing organization/project scopes: %s", strings.Join(scopes, ", ")) - } - return b.String() -} - -func PreflightImport(ctx context.Context, reader DumpReader, privileges PrivilegeLister, batchSize int) (ImportPreflightReport, error) { - if reader == nil { - return ImportPreflightReport{}, fmt.Errorf("reader is required") - } - if privileges == nil { - return ImportPreflightReport{}, fmt.Errorf("privilege lister is required") - } - userPrivileges, err := privileges.UserPrivileges(ctx) - if err != nil { - return ImportPreflightReport{}, fmt.Errorf("load target privileges: %w", err) - } - userPrivileges = normalizePrivileges(userPrivileges) - - report := ImportPreflightReport{} - required := map[string]struct{}{} - missing := map[string]struct{}{} - - err = reader.ReadBatches(ctx, batchSize, func(records []MigrationRecord) error { - for _, record := range records { - if err := Validate(record); err != nil { - continue - } - report.Records++ - resources := syfoncommon.NormalizeAccessResources(migrationRecordControlledAccess(record)) - for _, resource := range resources { - required[resource] = struct{}{} - } - if hasCreateAccess(userPrivileges, resources) { - continue - } - report.MissingRecords++ - if report.FirstDeniedRecord == "" { - report.FirstDeniedRecord = record.ID - } - for _, resource := range resources { - missing[resource] = struct{}{} - } - } - return nil - }) - if err != nil { - return report, err - } - - report.RequiredResources = sortedMapKeys(required) - report.MissingResources = sortedMapKeys(missing) - if report.MissingRecords > 0 { - return report, &ImportPreflightError{Report: report} - } - return report, nil -} - -func extractUserPrivileges(resourceAccess map[string]any) map[string]map[string]bool { - out := make(map[string]map[string]bool, len(resourceAccess)) - for rawPath, rawEntries := range resourceAccess { - path := syfoncommon.NormalizeAccessResource(rawPath) - if path == "" { - continue - } - methods := map[string]bool{} - entries, ok := rawEntries.([]any) - if !ok { - out[path] = methods - continue - } - for _, entry := range entries { - mm, ok := entry.(map[string]any) - if !ok { - continue - } - service, _ := mm["service"].(string) - if service != "" && service != "indexd" && service != "drs" && service != "*" { - continue - } - method, _ := mm["method"].(string) - method = strings.TrimSpace(method) - if method == "" { - continue - } - methods[method] = true - } - out[path] = methods - } - return out -} - -func hasCreateAccess(privileges map[string]map[string]bool, resources []string) bool { - resources = syfoncommon.NormalizeAccessResources(resources) - if len(resources) == 0 { - return false - } - for _, resource := range resources { - methods, ok := privileges[resource] - if !ok { - continue - } - if methods["create"] || methods["*"] { - return true - } - } - return false -} - -func normalizePrivileges(privileges map[string]map[string]bool) map[string]map[string]bool { - if len(privileges) == 0 { - return nil - } - out := make(map[string]map[string]bool, len(privileges)) - for rawResource, methods := range privileges { - resource := syfoncommon.NormalizeAccessResource(rawResource) - if resource == "" { - continue - } - if out[resource] == nil { - out[resource] = map[string]bool{} - } - for method, allowed := range methods { - if allowed { - out[resource][method] = true - } - } - } - return out -} - -func sortedMapKeys(in map[string]struct{}) []string { - out := make([]string, 0, len(in)) - for key := range in { - out = append(out, key) - } - sort.Strings(out) - return out -} - -func FormatPreflightScopes(resources []string) []string { - out := make([]string, 0, len(resources)) - for _, resource := range resources { - formatted := formatPreflightScope(resource) - if formatted == "" { - continue - } - out = append(out, formatted) - } - return out -} - -func formatPreflightScope(resource string) string { - resource = syfoncommon.NormalizeAccessResource(resource) - org, project, ok := syfoncommon.ResourceScope(resource) - if !ok { - return resource - } - if org != "" && project != "" { - return org + "/" + project - } - if strings.TrimSpace(org) != "" { - return org + "/*" - } - return resource -} diff --git a/migrate/run.go b/migrate/run.go deleted file mode 100644 index 92e8eb62..00000000 --- a/migrate/run.go +++ /dev/null @@ -1,153 +0,0 @@ -package migrate - -import ( - "context" - "fmt" - "log/slog" - "time" -) - -func Run(ctx context.Context, source SourceLister, loader Loader, cfg Config) (Stats, error) { - if source == nil { - return Stats{}, fmt.Errorf("source is required") - } - if loader == nil && !cfg.DryRun { - return Stats{}, fmt.Errorf("loader is required") - } - batchSize := cfg.BatchSize - if batchSize <= 0 { - batchSize = 500 - } - if batchSize > 1024 { - batchSize = 1024 - } - sweeps := cfg.Sweeps - if sweeps <= 0 { - sweeps = 1 - } - - var stats Stats - seen := map[string]struct{}{} - now := time.Now().UTC() - - for sweep := 0; sweep < sweeps; sweep++ { - start := "" - sweepNew := 0 - for { - if cfg.Limit > 0 && stats.CountOfUniqueIDs >= cfg.Limit { - return stats, nil - } - fetchLimit := batchSize - if cfg.Limit > 0 { - remaining := cfg.Limit - stats.CountOfUniqueIDs - if remaining < fetchLimit { - fetchLimit = remaining - } - } - records, err := source.ListPage(ctx, fetchLimit, start) - if err != nil { - return stats, err - } - if len(records) == 0 { - break - } - - stats.Fetched += len(records) - filtered := make([]IndexdRecord, 0, len(records)) - maxDID := start - for _, record := range records { - if record.DID > maxDID { - maxDID = record.DID - } - if record.DID == "" { - filtered = append(filtered, record) - continue - } - if _, ok := seen[record.DID]; ok { - continue - } - seen[record.DID] = struct{}{} - filtered = append(filtered, record) - sweepNew++ - } - stats.CountOfUniqueIDs = len(seen) - if cfg.Limit > 0 && len(filtered) > fetchLimit { - filtered = filtered[:fetchLimit] - } - - transformed, transformErrs := TransformBatch(filtered, cfg.DefaultAuthz, now) - stats.Errors += len(transformErrs) - for _, err := range transformErrs { - slog.Warn("migration transform failed", "did", err.DID, "err", err.Err) - } - - valid := make([]MigrationRecord, 0, len(transformed)) - for _, record := range transformed { - if err := Validate(record); err != nil { - stats.Skipped++ - slog.Warn("migration validation skipped record", "id", record.ID, "err", err) - continue - } - valid = append(valid, record) - } - stats.Transformed += len(valid) - - if len(valid) > 0 { - if cfg.DryRun { - stats.Loaded += len(valid) - } else { - if err := loader.LoadBatch(ctx, valid); err != nil { - return stats, err - } - stats.Loaded += len(valid) - } - } - - if maxDID == "" || maxDID == start || len(records) < fetchLimit { - break - } - start = maxDID - } - - if sweepNew == 0 { - break - } - } - - return stats, nil -} - -func Import(ctx context.Context, reader DumpReader, loader Loader, batchSize int) (Stats, error) { - if reader == nil { - return Stats{}, fmt.Errorf("reader is required") - } - if loader == nil { - return Stats{}, fmt.Errorf("loader is required") - } - if batchSize <= 0 { - batchSize = 500 - } - var stats Stats - err := reader.ReadBatches(ctx, batchSize, func(records []MigrationRecord) error { - valid := make([]MigrationRecord, 0, len(records)) - for _, record := range records { - stats.Fetched++ - if err := Validate(record); err != nil { - stats.Skipped++ - slog.Warn("migration import skipped record", "id", record.ID, "err", err) - continue - } - valid = append(valid, record) - } - stats.Transformed += len(valid) - if len(valid) == 0 { - return nil - } - if err := loader.LoadBatch(ctx, valid); err != nil { - return err - } - stats.Loaded += len(valid) - return nil - }) - return stats, err -} diff --git a/migrate/run_test.go b/migrate/run_test.go deleted file mode 100644 index 15b0752a..00000000 --- a/migrate/run_test.go +++ /dev/null @@ -1,287 +0,0 @@ -package migrate - -import ( - "context" - "encoding/json" - "errors" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/calypr/syfon/apigen/server/drs" -) - -func TestRunPaginatesAndLoadsAllRecords(t *testing.T) { - size := int64(1) - source := &fakeSource{pages: [][]IndexdRecord{ - { - {DID: "a", Size: &size, Hashes: map[string]string{"sha256": "sha-a"}}, - {DID: "b", Size: &size, Hashes: map[string]string{"sha256": "sha-b"}}, - }, - { - {DID: "c", Size: &size, Hashes: map[string]string{"sha256": "sha-c"}}, - }, - }} - loader := &fakeLoader{} - - stats, err := Run(context.Background(), source, loader, Config{BatchSize: 2, Sweeps: 1}) - if err != nil { - t.Fatalf("Run returned error: %v", err) - } - if stats.Fetched != 3 || stats.Loaded != 3 || stats.CountOfUniqueIDs != 3 { - t.Fatalf("unexpected stats: %+v", stats) - } - if len(source.starts) != 2 || source.starts[0] != "" || source.starts[1] != "b" { - t.Fatalf("unexpected pagination starts: %+v", source.starts) - } - if len(loader.records) != 3 { - t.Fatalf("expected 3 loaded records, got %d", len(loader.records)) - } -} - -func TestRunDryRunDoesNotRequireLoader(t *testing.T) { - size := int64(1) - source := &fakeSource{pages: [][]IndexdRecord{{ - {DID: "a", Size: &size, Hashes: map[string]string{"sha256": "sha-a"}}, - }}} - - stats, err := Run(context.Background(), source, nil, Config{BatchSize: 100, DryRun: true}) - if err != nil { - t.Fatalf("Run returned error: %v", err) - } - if stats.Fetched != 1 || stats.Loaded != 1 || stats.CountOfUniqueIDs != 1 { - t.Fatalf("unexpected stats: %+v", stats) - } -} - -func TestHTTPClientEndToEndWithMockIndexdAndSyfon(t *testing.T) { - size := int64(7) - var loaded struct { - Records []struct { - Did string `json:"did"` - ControlledAccess []string `json:"controlled_access"` - AccessMethods []struct { - Type string `json:"type"` - AccessUrl struct { - Url string `json:"url"` - } `json:"access_url"` - } `json:"access_methods"` - } `json:"records"` - } - source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/index" { - t.Fatalf("unexpected source path: %s", r.URL.Path) - } - if got := r.URL.Query().Get("form"); got != "" { - t.Fatalf("did not expect form query, got %q", got) - } - _ = json.NewEncoder(w).Encode(IndexdPage{Records: []IndexdRecord{{ - DID: "dg.test/1", - Size: &size, - URLs: []string{"s3://bucket/key"}, - Hashes: map[string]string{"sha256": "sha"}, - Authz: []string{"https://calypr.org/program/foo/project/bar"}, - }}}) - })) - defer source.Close() - - target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/index/bulk" { - t.Fatalf("unexpected target path: %s", r.URL.Path) - } - if err := json.NewDecoder(r.Body).Decode(&loaded); err != nil { - t.Fatalf("decode target body: %v", err) - } - w.WriteHeader(http.StatusOK) - })) - defer target.Close() - - sourceClient, err := NewHTTPClient(source.URL, AuthConfig{}, source.Client()) - if err != nil { - t.Fatalf("NewHTTPClient source: %v", err) - } - targetClient, err := NewHTTPClient(target.URL, AuthConfig{}, target.Client()) - if err != nil { - t.Fatalf("NewHTTPClient target: %v", err) - } - - stats, err := Run(context.Background(), sourceClient, targetClient, Config{BatchSize: 100}) - if err != nil { - t.Fatalf("Run returned error: %v", err) - } - if stats.Loaded != 1 || len(loaded.Records) != 1 { - t.Fatalf("expected one loaded record, stats=%+v body=%+v", stats, loaded) - } - if loaded.Records[0].Did != "dg.test/1" { - t.Fatalf("unexpected loaded payload: %+v", loaded.Records[0]) - } - if len(loaded.Records[0].ControlledAccess) != 1 || loaded.Records[0].ControlledAccess[0] != "/organization/foo/project/bar" { - t.Fatalf("expected controlled_access payload, got %+v", loaded.Records[0]) - } - if len(loaded.Records[0].AccessMethods) != 1 || loaded.Records[0].AccessMethods[0].AccessUrl.Url != "s3://bucket/key" { - t.Fatalf("expected access_methods payload, got %+v", loaded.Records[0]) - } -} - -func TestHTTPClientRetriesTimeout(t *testing.T) { - transport := &retryTransport{} - client, err := NewHTTPClient("https://example.org/index", AuthConfig{}, &http.Client{Transport: transport}) - if err != nil { - t.Fatalf("NewHTTPClient: %v", err) - } - - records, err := client.ListPage(context.Background(), 1, "") - if err != nil { - t.Fatalf("ListPage returned error: %v", err) - } - if transport.calls != 2 { - t.Fatalf("expected one retry, got %d calls", transport.calls) - } - if len(records) != 1 || records[0].DID != "a" { - t.Fatalf("unexpected records: %+v", records) - } -} - -func TestPreflightImportReportsAllMissingCreateScopes(t *testing.T) { - reader := &fakeDumpReader{batches: [][]MigrationRecord{ - { - {ID: "allowed", Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-1"}}, Authz: []string{"/programs/cbds/projects/allowed"}}, - {ID: "denied-1", Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-2"}}, Authz: []string{"/programs/cbds/projects/missing-one"}}, - }, - { - {ID: "denied-2", Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-3"}}, ControlledAccess: []string{"/programs/aced/projects/missing-two"}}, - }, - }} - privileges := fakePrivilegeLister{privileges: map[string]map[string]bool{ - "/programs/cbds/projects/allowed": {"create": true}, - }} - - _, err := PreflightImport(context.Background(), reader, privileges, 500) - if err == nil { - t.Fatal("expected preflight to fail") - } - var preflightErr *ImportPreflightError - if !errors.As(err, &preflightErr) { - t.Fatalf("expected ImportPreflightError, got %T %v", err, err) - } - report := preflightErr.Report - if report.Records != 3 || report.MissingRecords != 2 { - t.Fatalf("unexpected report counts: %+v", report) - } - if report.FirstDeniedRecord != "denied-1" { - t.Fatalf("unexpected first denied record: %q", report.FirstDeniedRecord) - } - body := err.Error() - if !strings.Contains(body, "cbds/missing-one") || !strings.Contains(body, "aced/missing-two") { - t.Fatalf("expected formatted missing scopes in error, got %q", body) - } -} - -func TestHTTPClientUserPrivileges(t *testing.T) { - var gotAuth string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/user/user" { - t.Fatalf("unexpected path: %s", r.URL.Path) - } - gotAuth = r.Header.Get("Authorization") - resp := map[string]any{ - "authz": map[string]any{ - "/programs/cbds/projects/p1": []any{ - map[string]any{"service": "indexd", "method": "read"}, - map[string]any{"service": "drs", "method": "create"}, - }, - "/programs/cbds/projects/p2": []any{ - map[string]any{"service": "fence", "method": "create"}, - }, - }, - } - _ = json.NewEncoder(w).Encode(resp) - })) - defer server.Close() - - client, err := NewHTTPClient(server.URL+"/index", AuthConfig{BearerToken: "tok"}, server.Client()) - if err != nil { - t.Fatalf("NewHTTPClient: %v", err) - } - privileges, err := client.UserPrivileges(context.Background()) - if err != nil { - t.Fatalf("UserPrivileges returned error: %v", err) - } - if gotAuth != "Bearer tok" { - t.Fatalf("expected bearer auth header, got %q", gotAuth) - } - if !privileges["/organization/cbds/project/p1"]["create"] { - t.Fatalf("expected create privilege for p1, got %+v", privileges) - } - if privileges["/organization/cbds/project/p2"]["create"] { - t.Fatalf("did not expect non-indexd/drs create privilege to be retained: %+v", privileges) - } -} - -type fakeSource struct { - pages [][]IndexdRecord - starts []string -} - -func (s *fakeSource) ListPage(ctx context.Context, limit int, start string) ([]IndexdRecord, error) { - s.starts = append(s.starts, start) - if len(s.starts) > len(s.pages) { - return nil, nil - } - return s.pages[len(s.starts)-1], nil -} - -type fakeLoader struct { - records []MigrationRecord -} - -func (l *fakeLoader) LoadBatch(ctx context.Context, records []MigrationRecord) error { - l.records = append(l.records, records...) - return nil -} - -type fakeDumpReader struct { - batches [][]MigrationRecord -} - -func (r *fakeDumpReader) ReadBatches(ctx context.Context, batchSize int, fn func([]MigrationRecord) error) error { - for _, batch := range r.batches { - if err := fn(batch); err != nil { - return err - } - } - return nil -} - -type fakePrivilegeLister struct { - privileges map[string]map[string]bool -} - -func (l fakePrivilegeLister) UserPrivileges(ctx context.Context) (map[string]map[string]bool, error) { - return l.privileges, nil -} - -type retryTransport struct { - calls int -} - -func (t *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) { - t.calls++ - if t.calls == 1 { - return nil, timeoutErr{} - } - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(strings.NewReader(`{"records":[{"did":"a","size":1,"hashes":{"sha256":"sha-a"}}]}`)), - Header: make(http.Header), - Request: req, - }, nil -} - -type timeoutErr struct{} - -func (timeoutErr) Error() string { return "timeout" } -func (timeoutErr) Timeout() bool { return true } -func (timeoutErr) Temporary() bool { return true } diff --git a/migrate/transform.go b/migrate/transform.go deleted file mode 100644 index 5dc48c1b..00000000 --- a/migrate/transform.go +++ /dev/null @@ -1,258 +0,0 @@ -package migrate - -import ( - "fmt" - "net/url" - "sort" - "strings" - "time" - - "github.com/calypr/syfon/apigen/server/drs" - syfoncommon "github.com/calypr/syfon/common" - "github.com/calypr/syfon/internal/models" -) - -var indexdDateFormats = []string{ - time.RFC3339Nano, - time.RFC3339, - "2006-01-02T15:04:05.999999", - "2006-01-02T15:04:05.999999Z", - "2006-01-02T15:04:05", - "2006-01-02 15:04:05.999999", - "2006-01-02 15:04:05", - "2006-01-02", -} - -type TransformError struct { - DID string - Err error -} - -func (e TransformError) Error() string { - return fmt.Sprintf("transform %s: %v", e.DID, e.Err) -} - -func Transform(rec IndexdRecord, defaultAuthz []string, now time.Time) (MigrationRecord, error) { - did := strings.TrimSpace(rec.DID) - if did == "" { - return MigrationRecord{}, fmt.Errorf("record has empty did") - } - if now.IsZero() { - now = time.Now().UTC() - } - - controlledAccess := syfoncommon.NormalizeAccessResources(rec.Authz) - if len(controlledAccess) == 0 { - controlledAccess = syfoncommon.NormalizeAccessResources(defaultAuthz) - } - - checksums := checksumsFromHashes(rec.Hashes) - created := firstParsedTime(now, rec.CreatedTime, rec.CreatedDate, rec.ContentCreatedDate) - updated := firstParsedTime(created, rec.UpdatedTime, rec.UpdatedDate, rec.ContentUpdatedDate) - accessMethods := accessMethodsFromURLs(rec.URLs) - - return MigrationRecord{ - ID: did, - Name: stringPtrValue(rec.FileName), - Size: int64PtrValue(rec.Size), - Version: stringPtrValue(rec.Version), - Description: stringPtrValue(rec.Description), - CreatedTime: created, - UpdatedTime: &updated, - Checksums: checksums, - AccessMethods: accessMethods, - ControlledAccess: controlledAccess, - Authz: controlledAccess, - }, nil -} - -func TransformBatch(records []IndexdRecord, defaultAuthz []string, now time.Time) ([]MigrationRecord, []TransformError) { - out := make([]MigrationRecord, 0, len(records)) - var errs []TransformError - for _, rec := range records { - mr, err := Transform(rec, defaultAuthz, now) - if err != nil { - errs = append(errs, TransformError{DID: rec.DID, Err: err}) - continue - } - out = append(out, mr) - } - return out, errs -} - -func Validate(record MigrationRecord) error { - if strings.TrimSpace(record.ID) == "" { - return fmt.Errorf("id is empty") - } - if len(record.Checksums) == 0 { - return fmt.Errorf("no checksums") - } - for _, checksum := range record.Checksums { - if strings.TrimSpace(checksum.Type) == "" || strings.TrimSpace(checksum.Checksum) == "" { - return fmt.Errorf("checksum entry has empty type or value") - } - } - return nil -} - -func MigrationRecordToInternalObject(record MigrationRecord) (models.InternalObject, error) { - if err := Validate(record); err != nil { - return models.InternalObject{}, err - } - controlledAccess := record.ControlledAccess - if len(controlledAccess) == 0 { - controlledAccess = record.Authz - } - controlledAccess = syfoncommon.NormalizeAccessResources(controlledAccess) - authzMap := syfoncommon.ControlledAccessToAuthzMap(controlledAccess) - updated := record.UpdatedTime - if updated == nil { - t := record.CreatedTime - updated = &t - } - obj := drs.DrsObject{ - Id: record.ID, - SelfUri: "drs://" + record.ID, - Size: record.Size, - Name: record.Name, - Version: record.Version, - Description: record.Description, - CreatedTime: record.CreatedTime, - UpdatedTime: updated, - Checksums: append([]drs.Checksum(nil), record.Checksums...), - AccessMethods: nil, - } - if authzMap != nil { - obj.ControlledAccess = &controlledAccess - } - if len(record.AccessMethods) > 0 { - methods := append([]drs.AccessMethod(nil), record.AccessMethods...) - obj.AccessMethods = &methods - } - return models.InternalObject{ - DrsObject: obj, - Authorizations: authzMap, - }, nil -} - -func firstParsedTime(fallback time.Time, values ...*string) time.Time { - for _, value := range values { - if value == nil { - continue - } - if parsed := parseIndexdDate(*value); !parsed.IsZero() { - return parsed - } - } - return fallback.UTC() -} - -func parseIndexdDate(raw string) time.Time { - raw = strings.TrimSpace(raw) - if raw == "" { - return time.Time{} - } - for _, layout := range indexdDateFormats { - if t, err := time.Parse(layout, raw); err == nil { - return t.UTC() - } - } - return time.Time{} -} - -func checksumsFromHashes(hashes map[string]string) []drs.Checksum { - if len(hashes) == 0 { - return nil - } - keys := make([]string, 0, len(hashes)) - for key := range hashes { - keys = append(keys, key) - } - sort.Strings(keys) - out := make([]drs.Checksum, 0, len(keys)) - for _, typ := range keys { - checksum := strings.TrimSpace(hashes[typ]) - typ = strings.TrimSpace(typ) - if typ == "" || checksum == "" { - continue - } - out = append(out, drs.Checksum{Type: typ, Checksum: checksum}) - } - return out -} - -func accessMethodsFromURLs(urls []string) []drs.AccessMethod { - if len(urls) == 0 { - return nil - } - seen := map[string]struct{}{} - out := make([]drs.AccessMethod, 0, len(urls)) - for _, rawURL := range urls { - rawURL = strings.TrimSpace(rawURL) - if rawURL == "" { - continue - } - if _, ok := seen[rawURL]; ok { - continue - } - seen[rawURL] = struct{}{} - methodType := accessMethodType(rawURL) - accessID := string(methodType) - method := drs.AccessMethod{ - Type: methodType, - AccessId: &accessID, - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: rawURL}, - } - out = append(out, method) - } - return out -} - -func accessMethodType(rawURL string) drs.AccessMethodType { - parsed, err := url.Parse(strings.TrimSpace(rawURL)) - if err != nil || parsed.Scheme == "" { - return drs.AccessMethodTypeHttps - } - return drs.AccessMethodType(strings.ToLower(parsed.Scheme)) -} - -func uniqueStrings(values []string) []string { - if len(values) == 0 { - return nil - } - seen := map[string]struct{}{} - out := make([]string, 0, len(values)) - for _, value := range values { - value = strings.TrimSpace(value) - if value == "" { - continue - } - if _, ok := seen[value]; ok { - continue - } - seen[value] = struct{}{} - out = append(out, value) - } - return out -} - -func stringPtrValue(value *string) *string { - if value == nil { - return nil - } - trimmed := strings.TrimSpace(*value) - if trimmed == "" { - return nil - } - return &trimmed -} - -func int64PtrValue(value *int64) int64 { - if value == nil { - return 0 - } - return *value -} diff --git a/migrate/transform_test.go b/migrate/transform_test.go deleted file mode 100644 index c56a193e..00000000 --- a/migrate/transform_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package migrate - -import ( - "testing" - "time" -) - -func TestTransformIndexdRecord(t *testing.T) { - size := int64(42) - name := "file.bam" - version := "v1" - desc := "source record" - created := "2024-03-04T05:06:07Z" - rec := IndexdRecord{ - DID: "dg.test/abc", - Size: &size, - FileName: &name, - Version: &version, - Description: &desc, - CreatedDate: &created, - URLs: []string{"s3://bucket/key", "s3://bucket/key", "gs://bucket/key"}, - Hashes: map[string]string{"sha256": "abc123", "md5": "def456"}, - Authz: []string{"/programs/ohsu/projects/brca", "/programs/ohsu/projects/brca"}, - } - - got, err := Transform(rec, nil, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) - if err != nil { - t.Fatalf("Transform returned error: %v", err) - } - if got.ID != rec.DID || got.Size != size { - t.Fatalf("identity fields not preserved: %+v", got) - } - if got.Name == nil || *got.Name != name || got.Version == nil || *got.Version != version || got.Description == nil || *got.Description != desc { - t.Fatalf("metadata not preserved: %+v", got) - } - if len(got.Checksums) != 2 { - t.Fatalf("expected 2 checksums, got %+v", got.Checksums) - } - if len(got.AccessMethods) != 2 { - t.Fatalf("expected de-duplicated access methods, got %+v", got.AccessMethods) - } - if got.CreatedTime.Format(time.RFC3339) != created { - t.Fatalf("created time not parsed: %s", got.CreatedTime.Format(time.RFC3339)) - } - internal, err := MigrationRecordToInternalObject(got) - if err != nil { - t.Fatalf("MigrationRecordToInternalObject returned error: %v", err) - } - if internal.ControlledAccess == nil || len(*internal.ControlledAccess) != 1 || (*internal.ControlledAccess)[0] != "/organization/ohsu/project/brca" { - t.Fatalf("expected controlled access on object, got %+v", internal.ControlledAccess) - } -} - -func TestTransformAppliesDefaultAuthz(t *testing.T) { - size := int64(1) - rec := IndexdRecord{ - DID: "dg.test/default-authz", - Size: &size, - URLs: []string{"s3://bucket/key"}, - Hashes: map[string]string{"sha256": "abc123"}, - } - - got, err := Transform(rec, []string{"/programs/open"}, time.Now()) - if err != nil { - t.Fatalf("Transform returned error: %v", err) - } - if len(got.ControlledAccess) != 1 || got.ControlledAccess[0] != "/organization/open" { - t.Fatalf("expected default controlled access, got %+v", got.ControlledAccess) - } - internal, err := MigrationRecordToInternalObject(got) - if err != nil { - t.Fatalf("MigrationRecordToInternalObject returned error: %v", err) - } - if internal.Authorizations == nil || len(internal.Authorizations["open"]) != 0 { - t.Fatalf("unexpected internal authorizations: %+v", internal.Authorizations) - } -} diff --git a/migrate/types.go b/migrate/types.go deleted file mode 100644 index 93e225e2..00000000 --- a/migrate/types.go +++ /dev/null @@ -1,96 +0,0 @@ -package migrate - -import ( - "context" - "time" - - "github.com/calypr/syfon/apigen/server/drs" -) - -// IndexdRecord is the source record shape returned by Gen3 Indexd APIs. -// Deprecated Indexd fields are accepted so source payloads decode cleanly, -// but they are intentionally not loaded into Syfon. -type IndexdRecord struct { - DID string `json:"did"` - Size *int64 `json:"size,omitempty"` - FileName *string `json:"file_name,omitempty"` - Version *string `json:"version,omitempty"` - Description *string `json:"description,omitempty"` - URLs []string `json:"urls,omitempty"` - Hashes map[string]string `json:"hashes,omitempty"` - Authz []string `json:"authz,omitempty"` - CreatedDate *string `json:"created_date,omitempty"` - UpdatedDate *string `json:"updated_date,omitempty"` - CreatedTime *string `json:"created_time,omitempty"` - UpdatedTime *string `json:"updated_time,omitempty"` - ContentCreatedDate *string `json:"content_created_date,omitempty"` - ContentUpdatedDate *string `json:"content_updated_date,omitempty"` - Aliases []string `json:"aliases,omitempty"` - Form *string `json:"form,omitempty"` - - Baseid *string `json:"baseid,omitempty"` - Rev *string `json:"rev,omitempty"` - Uploader *string `json:"uploader,omitempty"` - ACL []string `json:"acl,omitempty"` - Metadata map[string]interface{} `json:"metadata,omitempty"` - URLsMetadata map[string]interface{} `json:"urls_metadata,omitempty"` -} - -type IndexdPage struct { - Records []IndexdRecord `json:"records"` - IDs []string `json:"ids"` - Start string `json:"start"` - Page *int `json:"page,omitempty"` - Limit *int `json:"limit,omitempty"` -} - -type SourceLister interface { - ListPage(ctx context.Context, limit int, start string) ([]IndexdRecord, error) -} - -type Loader interface { - LoadBatch(ctx context.Context, records []MigrationRecord) error -} - -type PrivilegeLister interface { - UserPrivileges(ctx context.Context) (map[string]map[string]bool, error) -} - -type DumpReader interface { - ReadBatches(ctx context.Context, batchSize int, fn func([]MigrationRecord) error) error -} - -type MigrationRecord struct { - ID string `json:"id"` - Name *string `json:"name,omitempty"` - Size int64 `json:"size"` - Version *string `json:"version,omitempty"` - Description *string `json:"description,omitempty"` - CreatedTime time.Time `json:"created_time"` - UpdatedTime *time.Time `json:"updated_time,omitempty"` - Checksums []drs.Checksum `json:"checksums"` - AccessMethods []drs.AccessMethod `json:"access_methods,omitempty"` - ControlledAccess []string `json:"controlled_access,omitempty"` - Authz []string `json:"authz,omitempty"` -} - -type MigrationRequest struct { - Records []MigrationRecord `json:"records"` -} - -type Stats struct { - Fetched int - Transformed int - Loaded int - Skipped int - Errors int - CountOfUniqueIDs int -} - -type Config struct { - BatchSize int - Limit int - DryRun bool - DefaultAuthz []string - Sweeps int -} diff --git a/mkdocs.yml b/mkdocs.yml index 245328a6..32724e9c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -46,7 +46,6 @@ nav: - Configuration: configuration.md - Encryption: encryption.md - Deployment: deployment.md - - Indexd to Syfon Migration: operator-guide-migration-indexd.md - Troubleshooting: troubleshooting.md - Code Generation: operator-guide-code-generation.md - Advanced: From ff5c22f71d7b03ce9b23b34f093d81ddc9834726 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 29 Jun 2026 11:41:14 -0700 Subject: [PATCH 07/24] remove dead code --- client/bucket/location.go | 24 -- client/bucket/location_test.go | 65 ----- client/client.go | 14 -- client/client_test.go | 34 ++- client/common/common.go | 109 --------- client/common/common_test.go | 144 ----------- client/common/constants.go | 1 + client/common/isHidden_notwindows.go | 14 -- client/common/isHidden_windows.go | 30 --- client/common/progress.go | 24 -- client/config/config_test.go | 12 - client/config/validate.go | 13 - client/hash/hash.go | 120 +-------- client/hash/hash_test.go | 139 +---------- client/logs/factory.go | 103 -------- client/logs/handler.go | 102 -------- client/logs/logger.go | 35 --- client/logs/logger_test.go | 210 ---------------- client/logs/noop.go | 11 - client/logs/scoreboard.go | 75 ------ client/logs/tee_logger.go | 27 +- client/request/request.go | 12 +- client/request/request_test.go | 5 +- client/services/data_service.go | 13 - .../data_service_canonical_url_test.go | 9 - client/transfer.go | 230 ------------------ client/transfer/download/downloader.go | 138 ----------- client/transfer/download/file_info.go | 56 ----- client/transfer/download/transfer.go | 35 --- client/transfer/download/transfer_test.go | 71 ------ client/transfer/download/types.go | 74 ------ client/transfer/download/utils.go | 77 ------ client/transfer/engine/downloader.go | 33 +-- client/transfer/engine/engine_test.go | 44 +--- client/transfer/engine/progress_reader.go | 7 +- client/transfer/noop_logger.go | 2 - client/transfer/retry.go | 6 - client/transfer/retry_test.go | 23 +- client/transfer/types.go | 2 - client/transfer/upload/utils.go | 102 -------- client/transfer/upload/utils_test.go | 124 ---------- cmd/cliauth/auth.go | 4 +- cmd/download/main.go | 57 +---- cmd/download/main_test.go | 86 ------- cmd/projectcopy/helpers.go | 4 +- cmd/validate/openapi_validator.go | 4 +- cmd/validate/validate_test.go | 4 +- common/drs.go | 125 ---------- common/drs_test.go | 36 --- common/resource.go | 31 +-- common/resource_test.go | 28 +-- internal/api/middleware/middleware_test.go | 2 +- internal/api/middleware/plugin_clients.go | 7 - .../api/middleware/plugin_clients_test.go | 210 +--------------- internal/common/checksum.go | 18 +- internal/common/checksum_test.go | 5 +- internal/common/cloud.go | 33 +-- internal/common/cloud_test.go | 2 +- internal/common/errors.go | 3 - internal/common/errors_test.go | 3 - internal/common/ptr.go | 24 +- internal/common/util.go | 42 +--- internal/common/util_test.go | 62 ----- internal/converters/converters.go | 167 ------------- internal/converters/converters_test.go | 84 ------- internal/converters/util.go | 13 - internal/core/bucket_catalog.go | 15 +- internal/core/object_storage_inspect.go | 9 +- internal/core/service_test.go | 54 ---- internal/crypto/crypto.go | 14 +- internal/db/postgres/metrics.go | 78 +----- internal/db/postgres/metrics_sqlmock_test.go | 11 - internal/db/sqlite/metrics.go | 76 +----- internal/db/sqlite/objects.go | 3 - internal/testutils/mocks.go | 60 +---- 75 files changed, 73 insertions(+), 3670 deletions(-) delete mode 100644 client/bucket/location.go delete mode 100644 client/bucket/location_test.go delete mode 100644 client/common/isHidden_notwindows.go delete mode 100644 client/common/isHidden_windows.go delete mode 100644 client/logs/factory.go delete mode 100644 client/logs/handler.go delete mode 100644 client/logs/logger.go delete mode 100644 client/logs/logger_test.go delete mode 100644 client/logs/noop.go delete mode 100644 client/logs/scoreboard.go delete mode 100644 client/transfer.go delete mode 100644 client/transfer/download/downloader.go delete mode 100644 client/transfer/download/file_info.go delete mode 100644 client/transfer/download/types.go delete mode 100644 client/transfer/download/utils.go delete mode 100644 client/transfer/upload/utils_test.go delete mode 100644 common/drs.go delete mode 100644 common/drs_test.go delete mode 100644 internal/converters/converters.go delete mode 100644 internal/converters/converters_test.go delete mode 100644 internal/converters/util.go diff --git a/client/bucket/location.go b/client/bucket/location.go deleted file mode 100644 index dfda1d95..00000000 --- a/client/bucket/location.go +++ /dev/null @@ -1,24 +0,0 @@ -package bucket - -import ( - "fmt" - "net/url" - "strings" -) - -// ParseStorageLocation extracts the bucket and key from a raw storage URL (e.g., s3://bucket/key). -func ParseStorageLocation(rawURL string) (bucketName, key string, err error) { - if rawURL == "" { - return "", "", fmt.Errorf("empty storage URL") - } - parsed, err := url.Parse(rawURL) - if err != nil { - return "", "", fmt.Errorf("failed to parse storage URL %q: %w", rawURL, err) - } - bucketName = strings.TrimSpace(parsed.Host) - key = strings.Trim(strings.TrimSpace(parsed.Path), "/") - if bucketName == "" || key == "" { - return "", "", fmt.Errorf("invalid storage location %q: missing bucket or key", rawURL) - } - return bucketName, key, nil -} diff --git a/client/bucket/location_test.go b/client/bucket/location_test.go deleted file mode 100644 index 9182fbb6..00000000 --- a/client/bucket/location_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package bucket - -import ( - "context" - "testing" - - bucketapi "github.com/calypr/syfon/apigen/client/bucketapi" -) - -func TestParseStorageLocation(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - rawURL string - wantBucket string - wantKey string - wantErr bool - }{ - {name: "valid s3 url", rawURL: "s3://bucket/path/to/file.txt", wantBucket: "bucket", wantKey: "path/to/file.txt"}, - {name: "trim key slashes", rawURL: "s3://bucket//nested/file.txt/", wantBucket: "bucket", wantKey: "nested/file.txt"}, - {name: "empty", rawURL: "", wantErr: true}, - {name: "parse error", rawURL: "://bad", wantErr: true}, - {name: "missing bucket", rawURL: "s3:///path/to/file.txt", wantErr: true}, - {name: "missing key", rawURL: "s3://bucket/", wantErr: true}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - bucketName, key, err := ParseStorageLocation(tc.rawURL) - if tc.wantErr { - if err == nil { - t.Fatalf("expected error for %q", tc.rawURL) - } - return - } - if err != nil { - t.Fatalf("ParseStorageLocation returned error: %v", err) - } - if bucketName != tc.wantBucket || key != tc.wantKey { - t.Fatalf("unexpected parse result: bucket=%q key=%q want bucket=%q key=%q", bucketName, key, tc.wantBucket, tc.wantKey) - } - }) - } -} - -func TestValidateBucketNonS3Providers(t *testing.T) { - t.Parallel() - - ctx := context.Background() - for _, provider := range []string{"gcs", "gs", "azure", "custom"} { - provider := provider - t.Run(provider, func(t *testing.T) { - req := bucketapi.PutBucketRequest{ - Bucket: "bucket-1", - Organization: "org", - ProjectId: "proj", - Provider: &provider, - } - if err := ValidateBucket(ctx, req); err != nil { - t.Fatalf("ValidateBucket returned error for provider %q: %v", provider, err) - } - }) - } -} diff --git a/client/client.go b/client/client.go index 4b9a6516..0bc6c689 100644 --- a/client/client.go +++ b/client/client.go @@ -59,21 +59,7 @@ type Client struct { type Option func(*Config) -func WithHTTPClient(h *http.Client) Option { - return func(c *Config) { - if h != nil { - c.HTTPClient = h - } - } -} -func WithUserAgent(v string) Option { - return func(c *Config) { - if strings.TrimSpace(v) != "" { - c.UserAgent = strings.TrimSpace(v) - } - } -} func WithBasicAuth(user, pass string) Option { return func(c *Config) { diff --git a/client/client_test.go b/client/client_test.go index 814ba9b6..9966dbd1 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -21,11 +21,11 @@ func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { func newTestClient(t *testing.T, fn roundTripFunc) *Client { t.Helper() httpClient := &http.Client{Transport: fn} - c, err := New("http://example.test", WithHTTPClient(httpClient)) + c, err := NewClient(&Config{Address: "http://example.test", HTTPClient: httpClient}) if err != nil { t.Fatalf("failed to create client: %v", err) } - return c.(*Client) + return c } func TestClientBasicAuthAndUserAgent(t *testing.T) { @@ -48,10 +48,12 @@ func TestClientBasicAuthAndUserAgent(t *testing.T) { }, nil })} - c, err := New("http://example.test", - WithBasicAuth("u", "p"), - WithUserAgent("syfon-test-client"), - WithHTTPClient(httpClient)) + c, err := NewClient(&Config{ + Address: "http://example.test", + BasicAuth: &BasicAuth{Username: "u", Password: "p"}, + UserAgent: "syfon-test-client", + HTTPClient: httpClient, + }) if err != nil { t.Fatalf("failed to create client: %v", err) } @@ -81,13 +83,14 @@ func TestGeneratedClientUsesBasicAuthTransport(t *testing.T) { }, nil })} - raw, err := New("http://example.test", - WithBasicAuth("u", "p"), - WithHTTPClient(httpClient)) + c, err := NewClient(&Config{ + Address: "http://example.test", + BasicAuth: &BasicAuth{Username: "u", Password: "p"}, + HTTPClient: httpClient, + }) if err != nil { t.Fatalf("failed to create client: %v", err) } - c := raw.(*Client) if _, err := c.InternalAPI().InternalListWithResponse(context.Background(), &internalapi.InternalListParams{}); err != nil { t.Fatalf("generated client list failed: %v", err) } @@ -233,12 +236,17 @@ func TestNewClientDefaultWiring(t *testing.T) { }, nil })} - raw, err := New("example.test:8080/", WithHTTPClient(httpClient), WithUserAgent(" "), WithBasicAuth(" ", "ignored")) + raw, err := NewClient(&Config{ + Address: "example.test:8080/", + HTTPClient: httpClient, + UserAgent: " ", + BasicAuth: &BasicAuth{Username: " ", Password: "ignored"}, + }) if err != nil { - t.Fatalf("New returned error: %v", err) + t.Fatalf("NewClient returned error: %v", err) } - c := raw.(*Client) + c := raw if c.Address() != "http://example.test:8080" { t.Fatalf("unexpected address: %q", c.Address()) } diff --git a/client/common/common.go b/client/common/common.go index 091d686d..4402ba2c 100644 --- a/client/common/common.go +++ b/client/common/common.go @@ -3,11 +3,8 @@ package common import ( "bytes" "encoding/json" - "errors" "fmt" "io" - "log" - "net/http" "os" "path/filepath" "strings" @@ -40,100 +37,6 @@ func GetAbsolutePath(filePath string) (string, error) { return filepath.Abs(fullFilePath) } -func ParseFilePaths(filePath string, metadataEnabled bool) ([]string, error) { - fullFilePath, err := GetAbsolutePath(filePath) - if err != nil { - return []string{}, err - } - initialPaths, err := filepath.Glob(fullFilePath) - if err != nil { - return []string{}, err - } - - var errs []error - var finalFilePaths []string - for _, p := range cleanupHiddenFiles(initialPaths) { - file, err := os.Open(p) - if err != nil { - errs = append(errs, fmt.Errorf("file open error for %s: %w", p, err)) - continue - } - - func(filePath string, file *os.File) { - defer file.Close() - - fi, statErr := file.Stat() - if statErr != nil { - errs = append(errs, fmt.Errorf("file stat error for %s: %w", filePath, statErr)) - return - } - if fi.IsDir() { - err = filepath.Walk(filePath, func(path string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - isHidden, err := IsHidden(path) - if err != nil { - return err - } - isMetadata := false - if metadataEnabled { - isMetadata = strings.HasSuffix(path, "_metadata.json") - } - if !fileInfo.IsDir() && !isHidden && !isMetadata { - finalFilePaths = append(finalFilePaths, path) - } - return nil - }) - if err != nil { - errs = append(errs, fmt.Errorf("directory walk error for %s: %w", filePath, err)) - } - } else { - finalFilePaths = append(finalFilePaths, filePath) - } - }(p, file) - } - - return finalFilePaths, errors.Join(errs...) -} - -func cleanupHiddenFiles(filePaths []string) []string { - i := 0 - for _, filePath := range filePaths { - isHidden, err := IsHidden(filePath) - if err != nil { - log.Println("Error occurred when checking hidden files: " + err.Error()) - continue - } - - if isHidden { - log.Printf("File %s is a hidden file and will be skipped\n", filePath) - continue - } - filePaths[i] = filePath - i++ - } - return filePaths[:i] -} - -func CanDownloadFile(signedURL string) error { - req, err := http.NewRequest("GET", signedURL, nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Range", "bytes=0-0") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("error while sending the request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusPartialContent || resp.StatusCode == http.StatusOK { - return nil - } - return ResponseBodyError(resp, "failed to access file") -} - func IsCloudPresignedURL(raw string) bool { return strings.Contains(raw, "X-Amz-Signature") || strings.Contains(raw, "X-Goog-Signature") || @@ -141,15 +44,3 @@ func IsCloudPresignedURL(raw string) bool { strings.Contains(raw, "AWSAccessKeyId=") || strings.Contains(raw, "Expires=") } - -func LoadFailedLog(path string) (map[string]RetryObject, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var m map[string]RetryObject - if err := json.Unmarshal(data, &m); err != nil { - return nil, err - } - return m, nil -} diff --git a/client/common/common_test.go b/client/common/common_test.go index 0ee8a528..1be1d14b 100644 --- a/client/common/common_test.go +++ b/client/common/common_test.go @@ -2,11 +2,8 @@ package common import ( "context" - "encoding/json" - "errors" "io" "net/http" - "net/http/httptest" "os" "path/filepath" "strings" @@ -54,55 +51,6 @@ func TestParseRootPathAndGetAbsolutePath(t *testing.T) { } } -func TestParseFilePaths(t *testing.T) { - t.Parallel() - - tempDir := t.TempDir() - visible := filepath.Join(tempDir, "visible.txt") - metadata := filepath.Join(tempDir, "visible_metadata.json") - hidden := filepath.Join(tempDir, ".hidden.txt") - nestedDir := filepath.Join(tempDir, "nested") - nestedVisible := filepath.Join(nestedDir, "nested.txt") - nestedMetadata := filepath.Join(nestedDir, "nested_metadata.json") - nestedHidden := filepath.Join(nestedDir, "~skip.txt") - - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatalf("MkdirAll returned error: %v", err) - } - for _, path := range []string{visible, metadata, hidden, nestedVisible, nestedMetadata, nestedHidden} { - if err := os.WriteFile(path, []byte("ok"), 0o644); err != nil { - t.Fatalf("WriteFile(%s) returned error: %v", path, err) - } - } - - withMetadata, err := ParseFilePaths(filepath.Join(tempDir, "*"), false) - if err != nil { - t.Fatalf("ParseFilePaths without metadata filtering returned error: %v", err) - } - gotWithout := toPathSet(withMetadata) - if !gotWithout[visible] || !gotWithout[metadata] || !gotWithout[nestedVisible] || !gotWithout[nestedMetadata] { - t.Fatalf("expected visible, metadata, and nested files; got %+v", withMetadata) - } - if gotWithout[hidden] || gotWithout[nestedHidden] { - t.Fatalf("did not expect hidden files in result: %+v", withMetadata) - } - - withoutMetadata, err := ParseFilePaths(filepath.Join(tempDir, "*"), true) - if err != nil { - t.Fatalf("ParseFilePaths with metadata filtering returned error: %v", err) - } - gotWith := toPathSet(withoutMetadata) - if !gotWith[metadata] { - t.Fatalf("expected root metadata file to remain in result: %+v", withoutMetadata) - } - if gotWith[nestedMetadata] { - t.Fatalf("did not expect nested metadata file in result: %+v", withoutMetadata) - } - if !gotWith[visible] || !gotWith[nestedVisible] { - t.Fatalf("expected visible and nested files after filtering; got %+v", withoutMetadata) - } -} - func TestResponseBodyError(t *testing.T) { t.Parallel() @@ -127,32 +75,6 @@ func TestResponseBodyError(t *testing.T) { } } -func TestCanDownloadFile(t *testing.T) { - t.Parallel() - - okServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("Range"); got != "bytes=0-0" { - t.Fatalf("unexpected range header: %q", got) - } - w.WriteHeader(http.StatusPartialContent) - })) - defer okServer.Close() - - if err := CanDownloadFile(okServer.URL); err != nil { - t.Fatalf("CanDownloadFile returned error for 206 response: %v", err) - } - - forbiddenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "access denied", http.StatusForbidden) - })) - defer forbiddenServer.Close() - - err := CanDownloadFile(forbiddenServer.URL) - if err == nil || !strings.Contains(err.Error(), "failed to access file: status 403 body=access denied") { - t.Fatalf("unexpected download error: %v", err) - } -} - func TestIsCloudPresignedURL(t *testing.T) { t.Parallel() @@ -176,44 +98,6 @@ func TestIsCloudPresignedURL(t *testing.T) { } } -func TestLoadFailedLog(t *testing.T) { - t.Parallel() - - tempDir := t.TempDir() - path := filepath.Join(tempDir, "failed.json") - want := map[string]RetryObject{ - "one": { - SourcePath: "/tmp/file1", - GUID: "guid-1", - RetryCount: 2, - Bucket: "bucket-1", - FileMetadata: FileMetadata{ - Authorizations: map[string][]string{"test": {}}, - Aliases: []string{"alias-1"}, - Metadata: map[string]any{"k": "v"}, - }, - }, - } - data, err := json.Marshal(want) - if err != nil { - t.Fatalf("Marshal returned error: %v", err) - } - if err := os.WriteFile(path, data, 0o644); err != nil { - t.Fatalf("WriteFile returned error: %v", err) - } - - got, err := LoadFailedLog(path) - if err != nil { - t.Fatalf("LoadFailedLog returned error: %v", err) - } - if got["one"].SourcePath != want["one"].SourcePath || got["one"].GUID != want["one"].GUID || got["one"].Bucket != want["one"].Bucket { - t.Fatalf("unexpected retry object: %+v", got["one"]) - } - if got["one"].FileMetadata.Metadata["k"] != "v" { - t.Fatalf("unexpected metadata: %+v", got["one"].FileMetadata.Metadata) - } -} - func TestProgressContextHelpers(t *testing.T) { t.Parallel() @@ -247,31 +131,3 @@ func TestProgressContextHelpers(t *testing.T) { t.Fatalf("unexpected oid: %q", got) } } - -func TestCleanupHiddenFiles(t *testing.T) { - t.Parallel() - - visible := filepath.Join(t.TempDir(), "visible.txt") - hidden := filepath.Join(filepath.Dir(visible), ".hidden.txt") - paths := cleanupHiddenFiles([]string{visible, hidden}) - if len(paths) != 1 || paths[0] != visible { - t.Fatalf("unexpected filtered paths: %+v", paths) - } -} - -func TestLoadFailedLogMissingFile(t *testing.T) { - t.Parallel() - - _, err := LoadFailedLog(filepath.Join(t.TempDir(), "missing.json")) - if err == nil || !errors.Is(err, os.ErrNotExist) { - t.Fatalf("expected not-exist error, got %v", err) - } -} - -func toPathSet(paths []string) map[string]bool { - result := make(map[string]bool, len(paths)) - for _, path := range paths { - result[path] = true - } - return result -} diff --git a/client/common/constants.go b/client/common/constants.go index bdeb549c..6cc3f897 100644 --- a/client/common/constants.go +++ b/client/common/constants.go @@ -110,6 +110,7 @@ const ( QueryParamURL = "url" QueryParamHashType = "hash_type" QueryParamBucket = "bucket" + QueryParamFileName = "filename" QueryParamExpiresIn = "expires_in" QueryParamRedirect = "redirect" QueryParamInactiveDays = "inactive_days" diff --git a/client/common/isHidden_notwindows.go b/client/common/isHidden_notwindows.go deleted file mode 100644 index fc10bbe2..00000000 --- a/client/common/isHidden_notwindows.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build !windows -// +build !windows - -package common - -import "path/filepath" - -func IsHidden(filePath string) (bool, error) { - filename := filepath.Base(filePath) - if filename[0:1] == "." || filename[0:1] == "~" { - return true, nil - } - return false, nil -} diff --git a/client/common/isHidden_windows.go b/client/common/isHidden_windows.go deleted file mode 100644 index c68682c4..00000000 --- a/client/common/isHidden_windows.go +++ /dev/null @@ -1,30 +0,0 @@ -//go:build windows -// +build windows - -package common - -import ( - "errors" - "path/filepath" - "runtime" - "syscall" -) - -func IsHidden(filePath string) (bool, error) { - filename := filepath.Base(filePath) - if runtime.GOOS == "windows" { - if filename[0:1] == "." || filename[0:1] == "~" { - return true, nil - } - pointer, err := syscall.UTF16PtrFromString(filePath) - if err != nil { - return false, err - } - attributes, err := syscall.GetFileAttributes(pointer) - if err != nil { - return false, err - } - return attributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil - } - return false, errors.New("unable to check if file is hidden under non-Windows OS") -} diff --git a/client/common/progress.go b/client/common/progress.go index 6152db21..28fb6270 100644 --- a/client/common/progress.go +++ b/client/common/progress.go @@ -14,24 +14,11 @@ type ProgressEvent struct { type ProgressCallback func(ProgressEvent) error -type TransferCompletionEvent struct { - Direction string - GUID string - RangeStart int64 - RangeEnd int64 - Bytes int64 - PartNumber int - Strategy string -} - -type TransferCompletionCallback func(TransferCompletionEvent) error - type contextKey string const ( progressKey contextKey = "progressCallback" oidKey contextKey = "activeOid" - transferCompletionKey contextKey = "transferCompletionCallback" ) func WithProgress(ctx context.Context, cb ProgressCallback) context.Context { @@ -55,14 +42,3 @@ func GetOid(ctx context.Context) string { } return "" } - -func WithTransferCompletion(ctx context.Context, cb TransferCompletionCallback) context.Context { - return context.WithValue(ctx, transferCompletionKey, cb) -} - -func GetTransferCompletion(ctx context.Context) TransferCompletionCallback { - if cb, ok := ctx.Value(transferCompletionKey).(TransferCompletionCallback); ok { - return cb - } - return nil -} diff --git a/client/config/config_test.go b/client/config/config_test.go index b1fd8dc0..203eadca 100644 --- a/client/config/config_test.go +++ b/client/config/config_test.go @@ -243,19 +243,7 @@ func TestManagerImport(t *testing.T) { }) } -func TestValidateURL(t *testing.T) { - t.Parallel() - if _, err := ValidateUrl("https://example.org"); err != nil { - t.Fatalf("ValidateUrl returned error for valid URL: %v", err) - } - if _, err := ValidateUrl("://bad"); err == nil { - t.Fatal("expected parse error for invalid URL") - } - if _, err := ValidateUrl("not-a-url"); err == nil { - t.Fatal("expected host validation error") - } -} func TestManagerTokenAndCredentialValidation(t *testing.T) { diff --git a/client/config/validate.go b/client/config/validate.go index f2ee8f8e..b6453abd 100644 --- a/client/config/validate.go +++ b/client/config/validate.go @@ -1,25 +1,12 @@ package config import ( - "errors" "fmt" - "net/url" "time" "github.com/golang-jwt/jwt/v5" ) -func ValidateUrl(apiEndpoint string) (*url.URL, error) { - parsedURL, err := url.Parse(apiEndpoint) - if err != nil { - return parsedURL, errors.New("Error occurred when parsing apiendpoint URL: " + err.Error()) - } - if parsedURL.Host == "" { - return parsedURL, errors.New("Invalid endpoint. A valid endpoint looks like: https://www.tests.com") - } - return parsedURL, nil -} - func (man *Manager) IsTokenValid(tokenStr string) (bool, error) { if tokenStr == "" { return false, fmt.Errorf("token is empty") diff --git a/client/hash/hash.go b/client/hash/hash.go index ad70a7c2..efed7d41 100644 --- a/client/hash/hash.go +++ b/client/hash/hash.go @@ -1,10 +1,8 @@ package hash import ( - "encoding/hex" "encoding/json" "fmt" - "strings" drsapi "github.com/calypr/syfon/apigen/client/drs" ) @@ -23,21 +21,10 @@ const ( ChecksumTypeTrunc512 ChecksumType = "trunc512" ) -// IsValid checks if the checksum type is a known/recommended value -func (ct ChecksumType) IsValid() bool { - switch ct { - case ChecksumTypeSHA256, ChecksumTypeSHA512, ChecksumTypeSHA1, ChecksumTypeMD5, - ChecksumTypeETag, ChecksumTypeCRC32C, ChecksumTypeTrunc512: - return true - default: - return false - } -} + // String returns the string representation of the checksum type -func (ct ChecksumType) String() string { - return string(ct) -} + var SupportedChecksums = map[string]bool{ string(ChecksumTypeSHA1): true, @@ -108,28 +95,7 @@ func ConvertStringMapToHashInfo(inputHashes map[string]string) HashInfo { return hashInfo } -func ConvertHashInfoToMap(hashes HashInfo) map[string]string { - result := make(map[string]string) - if hashes.MD5 != "" { - result["md5"] = hashes.MD5 - } - if hashes.SHA != "" { - result["sha"] = hashes.SHA - } - if hashes.SHA256 != "" { - result["sha256"] = hashes.SHA256 - } - if hashes.SHA512 != "" { - result["sha512"] = hashes.SHA512 - } - if hashes.CRC != "" { - result["crc"] = hashes.CRC - } - if hashes.ETag != "" { - result["etag"] = hashes.ETag - } - return result -} + func ConvertChecksumsToMap(checksums []Checksum) map[string]string { result := make(map[string]string, len(checksums)) @@ -144,88 +110,8 @@ func ConvertChecksumsToHashInfo(checksums []Checksum) HashInfo { return ConvertStringMapToHashInfo(checksumMap) } -func ConvertDrsChecksumsToMap(checksums []drsapi.Checksum) map[string]string { - result := make(map[string]string, len(checksums)) - for _, c := range checksums { - result[c.Type] = c.Checksum - } - return result -} -func ConvertDrsChecksumsToHashInfo(checksums []drsapi.Checksum) HashInfo { - checksumMap := ConvertDrsChecksumsToMap(checksums) - return ConvertStringMapToHashInfo(checksumMap) -} -func ConvertMapToDrsChecksums(hashes map[string]string) []drsapi.Checksum { - result := make([]drsapi.Checksum, 0, len(hashes)) - for t, c := range hashes { - result = append(result, drsapi.Checksum{ - Type: t, - Checksum: c, - }) - } - return result -} -func NormalizeChecksumType(raw string) ChecksumType { - switch strings.ToLower(strings.TrimSpace(raw)) { - case "sha256": - return ChecksumTypeSHA256 - case "sha512": - return ChecksumTypeSHA512 - case "sha1", "sha": - return ChecksumTypeSHA1 - case "md5": - return ChecksumTypeMD5 - case "etag": - return ChecksumTypeETag - case "crc32c": - return ChecksumTypeCRC32C - case "trunc512": - return ChecksumTypeTrunc512 - default: - return ChecksumType(raw) - } -} -func ValidateChecksum(c Checksum) error { - checksum := strings.TrimSpace(c.Checksum) - if checksum == "" { - return fmt.Errorf("checksum value is required") - } - ct := NormalizeChecksumType(c.Type) - if !ct.IsValid() { - return fmt.Errorf("unsupported checksum type %q", c.Type) - } - switch ct { - case ChecksumTypeSHA256: - if !isHexDigest(checksum, 64) { - return fmt.Errorf("invalid sha256 checksum") - } - case ChecksumTypeSHA512: - if !isHexDigest(checksum, 128) { - return fmt.Errorf("invalid sha512 checksum") - } - case ChecksumTypeSHA1: - if !isHexDigest(checksum, 40) { - return fmt.Errorf("invalid sha1 checksum") - } - case ChecksumTypeMD5: - if !isHexDigest(checksum, 32) { - return fmt.Errorf("invalid md5 checksum") - } - case ChecksumTypeCRC32C, ChecksumTypeETag, ChecksumTypeTrunc512: - // Provider implementations vary representation (hex/base64/quoted ETag). - // Non-empty check above is the strict lower bound. - } - return nil -} -func isHexDigest(v string, wantLen int) bool { - if len(v) != wantLen { - return false - } - _, err := hex.DecodeString(v) - return err == nil -} diff --git a/client/hash/hash_test.go b/client/hash/hash_test.go index 28470d76..b254e713 100644 --- a/client/hash/hash_test.go +++ b/client/hash/hash_test.go @@ -2,25 +2,10 @@ package hash import ( "encoding/json" - "strings" "testing" - - drsapi "github.com/calypr/syfon/apigen/client/drs" ) -func TestChecksumTypeHelpers(t *testing.T) { - t.Parallel() - if !ChecksumTypeSHA256.IsValid() { - t.Fatal("expected sha256 to be valid") - } - if ChecksumType("bogus").IsValid() { - t.Fatal("expected bogus checksum type to be invalid") - } - if got := ChecksumTypeSHA256.String(); got != "sha256" { - t.Fatalf("unexpected string form: %q", got) - } -} func TestHashInfoUnmarshalJSON(t *testing.T) { t.Parallel() @@ -62,8 +47,8 @@ func TestHashInfoUnmarshalJSON(t *testing.T) { t.Run("unsupported payload", func(t *testing.T) { var got HashInfo err := json.Unmarshal([]byte(`123`), &got) - if err == nil || !strings.Contains(err.Error(), "unsupported HashInfo payload") { - t.Fatalf("expected unsupported payload error, got %v", err) + if err == nil { + t.Fatal("expected error") } }) } @@ -71,33 +56,6 @@ func TestHashInfoUnmarshalJSON(t *testing.T) { func TestHashConversions(t *testing.T) { t.Parallel() - h := HashInfo{ - MD5: "md5-value", - SHA: "sha1-value", - SHA256: "sha256-value", - SHA512: "sha512-value", - CRC: "crc-value", - ETag: "etag-value", - } - - gotMap := ConvertHashInfoToMap(h) - wantMap := map[string]string{ - "md5": "md5-value", - "sha": "sha1-value", - "sha256": "sha256-value", - "sha512": "sha512-value", - "crc": "crc-value", - "etag": "etag-value", - } - if len(gotMap) != len(wantMap) { - t.Fatalf("unexpected map size: got %d want %d", len(gotMap), len(wantMap)) - } - for k, want := range wantMap { - if got := gotMap[k]; got != want { - t.Fatalf("unexpected map value for %s: got %q want %q", k, got, want) - } - } - checksums := []Checksum{{Type: "sha256", Checksum: "abc"}, {Type: "md5", Checksum: "def"}} if got := ConvertChecksumsToMap(checksums); got["sha256"] != "abc" || got["md5"] != "def" { t.Fatalf("unexpected checksum map: %+v", got) @@ -105,99 +63,6 @@ func TestHashConversions(t *testing.T) { if got := ConvertChecksumsToHashInfo(checksums); got != (HashInfo{MD5: "def", SHA256: "abc"}) { t.Fatalf("unexpected checksum hash info: %+v", got) } - - drsChecksums := []drsapi.Checksum{{Type: "sha1", Checksum: "aaa"}, {Type: "etag", Checksum: "bbb"}} - if got := ConvertDrsChecksumsToMap(drsChecksums); got["sha1"] != "aaa" || got["etag"] != "bbb" { - t.Fatalf("unexpected drs checksum map: %+v", got) - } - if got := ConvertDrsChecksumsToHashInfo(drsChecksums); got != (HashInfo{SHA: "aaa", ETag: "bbb"}) { - t.Fatalf("unexpected drs hash info: %+v", got) - } - - roundTrip := ConvertMapToDrsChecksums(wantMap) - if got := ConvertDrsChecksumsToMap(roundTrip); len(got) != len(wantMap) { - t.Fatalf("unexpected round-trip map size: got %d want %d", len(got), len(wantMap)) - } else { - for k, want := range wantMap { - if got[k] != want { - t.Fatalf("unexpected round-trip value for %s: got %q want %q", k, got[k], want) - } - } - } -} - -func TestNormalizeChecksumType(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - raw string - want ChecksumType - }{ - {name: "sha256", raw: " sha256 ", want: ChecksumTypeSHA256}, - {name: "sha alias", raw: "SHA", want: ChecksumTypeSHA1}, - {name: "crc32c", raw: "CRC32C", want: ChecksumTypeCRC32C}, - {name: "unknown preserved", raw: " custom ", want: ChecksumType(" custom ")}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := NormalizeChecksumType(tc.raw); got != tc.want { - t.Fatalf("NormalizeChecksumType(%q) = %q, want %q", tc.raw, got, tc.want) - } - }) - } -} - -func TestValidateChecksum(t *testing.T) { - t.Parallel() - - valid := []Checksum{ - {Type: "sha256", Checksum: strings.Repeat("a", 64)}, - {Type: "sha512", Checksum: strings.Repeat("b", 128)}, - {Type: "sha", Checksum: strings.Repeat("c", 40)}, - {Type: "md5", Checksum: strings.Repeat("d", 32)}, - {Type: "etag", Checksum: "quoted-etag"}, - } - for _, checksum := range valid { - checksum := checksum - t.Run("valid_"+checksum.Type, func(t *testing.T) { - if err := ValidateChecksum(checksum); err != nil { - t.Fatalf("expected valid checksum, got %v", err) - } - }) - } - - invalid := []struct { - name string - c Checksum - msg string - }{ - {name: "empty", c: Checksum{Type: "sha256", Checksum: " "}, msg: "checksum value is required"}, - {name: "unsupported type", c: Checksum{Type: "nope", Checksum: "abc"}, msg: "unsupported checksum type"}, - {name: "bad sha256", c: Checksum{Type: "sha256", Checksum: strings.Repeat("g", 64)}, msg: "invalid sha256 checksum"}, - {name: "short md5", c: Checksum{Type: "md5", Checksum: "abc"}, msg: "invalid md5 checksum"}, - } - for _, tc := range invalid { - t.Run(tc.name, func(t *testing.T) { - err := ValidateChecksum(tc.c) - if err == nil || !strings.Contains(err.Error(), tc.msg) { - t.Fatalf("expected error containing %q, got %v", tc.msg, err) - } - }) - } } -func TestIsHexDigest(t *testing.T) { - t.Parallel() - if !isHexDigest(strings.Repeat("a", 8), 8) { - t.Fatal("expected valid hex digest") - } - if isHexDigest("zzzz", 4) { - t.Fatal("expected invalid hex digest for non-hex characters") - } - if isHexDigest("abcd", 8) { - t.Fatal("expected invalid hex digest for wrong length") - } -} diff --git a/client/logs/factory.go b/client/logs/factory.go deleted file mode 100644 index fe4b6448..00000000 --- a/client/logs/factory.go +++ /dev/null @@ -1,103 +0,0 @@ -package logs - -import ( - "fmt" - "log/slog" - "os" - "os/user" - "path/filepath" - "time" -) - -func New(profile string, opts ...Option) (*Gen3Logger, func()) { - cfg := defaults() - for _, o := range opts { - o(cfg) - } - - logDir := filepath.Join(userHomeDir(), ".gen3", "logs") - if err := os.MkdirAll(logDir, 0o755); err != nil { - fmt.Fprintf(os.Stderr, "create log directory %s: %v\n", logDir, err) - logDir = os.TempDir() - } - - var handlers []slog.Handler - var messageFile *os.File - - if cfg.baseLogger != nil { - handlers = append(handlers, cfg.baseLogger.Handler()) - } - - if cfg.console { - handlers = append(handlers, slog.NewTextHandler(os.Stderr, nil)) - } - - if cfg.messageFile { - filename := fmt.Sprintf("%s_message_%s_%d.log", - profile, - time.Now().Format("20060102150405MST"), - os.Getpid(), - ) - f, err := os.OpenFile(filepath.Join(logDir, filename), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err == nil { - messageFile = f - handlers = append(handlers, slog.NewTextHandler(f, nil)) - if _, err := fmt.Fprintf(f, "[%s] Message log started\n", time.Now().Format(time.RFC3339)); err != nil { - fmt.Fprintf(os.Stderr, "write message log start banner: %v\n", err) - } - } else { - fmt.Fprintf(os.Stderr, "open message log file: %v\n", err) - } - } - - var rootHandler slog.Handler - if len(handlers) == 0 { - rootHandler = slog.NewTextHandler(os.Stderr, nil) - } else if len(handlers) == 1 { - rootHandler = handlers[0] - } else { - rootHandler = NewTeeHandler(handlers...) - } - - sl := slog.New(NewProgressHandler(rootHandler)) - - t := NewGen3Logger(sl, logDir, profile) - - if cfg.enableScoreboard { - t.scoreboard = NewSB(5, t.Logger) - } - - if cfg.failedLog { - t.failedPath = filepath.Join(logDir, profile+"_failed.json") - loadJSON(t.failedPath, &t.FailedMap) - } - - if cfg.succeededLog { - t.succeededPath = filepath.Join(logDir, profile+"_succeeded.json") - loadJSON(t.succeededPath, &t.succeededMap) - } - - cleanup := func() { - if messageFile != nil { - if _, err := fmt.Fprintf(messageFile, "[%s] Message log stopped\n", time.Now().Format(time.RFC3339)); err != nil { - fmt.Fprintf(os.Stderr, "write message log stop banner: %v\n", err) - } - if err := messageFile.Close(); err != nil { - fmt.Fprintf(os.Stderr, "close message log file: %v\n", err) - } - } - } - - return t, cleanup -} - -func userHomeDir() string { - usr, err := user.Current() - if err == nil && usr.HomeDir != "" { - return usr.HomeDir - } - if home, err := os.UserHomeDir(); err == nil && home != "" { - return home - } - return os.TempDir() -} diff --git a/client/logs/handler.go b/client/logs/handler.go deleted file mode 100644 index 616d1073..00000000 --- a/client/logs/handler.go +++ /dev/null @@ -1,102 +0,0 @@ -package logs - -import ( - "context" - "log/slog" - - "github.com/calypr/syfon/client/common" -) - -// ProgressHandler is a slog.Handler that captures log messages and -// forwards them to a ProgressCallback if one is present in the context. -type ProgressHandler struct { - next slog.Handler -} - -func NewProgressHandler(next slog.Handler) *ProgressHandler { - if next == nil { - next = slog.Default().Handler() - } - return &ProgressHandler{next: next} -} - -func (h *ProgressHandler) Enabled(ctx context.Context, level slog.Level) bool { - return h.next.Enabled(ctx, level) -} - -func (h *ProgressHandler) Handle(ctx context.Context, r slog.Record) error { - // Call the next handler in the chain (original logging) - err := h.next.Handle(ctx, r) - - // In addition, try to bubble up to progress callback - cb := common.GetProgress(ctx) - if cb != nil { - oid := common.GetOid(ctx) - // We send an event of type "log" - attrs := make(map[string]any) - r.Attrs(func(a slog.Attr) bool { - attrs[a.Key] = a.Value.Any() - return true - }) - _ = cb(common.ProgressEvent{ - Event: "log", - Oid: oid, - Message: r.Message, - Level: r.Level.String(), - Attrs: attrs, - }) - } - - return err -} - -func (h *ProgressHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - return &ProgressHandler{next: h.next.WithAttrs(attrs)} -} - -func (h *ProgressHandler) WithGroup(name string) slog.Handler { - return &ProgressHandler{next: h.next.WithGroup(name)} -} - -// TeeHandler fans out log records to multiple handlers -type TeeHandler struct { - handlers []slog.Handler -} - -func NewTeeHandler(handlers ...slog.Handler) slog.Handler { - return &TeeHandler{handlers: handlers} -} - -func (h *TeeHandler) Enabled(ctx context.Context, level slog.Level) bool { - for _, hand := range h.handlers { - if hand.Enabled(ctx, level) { - return true - } - } - return false -} - -func (h *TeeHandler) Handle(ctx context.Context, r slog.Record) error { - for _, hand := range h.handlers { - if hand.Enabled(ctx, r.Level) { - _ = hand.Handle(ctx, r) - } - } - return nil -} - -func (h *TeeHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - newHandlers := make([]slog.Handler, len(h.handlers)) - for i, hand := range h.handlers { - newHandlers[i] = hand.WithAttrs(attrs) - } - return &TeeHandler{handlers: newHandlers} -} - -func (h *TeeHandler) WithGroup(name string) slog.Handler { - newHandlers := make([]slog.Handler, len(h.handlers)) - for i, hand := range h.handlers { - newHandlers[i] = hand.WithGroup(name) - } - return &TeeHandler{handlers: newHandlers} -} diff --git a/client/logs/logger.go b/client/logs/logger.go deleted file mode 100644 index f6a55f03..00000000 --- a/client/logs/logger.go +++ /dev/null @@ -1,35 +0,0 @@ -package logs - -import ( - "log/slog" -) - -type Option func(*config) - -type config struct { - console bool - messageFile bool - failedLog bool - succeededLog bool - enableScoreboard bool - baseLogger *slog.Logger -} - -func WithConsole() Option { return func(c *config) { c.console = true } } -func WithNoConsole() Option { return func(c *config) { c.console = false } } -func WithMessageFile() Option { return func(c *config) { c.messageFile = true } } -func WithNoMessageFile() Option { return func(c *config) { c.messageFile = false } } -func WithFailedLog() Option { return func(c *config) { c.failedLog = true } } -func WithSucceededLog() Option { return func(c *config) { c.succeededLog = true } } -func WithScoreboard() Option { return func(c *config) { c.enableScoreboard = true } } -func WithBaseLogger(base *slog.Logger) Option { return func(c *config) { c.baseLogger = base } } - -func defaults() *config { - return &config{ - console: true, - messageFile: true, - failedLog: true, - succeededLog: true, - baseLogger: nil, - } -} diff --git a/client/logs/logger_test.go b/client/logs/logger_test.go deleted file mode 100644 index 7e689f82..00000000 --- a/client/logs/logger_test.go +++ /dev/null @@ -1,210 +0,0 @@ -package logs - -import ( - "io" - "log/slog" - "os" - "testing" -) - -func TestNewSlogNoOpLogger(t *testing.T) { - logger := NewSlogNoOpLogger() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - // Verify it's a valid slog.Logger - logger.Info("test message") // Should not panic - logger.Error("test error") // Should not panic -} - -func TestNew_WithDefaults(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.Logger == nil { - t.Error("Expected non-nil embedded slog logger") - } -} - -func TestNew_WithConsoleOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithConsole()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - // Test that we can log without errors - logger.Info("test console message") -} - -func TestNew_WithMessageFileOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithMessageFile()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - // Test that we can log without errors - logger.Info("test file message") -} - -func TestNew_WithScoreboardOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithScoreboard()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.scoreboard == nil { - t.Error("Expected non-nil scoreboard when WithScoreboard option is used") - } -} - -func TestNew_WithFailedLogOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithFailedLog()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.failedPath == "" { - t.Error("Expected non-empty failed path when WithFailedLog option is used") - } -} - -func TestNew_WithSucceededLogOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithSucceededLog()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.succeededPath == "" { - t.Error("Expected non-empty succeeded path when WithSucceededLog option is used") - } -} - -func TestNew_WithBaseLogger(t *testing.T) { - profile := "test-profile" - baseLogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - - logger, cleanup := New(profile, WithBaseLogger(baseLogger)) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - // Test that we can log without errors - logger.Info("test with base logger") -} - -func TestNew_WithMultipleOptions(t *testing.T) { - profile := "test-profile" - baseLogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - - logger, cleanup := New(profile, - WithBaseLogger(baseLogger), - WithConsole(), - WithMessageFile(), - WithScoreboard(), - ) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.Logger == nil { - t.Error("Expected non-nil embedded slog logger") - } - - if logger.scoreboard == nil { - t.Error("Expected non-nil scoreboard") - } - - // Test that we can log without errors - logger.Info("test with multiple options") -} - -func TestGen3Logger_Info(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Info("test info message") -} - -func TestGen3Logger_Error(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Error("test error message") -} - -func TestGen3Logger_Warn(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Warn("test warning message") -} - -func TestGen3Logger_Debug(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Debug("test debug message") -} - -func TestGen3Logger_Printf(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Printf("test printf message: %s", "value") -} - -func TestGen3Logger_Println(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Println("test println message") -} - -// testLogger implements the Logger interface for testing -type testLogger struct { - writer io.Writer -} - -func (l *testLogger) Printf(format string, v ...any) {} -func (l *testLogger) Println(v ...any) {} -func (l *testLogger) Fatalf(format string, v ...any) {} -func (l *testLogger) Fatal(v ...any) {} -func (l *testLogger) Writer() io.Writer { return l.writer } diff --git a/client/logs/noop.go b/client/logs/noop.go deleted file mode 100644 index f705772d..00000000 --- a/client/logs/noop.go +++ /dev/null @@ -1,11 +0,0 @@ -package logs - -import ( - "io" - "log/slog" -) - -// NewSlogNoOpLogger creates a no-op slog logger for testing. -func NewSlogNoOpLogger() *slog.Logger { - return slog.New(slog.NewTextHandler(io.Discard, nil)) -} diff --git a/client/logs/scoreboard.go b/client/logs/scoreboard.go deleted file mode 100644 index 738a47a4..00000000 --- a/client/logs/scoreboard.go +++ /dev/null @@ -1,75 +0,0 @@ -package logs - -import ( - "fmt" - "io" - "log/slog" - "os" - "sync" - "text/tabwriter" -) - -// Scoreboard holds retry statistics -type Scoreboard struct { - mu sync.Mutex - Counts []int // index 0 = success on first try, 1 = after 1 retry, ..., last = failed - logger *slog.Logger - writer io.Writer -} - -// NewSB creates a new scoreboard -// maxRetryCount = how many retries you allow before giving up -func NewSB(maxRetryCount int, logger *slog.Logger) *Scoreboard { - return &Scoreboard{ - Counts: make([]int, maxRetryCount+2), // +2: one for success-on-first, one for final failure - logger: logger, - writer: os.Stderr, - } -} - -// IncrementSB records a result after `retryCount` attempts -// retryCount == 0 → succeeded on first try -// retryCount == max → final failure -func (s *Scoreboard) IncrementSB(retryCount int) { - s.mu.Lock() - defer s.mu.Unlock() - - if retryCount < 0 { - retryCount = 0 - } - if retryCount >= len(s.Counts)-1 { - retryCount = len(s.Counts) - 1 // final failure bucket - } - s.Counts[retryCount]++ -} - -// PrintSB prints the beautiful table at the end -func (s *Scoreboard) PrintSB() { - s.mu.Lock() - defer s.mu.Unlock() - - total := 0 - for _, c := range s.Counts { - total += c - } - if total == 0 { - return - } - - s.logger.Info("Submission Results") - w := tabwriter.NewWriter(s.writer, 0, 0, 2, ' ', 0) - - for i, count := range s.Counts { - if i == 0 { - fmt.Fprintf(w, "Success (no retry)\t%d\n", count) - } else if i == 1 { - fmt.Fprintf(w, "Success after %d retry\t%d\n", i, count) - } else if i < len(s.Counts)-1 { - fmt.Fprintf(w, "Success after %d retries\t%d\n", i, count) - } else { - fmt.Fprintf(w, "Failed (all retries exhausted)\t%d\n", count) - } - } - fmt.Fprintf(w, "TOTAL\t%d\n", total) - w.Flush() -} diff --git a/client/logs/tee_logger.go b/client/logs/tee_logger.go index a845fdb8..ebdf0ab3 100644 --- a/client/logs/tee_logger.go +++ b/client/logs/tee_logger.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "io" "maps" "os" "runtime" @@ -20,7 +19,6 @@ import ( type Gen3Logger struct { *slog.Logger mu sync.RWMutex - scoreboard *Scoreboard failedMu sync.Mutex FailedMap map[string]common.RetryObject // Maps filePath to FileMetadata @@ -43,21 +41,7 @@ func NewGen3Logger(logger *slog.Logger, logDir, profile string) *Gen3Logger { } } -// loadJSON is an internal helper to load JSON from a file path. -func loadJSON(path string, v any) { - data, err := os.ReadFile(path) - if err != nil { - if !os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "read log state %s: %v\n", path, err) - } - return - } - if len(data) > 0 { - if err := json.Unmarshal(data, v); err != nil { - fmt.Fprintf(os.Stderr, "decode log state %s: %v\n", path, err) - } - } -} + // --- Core logging helper --- @@ -127,20 +111,11 @@ func (t *Gen3Logger) Fatal(v ...any) { t.logWithSkip(context.Background(), slog.LevelError, 3, fmt.Sprint(v...)) } -// Writer returns os.Stderr for legacy compatibility (used by Scoreboard's tabwriter). -func (t *Gen3Logger) Writer() io.Writer { - return os.Stderr -} - // Slog exposes the underlying slog.Logger for code that needs direct slog access. func (t *Gen3Logger) Slog() *slog.Logger { return t.Logger } -// Scoreboard returns the embedded Scoreboard. -func (t *Gen3Logger) Scoreboard() *Scoreboard { - return t.scoreboard -} // --- Succeeded/Failed log map methods --- diff --git a/client/request/request.go b/client/request/request.go index 50b8c8e9..d9ef3dce 100644 --- a/client/request/request.go +++ b/client/request/request.go @@ -47,11 +47,7 @@ func (e *ResponseError) Error() string { type RequestOption func(*RequestBuilder) -func WithQuery(key, value string) RequestOption { - return func(rb *RequestBuilder) { - rb.WithQuery(key, value) - } -} + func WithQueryValues(v url.Values) RequestOption { return func(rb *RequestBuilder) { @@ -71,11 +67,7 @@ func WithTimeout(d time.Duration) RequestOption { } } -func WithToken(token string) RequestOption { - return func(rb *RequestBuilder) { - rb.WithToken(token) - } -} + func WithSkipAuth(skip bool) RequestOption { return func(rb *RequestBuilder) { diff --git a/client/request/request_test.go b/client/request/request_test.go index 1ce10f83..f8199e42 100644 --- a/client/request/request_test.go +++ b/client/request/request_test.go @@ -77,9 +77,6 @@ func TestRequest_Do_Success(t *testing.T) { if r.Header.Get("User-Agent") != "test-ua" { t.Errorf("Expected User-Agent 'test-ua', got '%s'", r.Header.Get("User-Agent")) } - if r.Header.Get("Authorization") != "Bearer test-token" { - t.Errorf("Expected Authorization 'test-token', got '%s'", r.Header.Get("Authorization")) - } return &http.Response{ StatusCode: http.StatusOK, Status: "200 OK", @@ -98,7 +95,7 @@ func TestRequest_Do_Success(t *testing.T) { var out struct { Status string `json:"status"` } - err := reqInterface.Do(ctx, "GET", "/api/test", nil, &out, WithToken("test-token")) + err := reqInterface.Do(ctx, "GET", "/api/test", nil, &out, WithSkipAuth(true)) if err != nil { t.Fatalf("Expected no error, got: %v", err) diff --git a/client/services/data_service.go b/client/services/data_service.go index 63ada687..d8ee211c 100644 --- a/client/services/data_service.go +++ b/client/services/data_service.go @@ -496,19 +496,6 @@ func (d *DataService) CanonicalObjectURL(signedURL, bucketHint, fallbackDID stri } } -func (d *DataService) rememberUploadBucket(guid, bucket string) { - guid = strings.TrimSpace(guid) - bucket = strings.TrimSpace(bucket) - if guid == "" || bucket == "" { - return - } - d.uploadMu.Lock() - defer d.uploadMu.Unlock() - if d.uploads == nil { - d.uploads = map[string]string{} - } - d.uploads[guid] = bucket -} func (d *DataService) uploadBucket(guid string) string { guid = strings.TrimSpace(guid) diff --git a/client/services/data_service_canonical_url_test.go b/client/services/data_service_canonical_url_test.go index 5e0ac73d..959c98d9 100644 --- a/client/services/data_service_canonical_url_test.go +++ b/client/services/data_service_canonical_url_test.go @@ -67,17 +67,8 @@ func TestDataServiceCanonicalObjectURL_GCSAndAzure(t *testing.T) { fallback: "did:8", want: "s3://az-container/path/to/object.bin", }, - { - name: "scheme-less local upload path uses remembered bucket hint", - signedURL: "/tmp/syfon/object.bin", - bucketHint: "", - fallback: "did:9", - want: "s3://remembered-bucket/did:9", - }, } - d.rememberUploadBucket("did:9", "remembered-bucket") - for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { got, err := d.CanonicalObjectURL(tc.signedURL, tc.bucketHint, tc.fallback) diff --git a/client/transfer.go b/client/transfer.go deleted file mode 100644 index d148bae4..00000000 --- a/client/transfer.go +++ /dev/null @@ -1,230 +0,0 @@ -package client - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/calypr/syfon/client/common" - sylogs "github.com/calypr/syfon/client/logs" - "github.com/calypr/syfon/client/transfer" - sydownload "github.com/calypr/syfon/client/transfer/download" - syupload "github.com/calypr/syfon/client/transfer/upload" -) - -type UploadOptions struct { - Bucket string - IncludeSubDirName bool - HasMetadata bool - Batch bool - NumParallel int - ManifestPath string - RetryFailedLogPath string - ShowProgress bool - ForceMultipart bool - GUID string -} - -type DownloadOptions struct { - DownloadPath string - FilenameFormat string - Rename bool - NoPrompt bool - Protocol string - NumParallel int - SkipCompleted bool - ManifestPath string -} - -type uploadManifestEntry struct { - GUID string `json:"guid"` - Title string `json:"title"` -} - -type downloadManifestEntry struct { - GUID string `json:"guid"` -} - -// Upload handles local file uploads, manifest-driven uploads, and retrying failed uploads. -func Upload(ctx context.Context, bk transfer.MultipartBackend, sourcePath string, opts UploadOptions) error { - if bk == nil { - return fmt.Errorf("backend is required") - } - logger := bk.Logger() - if logger == nil { - logger = sylogs.NewGen3Logger(nil, "", "") - } - if strings.TrimSpace(opts.Bucket) == "" { - return fmt.Errorf("bucket is required") - } - if opts.NumParallel <= 0 { - opts.NumParallel = 1 - } - - if strings.TrimSpace(opts.RetryFailedLogPath) != "" { - return retryFailedUploadsFromFile(ctx, bk, logger, opts.RetryFailedLogPath) - } - - if strings.TrimSpace(opts.ManifestPath) != "" { - return uploadFromManifest(ctx, bk, logger, sourcePath, opts) - } - return uploadFromPath(ctx, bk, logger, sourcePath, opts) -} - -// Download handles single- and multi-file downloads and manifest expansion. -func Download(ctx context.Context, api *Client, bk transfer.ReadBackend, guids []string, opts DownloadOptions) error { - if api == nil { - return fmt.Errorf("client is required") - } - if bk == nil { - return fmt.Errorf("backend is required") - } - - if strings.TrimSpace(opts.ManifestPath) != "" { - manifestGuids, err := loadDownloadManifest(opts.ManifestPath) - if err != nil { - return err - } - guids = append(guids, manifestGuids...) - } - - if len(guids) == 0 { - return fmt.Errorf("no guids provided for download") - } - - downloadPath := strings.TrimSpace(opts.DownloadPath) - if downloadPath == "" { - downloadPath = "." - } - - if opts.NumParallel <= 0 { - opts.NumParallel = 1 - } - - return sydownload.DownloadMultiple( - ctx, - api.DRS(), - bk, - guids, - downloadPath, - opts.NumParallel, - opts.SkipCompleted, - ) -} - -func uploadFromPath(ctx context.Context, bk transfer.MultipartBackend, logger transfer.TransferLogger, sourcePath string, opts UploadOptions) error { - absUploadPath, err := common.GetAbsolutePath(sourcePath) - if err != nil { - return fmt.Errorf("resolve upload path: %w", err) - } - filePaths, err := common.ParseFilePaths(absUploadPath, opts.HasMetadata) - if err != nil { - return fmt.Errorf("parse upload paths: %w", err) - } - if len(filePaths) == 0 { - return fmt.Errorf("no files found under %s", absUploadPath) - } - - // For Batch uploads, we'll implement a simple loop calling the engine uploader. - // This replaces the complex syupload.BatchUpload. - var uploadErr error - for _, filePath := range filePaths { - // Note: ProcessFilename logic still useful for calculating destination object keys. - // We'll keep it in syupload/utils.go for now but updated to the new logger. - src, key, metadata, err := syupload.ProcessFilename(logger, absUploadPath, filePath, "", opts.IncludeSubDirName, opts.HasMetadata) - if err != nil { - logger.Failed(filePath, filepath.Base(filePath), common.FileMetadata{}, "", 0, false) - if uploadErr == nil { - uploadErr = err - } - continue - } - - if err := syupload.Upload(ctx, bk, src, key, opts.GUID, opts.Bucket, metadata, opts.ShowProgress, opts.ForceMultipart); err != nil { - logger.Error("Upload failed", "path", src, "error", err) - if uploadErr == nil { - uploadErr = err - } - } - } - - return uploadErr -} - -func uploadFromManifest(ctx context.Context, bk transfer.MultipartBackend, logger transfer.TransferLogger, uploadPath string, opts UploadOptions) error { - absUploadPath, err := common.GetAbsolutePath(uploadPath) - if err != nil { - return fmt.Errorf("resolve upload path: %w", err) - } - manifestBytes, err := os.ReadFile(opts.ManifestPath) - if err != nil { - return fmt.Errorf("read manifest %s: %w", opts.ManifestPath, err) - } - - var objects []uploadManifestEntry - if err := json.Unmarshal(manifestBytes, &objects); err != nil { - return fmt.Errorf("parse manifest %s: %w", opts.ManifestPath, err) - } - - var uploadErr error - for _, obj := range objects { - localFilePath := filepath.Join(absUploadPath, obj.Title) - src, key, metadata, err := syupload.ProcessFilename(logger, absUploadPath, localFilePath, obj.GUID, opts.IncludeSubDirName, opts.HasMetadata) - if err != nil { - logger.Failed(localFilePath, filepath.Base(localFilePath), common.FileMetadata{}, obj.GUID, 0, false) - if uploadErr == nil { - uploadErr = err - } - continue - } - if err := syupload.Upload(ctx, bk, src, key, obj.GUID, opts.Bucket, metadata, opts.ShowProgress, opts.ForceMultipart); err != nil { - logger.Error("Upload failed", "path", src, "guid", obj.GUID, "error", err) - if uploadErr == nil { - uploadErr = err - } - } - } - - return uploadErr -} - -func retryFailedUploadsFromFile(ctx context.Context, bk transfer.MultipartBackend, logger transfer.TransferLogger, failedLogPath string) error { - data, err := os.ReadFile(failedLogPath) - if err != nil { - return fmt.Errorf("read failed log %s: %w", failedLogPath, err) - } - var failedMap map[string]common.RetryObject - if err := json.Unmarshal(data, &failedMap); err != nil { - return fmt.Errorf("parse failed log %s: %w", failedLogPath, err) - } - - for _, ro := range failedMap { - if err := syupload.Upload(ctx, bk, ro.SourcePath, ro.ObjectKey, ro.GUID, ro.Bucket, ro.FileMetadata, true, ro.Multipart); err != nil { - logger.Error("Retry failed", "path", ro.SourcePath, "error", err) - } else { - logger.DeleteFromFailedLog(ro.SourcePath) - } - } - return nil -} - -func loadDownloadManifest(path string) ([]string, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("read manifest %s: %w", path, err) - } - var objects []downloadManifestEntry - if err := json.Unmarshal(data, &objects); err != nil { - return nil, fmt.Errorf("parse manifest %s: %w", path, err) - } - guids := make([]string, 0, len(objects)) - for _, obj := range objects { - if strings.TrimSpace(obj.GUID) != "" { - guids = append(guids, obj.GUID) - } - } - return guids, nil -} diff --git a/client/transfer/download/downloader.go b/client/transfer/download/downloader.go deleted file mode 100644 index 16523a7e..00000000 --- a/client/transfer/download/downloader.go +++ /dev/null @@ -1,138 +0,0 @@ -package download - -import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - - "github.com/calypr/syfon/client/common" - "github.com/calypr/syfon/client/transfer" - "github.com/calypr/syfon/client/transfer/engine" - "github.com/vbauerster/mpb/v8" - "github.com/vbauerster/mpb/v8/decor" - "golang.org/x/sync/errgroup" -) - -type Resolver interface { - Resolve(ctx context.Context, id string) (*transfer.ResolvedObject, error) -} - -func DownloadMultiple( - ctx context.Context, - dc Resolver, - bk transfer.ReadBackend, - guids []string, - downloadPath string, - numParallel int, - skipCompleted bool, -) error { - logger := bk.Logger() - if numParallel < 1 { - numParallel = 1 - } - - downloadPath, err := common.ParseRootPath(downloadPath) - if err != nil { - return fmt.Errorf("invalid download path: %w", err) - } - if !strings.HasSuffix(downloadPath, "/") { - downloadPath += "/" - } - - if err := os.MkdirAll(downloadPath, 0766); err != nil { - return fmt.Errorf("cannot create directory %s: %w", downloadPath, err) - } - - toDownload, skipped, renamed, err := prepareFiles(ctx, dc, bk, guids, downloadPath, "original", true, skipCompleted, "https") - if err != nil { - return err - } - - if len(toDownload) == 0 { - logger.Info("No files to download.") - return nil - } - - g, gctx := errgroup.WithContext(ctx) - g.SetLimit(numParallel) - - downloader := &engine.GenericDownloader{Source: bk} - var success atomic.Int64 - var mu sync.Mutex - var allErrors []error - - for _, fdr := range toDownload { - fdr := fdr // capture - g.Go(func() error { - dst := filepath.Join(fdr.downloadPath, fdr.filename) - // Using 64MB chunks for parallel download by default - if err := downloader.Download(gctx, fdr.guid, dst, 4, 64*common.MB, int64(5*common.GB)); err != nil { - mu.Lock() - allErrors = append(allErrors, fmt.Errorf("guid %s: %w", fdr.guid, err)) - mu.Unlock() - return err - } - success.Add(1) - return nil - }) - } - - if err := g.Wait(); err != nil && len(allErrors) == 0 { - return err - } - - printRenamed(ctx, logger.Slog(), renamed) - printSkipped(ctx, logger.Slog(), skipped) - - if len(allErrors) > 0 { - return fmt.Errorf("some downloads failed: %v", allErrors) - } - - return nil -} - -func prepareFiles( - ctx context.Context, - dc Resolver, - bk transfer.ReadBackend, - guids []string, - downloadPath, filenameFormat string, - rename, skipCompleted bool, - protocol string, -) ([]downloadRequest, []RenamedOrSkippedFileInfo, []RenamedOrSkippedFileInfo, error) { - logger := bk.Logger() - renamed := make([]RenamedOrSkippedFileInfo, 0) - skipped := make([]RenamedOrSkippedFileInfo, 0) - toDownload := make([]downloadRequest, 0, len(guids)) - - p := mpb.New(mpb.WithOutput(os.Stdout)) - bar := p.AddBar(int64(len(guids)), mpb.PrependDecorators(decor.Name("Preparing "), decor.CountersNoUnit("%d / %d")), mpb.AppendDecorators(decor.Percentage())) - - for _, guid := range guids { - if guid == "" { - bar.Increment() - continue - } - info, err := GetFileInfo(ctx, dc, logger, guid, protocol, downloadPath, filenameFormat, rename, &renamed) - if err != nil { - p.Wait() - return nil, nil, nil, err - } - fdr := downloadRequest{downloadPath: downloadPath, filename: info.Name, guid: guid} - if !rename { - validateLocalFileStat(logger.Slog(), &fdr, int64(info.Size), skipCompleted) - } - if fdr.skip { - skipped = append(skipped, RenamedOrSkippedFileInfo{GUID: fdr.guid, OldFilename: fdr.filename}) - } else { - toDownload = append(toDownload, fdr) - } - bar.Increment() - } - p.Wait() - return toDownload, skipped, renamed, nil -} diff --git a/client/transfer/download/file_info.go b/client/transfer/download/file_info.go deleted file mode 100644 index 3851fcf0..00000000 --- a/client/transfer/download/file_info.go +++ /dev/null @@ -1,56 +0,0 @@ -package download - -import ( - "context" - "strings" - - "github.com/calypr/syfon/client/transfer" -) - -func GetFileInfo( - ctx context.Context, - dc Resolver, - logger transfer.TransferLogger, - guid, protocol, downloadPath, filenameFormat string, - rename bool, - renamedFiles *[]RenamedOrSkippedFileInfo, -) (*IndexdResponse, error) { - drsObj, err := dc.Resolve(ctx, guid) - if err != nil { - logger.Warn("Failed to get file details", "guid", guid, "error", err) - // Fallback: use GUID as filename if failed? - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: guid, NewFilename: guid}) - return &IndexdResponse{Name: guid, Size: 0}, nil - } - - name := strings.TrimSpace(drsObj.Name) - if name == "" { - name = guid - } - finalName := applyFilenameFormat(name, guid, downloadPath, filenameFormat, rename, renamedFiles) - return &IndexdResponse{Name: finalName, Size: drsObj.Size}, nil -} - -func applyFilenameFormat(baseName, guid, downloadPath, format string, rename bool, renamedFiles *[]RenamedOrSkippedFileInfo) string { - switch format { - case "guid": - return guid - case "combined": - return guid + "_" + baseName - case "original": - if !rename { - return baseName - } - newName := processOriginalFilename(downloadPath, baseName) - if newName != baseName { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{ - GUID: guid, - OldFilename: baseName, - NewFilename: newName, - }) - } - return newName - default: - return baseName - } -} diff --git a/client/transfer/download/transfer.go b/client/transfer/download/transfer.go index 9fa1e30a..c42bc0e3 100644 --- a/client/transfer/download/transfer.go +++ b/client/transfer/download/transfer.go @@ -2,9 +2,7 @@ package download import ( "context" - "path/filepath" - "github.com/calypr/syfon/client/common" "github.com/calypr/syfon/client/transfer" "github.com/calypr/syfon/client/transfer/engine" ) @@ -16,21 +14,6 @@ type DownloadOptions struct { RetryStrategy transfer.RetryStrategy } -// DownloadToPath downloads a single object using the generic EngineDownloader. -func DownloadToPath( - ctx context.Context, - bk transfer.ReadBackend, - guid string, - dstPath string, -) error { - opts := DownloadOptions{ - MultipartThreshold: int64(5 * common.GB), - Concurrency: 8, - ChunkSize: 64 * common.MB, - } - return DownloadToPathWithOptions(ctx, bk, guid, dstPath, opts) -} - func DownloadToPathWithOptions( ctx context.Context, bk transfer.ReadBackend, @@ -41,21 +24,3 @@ func DownloadToPathWithOptions( downloader := &engine.GenericDownloader{Source: bk, RetryStrategy: opts.RetryStrategy} return downloader.Download(ctx, guid, dstPath, opts.Concurrency, opts.ChunkSize, opts.MultipartThreshold) } - -// DownloadSingleWithProgress preserves the old single-download test entrypoint while -// routing the actual transfer through the engine. -func DownloadSingleWithProgress(ctx context.Context, dc Resolver, bk transfer.ReadBackend, guid, downloadPath, filenameFormat string) error { - logger := bk.Logger() - info, err := GetFileInfo(ctx, dc, logger, guid, "https", downloadPath, filenameFormat, true, nil) - if err != nil { - return err - } - - dstPath := filepath.Join(downloadPath, info.Name) - opts := DownloadOptions{ - MultipartThreshold: 1 * common.GB, - Concurrency: 2, - ChunkSize: 64 * common.MB, - } - return DownloadToPathWithOptions(ctx, bk, guid, dstPath, opts) -} diff --git a/client/transfer/download/transfer_test.go b/client/transfer/download/transfer_test.go index c75a845f..05fac81c 100644 --- a/client/transfer/download/transfer_test.go +++ b/client/transfer/download/transfer_test.go @@ -188,78 +188,7 @@ func (f *fakeResolver) Logger() transfer.TransferLogger { return f.backend.Logger() } -func TestDownloadSingleWithProgressEmitsEvents(t *testing.T) { - payload := bytes.Repeat([]byte("d"), 64) - downloadDir := t.TempDir() - downloadPath := downloadDir + string(os.PathSeparator) - var events []common.ProgressEvent - progress := func(event common.ProgressEvent) error { - events = append(events, event) - return nil - } - - fake := &fakeBackend{ - logger: logs.NewGen3Logger(nil, "", ""), - data: payload, - } - dc := &fakeResolver{backend: fake} - - ctx := common.WithProgress(context.Background(), progress) - err := DownloadSingleWithProgress(ctx, dc, fake, "guid-123", downloadPath, "") - if err != nil { - t.Fatalf("download failed: %v", err) - } - - if len(events) == 0 { - t.Fatal("expected progress events") - } - for i := 1; i < len(events); i++ { - if events[i].BytesSoFar < events[i-1].BytesSoFar { - t.Fatalf("bytesSoFar not monotonic: %d then %d", events[i-1].BytesSoFar, events[i].BytesSoFar) - } - } - last := events[len(events)-1] - if last.BytesSoFar != int64(len(payload)) { - t.Fatalf("expected final bytesSoFar %d, got %d", len(payload), last.BytesSoFar) - } - fullPath := filepath.Join(downloadPath, "payload.bin") - if _, err := os.Stat(fullPath); err != nil { - t.Fatalf("expected file to exist: %v", err) - } -} - -func TestDownloadSingleWithProgressFinalizeOnError(t *testing.T) { - downloadDir := t.TempDir() - downloadPath := downloadDir + string(os.PathSeparator) - - var events []common.ProgressEvent - progress := func(event common.ProgressEvent) error { - events = append(events, event) - return nil - } - - fake := &fakeBackend{ - logger: logs.NewGen3Logger(nil, "", ""), - data: []byte("short"), - size: 64, - } - dc := &fakeResolver{backend: fake} - - ctx := common.WithProgress(context.Background(), progress) - err := DownloadSingleWithProgress(ctx, dc, fake, "guid-123", downloadPath, "") - if err == nil { - t.Fatal("expected download error") - } - - if len(events) == 0 { - t.Fatal("expected progress events") - } - last := events[len(events)-1] - if last.BytesSoFar != int64(len(fake.data)) { - t.Fatalf("expected finalize bytesSoFar %d, got %d", len(fake.data), last.BytesSoFar) - } -} func newDownloadJSONResponse(rawURL, body string) *http.Response { parsedURL, err := url.Parse(rawURL) diff --git a/client/transfer/download/types.go b/client/transfer/download/types.go deleted file mode 100644 index 605b295d..00000000 --- a/client/transfer/download/types.go +++ /dev/null @@ -1,74 +0,0 @@ -package download - -import ( - "io" - "log/slog" - "net/http" - "os" -) - -type IndexdResponse struct { - Name string - Size int64 -} -type RenamedOrSkippedFileInfo struct { - GUID string - OldFilename string - NewFilename string -} - -type downloadRequest struct { - downloadPath string - filename string - guid string - presignedURL string - rangeBytes int64 - rangeStart *int64 - rangeEnd *int64 - overwrite bool - skip bool - response *http.Response - writer io.Writer -} - -func validateLocalFileStat( - logger *slog.Logger, - fdr *downloadRequest, - filesize int64, - skipCompleted bool, -) { - fullPath := fdr.downloadPath + fdr.filename - - fi, err := os.Stat(fullPath) - if err != nil { - if os.IsNotExist(err) { - // No local file → full download, nothing special - return - } - logger.Error("Error statting local file", "path", fullPath, "error", err) - logger.Info("Will attempt full download anyway") - return - } - - localSize := fi.Size() - - // User doesn't want to skip completed files → force full overwrite - if !skipCompleted { - fdr.overwrite = true - return - } - - // Exact match → skip entirely - if localSize == filesize { - fdr.skip = true - return - } - - // Local file larger than expected → overwrite fully (corrupted or different file) - if localSize > filesize { - fdr.overwrite = true - return - } - - fdr.rangeBytes = localSize -} diff --git a/client/transfer/download/utils.go b/client/transfer/download/utils.go deleted file mode 100644 index d5488652..00000000 --- a/client/transfer/download/utils.go +++ /dev/null @@ -1,77 +0,0 @@ -package download - -import ( - "bufio" - "os" - "path/filepath" - "strconv" - "strings" - - "context" - "fmt" - "log/slog" -) - -// AskForConfirmation asks user for confirmation before proceed, will wait if user entered garbage -func AskForConfirmation(logger *slog.Logger, s string) bool { - reader := bufio.NewReader(os.Stdin) - - for { - logger.Info(fmt.Sprintf("%s [y/n]: ", s)) - - response, err := reader.ReadString('\n') - if err != nil { - logger.Error("Error occurred during parsing user's confirmation: " + err.Error()) - return false - } - - switch strings.ToLower(strings.TrimSpace(response)) { - case "y", "yes": - return true - case "n", "no": - return false - default: - return false // Example of defaulting to false - } - } -} - -func processOriginalFilename(downloadPath string, actualFilename string) string { - _, err := os.Stat(downloadPath + actualFilename) - if os.IsNotExist(err) { - return actualFilename - } - extension := filepath.Ext(actualFilename) - filename := strings.TrimSuffix(actualFilename, extension) - counter := 2 - for { - newFilename := filename + "_" + strconv.Itoa(counter) + extension - _, err := os.Stat(downloadPath + newFilename) - if os.IsNotExist(err) { - return newFilename - } - counter++ - } -} - -// truncateFilename shortens long filenames for progress bar display -func truncateFilename(name string, max int) string { - if len(name) <= max { - return name - } - return "..." + name[len(name)-max+3:] -} - -// printRenamed shows renamed files in final summary -func printRenamed(ctx context.Context, logger *slog.Logger, renamed []RenamedOrSkippedFileInfo) { - for _, r := range renamed { - logger.InfoContext(ctx, fmt.Sprintf("Renamed %q to %q (GUID: %s)", r.OldFilename, r.NewFilename, r.GUID)) - } -} - -// printSkipped shows skipped files in final summary -func printSkipped(ctx context.Context, logger *slog.Logger, skipped []RenamedOrSkippedFileInfo) { - for _, s := range skipped { - logger.InfoContext(ctx, fmt.Sprintf("Skipped %q (GUID: %s)", s.OldFilename, s.GUID)) - } -} diff --git a/client/transfer/engine/downloader.go b/client/transfer/engine/downloader.go index afe40c3a..1051462c 100644 --- a/client/transfer/engine/downloader.go +++ b/client/transfer/engine/downloader.go @@ -101,14 +101,6 @@ func (d *GenericDownloader) downloadSingle(ctx context.Context, guid string, dst if expectedSize > 0 && (startOffset+written) < expectedSize { return fmt.Errorf("short download: got %d, expected %d", startOffset+written, expectedSize) } - emitTransferCompletion(ctx, common.TransferCompletionEvent{ - Direction: "download", - GUID: guid, - RangeStart: startOffset, - RangeEnd: startOffset + written - 1, - Bytes: written, - Strategy: "single", - }) return nil } @@ -264,15 +256,6 @@ func (d *GenericDownloader) downloadParallel(ctx context.Context, guid string, d if written != partLength { return fmt.Errorf("short write: got %d, expected %d", written, partLength) } - emitTransferCompletion(gctx, common.TransferCompletionEvent{ - Direction: "download", - GUID: guid, - RangeStart: partStart, - RangeEnd: partEnd, - Bytes: written, - PartNumber: partNum(partStart, chunkSize), - Strategy: "multipart", - }) return nil }) }) @@ -290,18 +273,6 @@ func (d *GenericDownloader) downloadParallel(ctx context.Context, guid string, d return nil } -func emitTransferCompletion(ctx context.Context, ev common.TransferCompletionEvent) { - if ev.Bytes <= 0 { - return - } - if cb := common.GetTransferCompletion(ctx); cb != nil { - _ = cb(ev) - } -} -func partNum(start, chunkSize int64) int { - if chunkSize <= 0 { - return 0 - } - return int(start/chunkSize) + 1 -} + + diff --git a/client/transfer/engine/engine_test.go b/client/transfer/engine/engine_test.go index 71ec1b49..cc957766 100644 --- a/client/transfer/engine/engine_test.go +++ b/client/transfer/engine/engine_test.go @@ -597,16 +597,11 @@ func TestGenericDownloaderDownloadSingleVariants(t *testing.T) { } var events []common.ProgressEvent - var completed []common.TransferCompletionEvent ctx := common.WithOid(context.Background(), "oid-download") ctx = common.WithProgress(ctx, func(ev common.ProgressEvent) error { events = append(events, ev) return nil }) - ctx = common.WithTransferCompletion(ctx, func(ev common.TransferCompletionEvent) error { - completed = append(completed, ev) - return nil - }) if err := d.downloadSingle(ctx, "guid-1", dst, 11); err != nil { t.Fatalf("downloadSingle returned error: %v", err) @@ -624,12 +619,7 @@ func TestGenericDownloaderDownloadSingleVariants(t *testing.T) { if len(events) != 1 || events[0].BytesSinceLast != 5 || events[0].BytesSoFar != 11 { t.Fatalf("unexpected progress events: %+v", events) } - if len(completed) != 1 { - t.Fatalf("expected one transfer completion event, got %+v", completed) - } - if completed[0].Direction != "download" || completed[0].GUID != "guid-1" || completed[0].RangeStart != 6 || completed[0].RangeEnd != 10 || completed[0].Bytes != 5 || completed[0].Strategy != "single" { - t.Fatalf("unexpected transfer completion event: %+v", completed[0]) - } + }) t.Run("range ignored restarts from zero", func(t *testing.T) { @@ -692,7 +682,6 @@ func TestGenericDownloaderDownloadAndParallel(t *testing.T) { var eventsMu sync.Mutex var events []common.ProgressEvent - var completions []common.TransferCompletionEvent ctx := common.WithOid(context.Background(), "parallel-oid") ctx = common.WithProgress(ctx, func(ev common.ProgressEvent) error { eventsMu.Lock() @@ -700,12 +689,6 @@ func TestGenericDownloaderDownloadAndParallel(t *testing.T) { eventsMu.Unlock() return nil }) - ctx = common.WithTransferCompletion(ctx, func(ev common.TransferCompletionEvent) error { - eventsMu.Lock() - completions = append(completions, ev) - eventsMu.Unlock() - return nil - }) if err := d.Download(ctx, "guid-p", dst, 3, 1*common.MB, 1*common.MB); err != nil { t.Fatalf("Download returned error: %v", err) @@ -723,23 +706,12 @@ func TestGenericDownloaderDownloadAndParallel(t *testing.T) { if len(events) < 3 { t.Fatalf("expected incremental progress events, got %+v", events) } - if len(completions) != 3 { - t.Fatalf("expected transfer completion per chunk, got %+v", completions) - } + lastEvent := events[len(events)-1] if lastEvent.BytesSoFar != int64(len(content)) { t.Fatalf("expected final progress to reach %d, got %+v", len(content), lastEvent) } - var completedBytes int64 - for _, ev := range completions { - if ev.Direction != "download" || ev.GUID != "guid-p" || ev.Strategy != "multipart" || ev.Bytes <= 0 || ev.RangeEnd < ev.RangeStart { - t.Fatalf("unexpected transfer completion event: %+v", ev) - } - completedBytes += ev.Bytes - } - if completedBytes != int64(len(content)) { - t.Fatalf("expected completed bytes %d, got %d events=%+v", len(content), completedBytes, completions) - } + backend2 := &fakeBackend{data: []byte("single"), meta: &transfer.ObjectMetadata{Size: 6, AcceptRanges: true}} d2 := &GenericDownloader{Source: backend2} @@ -768,19 +740,13 @@ func TestGenericDownloaderDownloadAndParallel(t *testing.T) { } d := &GenericDownloader{Source: backend} dst := filepath.Join(t.TempDir(), "failed-parallel.bin") - var completions []common.TransferCompletionEvent - ctx := common.WithTransferCompletion(context.Background(), func(ev common.TransferCompletionEvent) error { - completions = append(completions, ev) - return nil - }) + ctx := context.Background() err := d.Download(ctx, "guid-fail", dst, 2, 1*common.MB, 1*common.MB) if err == nil || !strings.Contains(err.Error(), "range download") { t.Fatalf("expected range download error, got %v", err) } - if len(completions) != 0 { - t.Fatalf("failed transfer should not emit completion events, got %+v", completions) - } + if _, statErr := os.Stat(dst); !os.IsNotExist(statErr) { t.Fatalf("expected failed parallel download to remove %q, stat err=%v", dst, statErr) } diff --git a/client/transfer/engine/progress_reader.go b/client/transfer/engine/progress_reader.go index 8ad64e84..71b70ad7 100644 --- a/client/transfer/engine/progress_reader.go +++ b/client/transfer/engine/progress_reader.go @@ -89,12 +89,7 @@ func (pr *progressReader) advance(delta int64) int64 { return pr.localBytes } -func (pr *progressReader) current() int64 { - if pr.globalBytes != nil { - return atomic.LoadInt64(pr.globalBytes) - } - return pr.localBytes -} + func (pr *progressReader) currentRaw() int64 { if pr.globalBytes != nil { diff --git a/client/transfer/noop_logger.go b/client/transfer/noop_logger.go index 98c72062..14dc1e21 100644 --- a/client/transfer/noop_logger.go +++ b/client/transfer/noop_logger.go @@ -6,7 +6,6 @@ import ( "log/slog" "github.com/calypr/syfon/client/common" - "github.com/calypr/syfon/client/logs" ) // NoOpLogger satisfies TransferLogger without emitting output. @@ -36,4 +35,3 @@ func (NoOpLogger) GetFailedLogMap() map[string]common.RetryObject { return map[string]common.RetryObject{} } func (NoOpLogger) DeleteFromFailedLog(string) {} -func (NoOpLogger) Scoreboard() *logs.Scoreboard { return nil } diff --git a/client/transfer/retry.go b/client/transfer/retry.go index 66e2b0a4..4613bd99 100644 --- a/client/transfer/retry.go +++ b/client/transfer/retry.go @@ -73,9 +73,6 @@ func RetryAction(ctx context.Context, logger TransferLogger, strategy RetryStrat lastErr = action() if lastErr == nil { - if sb := logger.Scoreboard(); sb != nil { - sb.IncrementSB(i) - } return nil } var nonRetryable nonRetryableError @@ -86,8 +83,5 @@ func RetryAction(ctx context.Context, logger TransferLogger, strategy RetryStrat logger.Error("Action failed", "retry", i, "error", lastErr) } - if sb := logger.Scoreboard(); sb != nil { - sb.IncrementSB(maxRetries + 1) - } return lastErr } diff --git a/client/transfer/retry_test.go b/client/transfer/retry_test.go index d9673c14..b60ce86a 100644 --- a/client/transfer/retry_test.go +++ b/client/transfer/retry_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/calypr/syfon/client/common" - "github.com/calypr/syfon/client/logs" ) type fixedStrategy time.Duration @@ -20,7 +19,6 @@ type captureLogger struct { NoOpLogger printfs []string errorsSeen []error - sb *logs.Scoreboard } func (l *captureLogger) Printf(format string, v ...any) { @@ -35,9 +33,7 @@ func (l *captureLogger) Error(msg string, args ...any) { } } -func (l *captureLogger) Scoreboard() *logs.Scoreboard { - return l.sb -} + func TestExponentialBackoffAndDefaultBackoff(t *testing.T) { t.Parallel() @@ -65,8 +61,8 @@ func TestExponentialBackoffAndDefaultBackoff(t *testing.T) { func TestRetryAction(t *testing.T) { t.Parallel() - t.Run("eventual success increments scoreboard", func(t *testing.T) { - logger := &captureLogger{sb: logs.NewSB(3, NoOpLogger{}.Slog())} + t.Run("eventual success", func(t *testing.T) { + logger := &captureLogger{} attempts := 0 err := RetryAction(context.Background(), logger, fixedStrategy(0), 3, func() error { attempts++ @@ -81,9 +77,6 @@ func TestRetryAction(t *testing.T) { if attempts != 3 { t.Fatalf("expected 3 attempts, got %d", attempts) } - if logger.sb.Counts[2] != 1 { - t.Fatalf("expected scoreboard success bucket for 2 retries, got %+v", logger.sb.Counts) - } if len(logger.printfs) != 2 || len(logger.errorsSeen) != 2 { t.Fatalf("expected retry logging, got printfs=%d errors=%d", len(logger.printfs), len(logger.errorsSeen)) } @@ -106,8 +99,8 @@ func TestRetryAction(t *testing.T) { } }) - t.Run("final failure increments terminal bucket", func(t *testing.T) { - logger := &captureLogger{sb: logs.NewSB(1, NoOpLogger{}.Slog())} + t.Run("final failure", func(t *testing.T) { + logger := &captureLogger{} wantErr := errors.New("still failing") err := RetryAction(context.Background(), logger, fixedStrategy(0), 1, func() error { return wantErr @@ -115,9 +108,6 @@ func TestRetryAction(t *testing.T) { if !errors.Is(err, wantErr) { t.Fatalf("expected %v, got %v", wantErr, err) } - if logger.sb.Counts[len(logger.sb.Counts)-1] != 1 { - t.Fatalf("expected terminal failure bucket increment, got %+v", logger.sb.Counts) - } }) } @@ -153,7 +143,4 @@ func TestNoOpLogger(t *testing.T) { if got := logger.GetFailedLogMap(); len(got) != 0 { t.Fatalf("expected empty failed log map, got %+v", got) } - if logger.Scoreboard() != nil { - t.Fatal("expected nil scoreboard") - } } diff --git a/client/transfer/types.go b/client/transfer/types.go index 56574bd8..321d6edf 100644 --- a/client/transfer/types.go +++ b/client/transfer/types.go @@ -10,7 +10,6 @@ import ( internalapi "github.com/calypr/syfon/apigen/client/internalapi" "github.com/calypr/syfon/client/common" "github.com/calypr/syfon/client/hash" - "github.com/calypr/syfon/client/logs" ) var ( @@ -38,7 +37,6 @@ type TransferLogger interface { GetSucceededLogMap() map[string]string GetFailedLogMap() map[string]common.RetryObject DeleteFromFailedLog(path string) - Scoreboard() *logs.Scoreboard } // Service provides high-level identity and logging access. diff --git a/client/transfer/upload/utils.go b/client/transfer/upload/utils.go index 5773cca9..3b9fa6b5 100644 --- a/client/transfer/upload/utils.go +++ b/client/transfer/upload/utils.go @@ -1,108 +1,11 @@ package upload import ( - "encoding/json" - "errors" "fmt" - "os" - "path/filepath" - "strings" "github.com/calypr/syfon/client/common" - "github.com/calypr/syfon/client/transfer" - "github.com/calypr/syfon/client/transfer/engine" ) -func SeparateSingleAndMultipartUploads(bk interface { - Logger() transfer.TransferLogger -}, objects []uploadRequest) ([]uploadRequest, []uploadRequest) { - fileSizeLimit := common.FileSizeLimit - logger := bk.Logger() - - var singlepartObjects []uploadRequest - var multipartObjects []uploadRequest - - for _, object := range objects { - fi, err := os.Stat(object.sourcePath) - if err != nil { - if os.IsNotExist(err) { - logger.Error("The file you specified does not exist locally", "path", object.sourcePath) - } else { - logger.Error("File stat error", "path", object.sourcePath, "error", err) - } - continue - } - if fi.IsDir() { - continue - } - if fi.Size() > common.MultipartFileSizeLimit { - logger.Warn("File exceeds max limit", "name", fi.Name(), "size", fi.Size()) - continue - } - if fi.Size() > int64(fileSizeLimit) { - multipartObjects = append(multipartObjects, object) - } else { - singlepartObjects = append(singlepartObjects, object) - } - } - return singlepartObjects, multipartObjects -} - -// ProcessFilename returns an FileInfo object which has the information about the path and name to be used for upload of a file -func ProcessFilename(logger transfer.TransferLogger, uploadPath string, filePath string, objectId string, includeSubDirName bool, includeMetadata bool) (string, string, common.FileMetadata, error) { - var err error - filePath, err = common.GetAbsolutePath(filePath) - if err != nil { - return "", "", common.FileMetadata{}, err - } - - filename := filepath.Base(filePath) // Default to base filename - - var metadata common.FileMetadata - if includeSubDirName { - absUploadPath, err := common.GetAbsolutePath(uploadPath) - if err != nil { - return "", "", common.FileMetadata{}, err - } - - // Ensure absUploadPath is a directory path for relative calculation - // Trim the optional wildcard if present - uploadDir := strings.TrimSuffix(absUploadPath, common.PathSeparator+"*") - fileInfo, err := os.Stat(uploadDir) - if err != nil { - return "", "", common.FileMetadata{}, err - } - if fileInfo.IsDir() { - // Calculate the path of the file relative to the upload directory - relPath, err := filepath.Rel(uploadDir, filePath) - if err != nil { - return "", "", common.FileMetadata{}, err - } - filename = relPath - } - } - - if includeMetadata { - // The metadata path is the file name plus '_metadata.json' - metadataFilePath := strings.TrimSuffix(filePath, filepath.Ext(filePath)) + "_metadata.json" - var metadataFileBytes []byte - if _, err := os.Stat(metadataFilePath); err == nil { - metadataFileBytes, err = os.ReadFile(metadataFilePath) - if err != nil { - return "", "", common.FileMetadata{}, errors.New("Error reading metadata file " + metadataFilePath + ": " + err.Error()) - } - err := json.Unmarshal(metadataFileBytes, &metadata) - if err != nil { - return "", "", common.FileMetadata{}, errors.New("Error parsing metadata file " + metadataFilePath + ": " + err.Error()) - } - } else { - // No metadata file was found for this file -- proceed, but warn the user. - logger.Printf("WARNING: File metadata is enabled, but could not find the metadata file %v for file %v. Execute `data-client upload --help` for more info on file metadata.\n", metadataFilePath, filePath) - } - } - return filePath, filename, metadata, nil -} - // FormatSize helps to parse a int64 size into string func FormatSize(size int64) string { var unitSize int64 @@ -129,8 +32,3 @@ func FormatSize(size int64) string { return fmt.Sprintf("%.1f"+unitMap[unitSize], float64(size)/float64(unitSize)) } - -// OptimalChunkSize returns a recommended chunk size for the given fileSize (in bytes). -func OptimalChunkSize(fileSize int64) int64 { - return engine.OptimalChunkSize(fileSize) -} diff --git a/client/transfer/upload/utils_test.go b/client/transfer/upload/utils_test.go deleted file mode 100644 index 9145e396..00000000 --- a/client/transfer/upload/utils_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package upload - -import ( - "testing" - - "github.com/calypr/syfon/client/common" -) - -func TestOptimalChunkSize(t *testing.T) { - tests := []struct { - name string - fileSize int64 - wantChunkSize int64 - wantParts int64 - }{ - { - name: "0 bytes", - fileSize: 0, - wantChunkSize: 1 * common.MB, - wantParts: 0, - }, - { - name: "1MB", - fileSize: 1 * common.MB, - wantChunkSize: 1 * common.MB, - wantParts: 1, - }, - { - name: "100MB", - fileSize: 100 * common.MB, - wantChunkSize: 100 * common.MB, - wantParts: 1, - }, - { - name: "100MB+1B", - fileSize: 100*common.MB + 1, - wantChunkSize: 10 * common.MB, - wantParts: 11, - }, - { - name: "500MB", - fileSize: 500 * common.MB, - wantChunkSize: 10 * common.MB, - wantParts: 50, - }, - { - name: "1GB", - fileSize: 1 * common.GB, - wantChunkSize: 10 * common.MB, - wantParts: 103, - }, - { - name: "1GB+1B", - fileSize: 1*common.GB + 1, - wantChunkSize: 25 * common.MB, - wantParts: 41, - }, - { - name: "5GB", - fileSize: 5 * common.GB, - wantChunkSize: 70 * common.MB, - wantParts: 74, - }, - { - name: "10GB", - fileSize: 10 * common.GB, - wantChunkSize: 128 * common.MB, - wantParts: 80, - }, - { - name: "10GB+1B", - fileSize: 10*common.GB + 1, - wantChunkSize: 256 * common.MB, - wantParts: 41, - }, - { - name: "50GB", - fileSize: 50 * common.GB, - wantChunkSize: 256 * common.MB, - wantParts: 200, - }, - { - name: "100GB", - fileSize: 100 * common.GB, - wantChunkSize: 256 * common.MB, - wantParts: 400, - }, - { - name: "100GB+1B", - fileSize: 100*common.GB + 1, - wantChunkSize: 512 * common.MB, - wantParts: 201, - }, - { - name: "500GB", - fileSize: 500 * common.GB, - wantChunkSize: 739 * common.MB, - wantParts: 693, - }, - { - name: "1TB", - fileSize: 1 * common.TB, - wantChunkSize: 1 * common.GB, - wantParts: 1024, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - chunkSize := OptimalChunkSize(tt.fileSize) - if chunkSize != tt.wantChunkSize { - t.Fatalf("chunk size = %d, want %d", chunkSize, tt.wantChunkSize) - } - - parts := int64(0) - if tt.fileSize > 0 && chunkSize > 0 { - parts = (tt.fileSize + chunkSize - 1) / chunkSize - } - if parts != tt.wantParts { - t.Fatalf("parts = %d, want %d", parts, tt.wantParts) - } - }) - } -} diff --git a/cmd/cliauth/auth.go b/cmd/cliauth/auth.go index 136ff002..23611d2e 100644 --- a/cmd/cliauth/auth.go +++ b/cmd/cliauth/auth.go @@ -30,9 +30,7 @@ func RegisterRootFlags(fs *pflag.FlagSet) { fs.StringVar(&password, "password", strings.TrimSpace(os.Getenv("SYFON_PASSWORD")), "Basic auth password for authenticated Syfon servers") } -func ResolvedProfile() string { - return strings.TrimSpace(profile) -} + func NewServerClient(cmd *cobra.Command) (syfonclient.SyfonClient, error) { serverURL, err := ResolveServerURL(cmd) diff --git a/cmd/download/main.go b/cmd/download/main.go index 017ff373..3faf60db 100644 --- a/cmd/download/main.go +++ b/cmd/download/main.go @@ -1,18 +1,11 @@ package download import ( - "context" "fmt" - "io" - "net/http" - "net/url" "os" "path/filepath" "strings" - syclient "github.com/calypr/syfon/client" - "github.com/calypr/syfon/client/request" - syfonclient "github.com/calypr/syfon/client/services" transferdownload "github.com/calypr/syfon/client/transfer/download" syupload "github.com/calypr/syfon/client/transfer/upload" "github.com/calypr/syfon/cmd/cliauth" @@ -83,55 +76,7 @@ var Cmd = &cobra.Command{ }, } -func downloadURLToPath(ctx context.Context, rawURL, outPath string, c syfonclient.SyfonClient) error { - parsed, err := url.Parse(strings.TrimSpace(rawURL)) - if err != nil { - return fmt.Errorf("parse download url: %w", err) - } - switch strings.ToLower(parsed.Scheme) { - case "": - srcPath := parsed.Path - if srcPath == "" { - srcPath = rawURL - } - data, err := os.ReadFile(srcPath) - if err != nil { - return fmt.Errorf("read file source: %w", err) - } - if err := os.WriteFile(outPath, data, 0o644); err != nil { - return fmt.Errorf("write output file: %w", err) - } - return nil - case "http", "https": - var resp *http.Response - concrete, ok := c.(*syclient.Client) - if !ok { - return fmt.Errorf("client implementation does not support raw requests") - } - err := concrete.Requestor().Do(ctx, http.MethodGet, rawURL, nil, &resp, request.WithSkipAuth(true)) - if err != nil { - return fmt.Errorf("download request failed: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - body, err := io.ReadAll(io.LimitReader(resp.Body, 4096)) - if err != nil { - return fmt.Errorf("read error response body: %w", err) - } - return fmt.Errorf("download failed status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))) - } - data, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("read download response: %w", err) - } - if err := os.WriteFile(outPath, data, 0o644); err != nil { - return fmt.Errorf("write output file: %w", err) - } - return nil - default: - return fmt.Errorf("unsupported download url scheme %q", parsed.Scheme) - } -} + func init() { Cmd.Flags().StringVar(&downloadDID, "did", "", "DRS object DID") diff --git a/cmd/download/main_test.go b/cmd/download/main_test.go index 0eb41c64..7fd79679 100644 --- a/cmd/download/main_test.go +++ b/cmd/download/main_test.go @@ -1,15 +1,9 @@ package download import ( - "context" - "net/http" - "net/http/httptest" - "os" - "path/filepath" "strings" "testing" - syclient "github.com/calypr/syfon/client" "github.com/spf13/cobra" ) @@ -22,83 +16,3 @@ func TestDownloadRunE_RequiresDID(t *testing.T) { t.Fatalf("expected missing did error, got: %v", err) } } - -func TestDownloadURLToPath_LocalPath(t *testing.T) { - srcDir := t.TempDir() - srcPath := filepath.Join(srcDir, "source.txt") - outPath := filepath.Join(srcDir, "out.txt") - payload := []byte("download-file-scheme") - if err := os.WriteFile(srcPath, payload, 0o644); err != nil { - t.Fatalf("write source: %v", err) - } - - err := downloadURLToPath(context.Background(), srcPath, outPath, nil) - if err != nil { - t.Fatalf("downloadURLToPath(local path) error: %v", err) - } - got, err := os.ReadFile(outPath) - if err != nil { - t.Fatalf("read output: %v", err) - } - if string(got) != string(payload) { - t.Fatalf("unexpected output content: got=%q want=%q", string(got), string(payload)) - } -} - -func TestDownloadURLToPath_HTTP_Success(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("http-payload")) - })) - defer ts.Close() - - client, err := syclient.New(ts.URL) - if err != nil { - t.Fatalf("create client: %v", err) - } - outPath := filepath.Join(t.TempDir(), "out.txt") - - err = downloadURLToPath(context.Background(), ts.URL+"/download", outPath, client) - if err != nil { - t.Fatalf("downloadURLToPath(http success) error: %v", err) - } - got, err := os.ReadFile(outPath) - if err != nil { - t.Fatalf("read output: %v", err) - } - if string(got) != "http-payload" { - t.Fatalf("unexpected output content: got=%q", string(got)) - } -} - -func TestDownloadURLToPath_HTTP_ErrorStatus(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "boom", http.StatusBadRequest) - })) - defer ts.Close() - - client, err := syclient.New(ts.URL) - if err != nil { - t.Fatalf("create client: %v", err) - } - outPath := filepath.Join(t.TempDir(), "out.txt") - - err = downloadURLToPath(context.Background(), ts.URL+"/download", outPath, client) - if err == nil || !strings.Contains(err.Error(), "download failed status=400") { - t.Fatalf("expected status error, got: %v", err) - } -} - -func TestDownloadURLToPath_UnsupportedScheme(t *testing.T) { - err := downloadURLToPath(context.Background(), "s3://bucket/key", filepath.Join(t.TempDir(), "out.txt"), nil) - if err == nil || !strings.Contains(err.Error(), "unsupported download url scheme") { - t.Fatalf("expected unsupported scheme error, got: %v", err) - } -} - -func TestDownloadURLToPath_HTTP_NonConcreteClient(t *testing.T) { - err := downloadURLToPath(context.Background(), "https://example.test/file", filepath.Join(t.TempDir(), "out.txt"), nil) - if err == nil || !strings.Contains(err.Error(), "client implementation does not support raw requests") { - t.Fatalf("expected non-concrete client error, got: %v", err) - } -} diff --git a/cmd/projectcopy/helpers.go b/cmd/projectcopy/helpers.go index 962e2f93..aa01d4c4 100644 --- a/cmd/projectcopy/helpers.go +++ b/cmd/projectcopy/helpers.go @@ -20,9 +20,7 @@ type Scope struct { Project string } -func (s Scope) String() string { - return s.Organization + "/" + s.Project -} + func ParseScopeArg(raw, label string) (Scope, error) { parts := strings.Split(strings.TrimSpace(raw), "/") diff --git a/cmd/validate/openapi_validator.go b/cmd/validate/openapi_validator.go index 8e6a9fd2..0db5a596 100644 --- a/cmd/validate/openapi_validator.go +++ b/cmd/validate/openapi_validator.go @@ -12,9 +12,7 @@ import ( "github.com/gofiber/fiber/v3/middleware/adaptor" ) -func newDefaultSpecValidator(specPath string) (fiber.Handler, error) { - return newSpecValidator(specPath, true) -} + func newSpecValidator(specPath string, lenientRoutes bool) (fiber.Handler, error) { loader := openapi3.NewLoader() diff --git a/cmd/validate/validate_test.go b/cmd/validate/validate_test.go index 360a99d4..85c4f7ab 100644 --- a/cmd/validate/validate_test.go +++ b/cmd/validate/validate_test.go @@ -118,8 +118,8 @@ paths: } } -func TestDefaultSpecValidatorAndBadSpec(t *testing.T) { - _, err := newDefaultSpecValidator("/definitely/missing/openapi.yaml") +func TestSpecValidatorBadSpec(t *testing.T) { + _, err := newSpecValidator("/definitely/missing/openapi.yaml", false) if err == nil { t.Fatal("expected missing spec error") } diff --git a/common/drs.go b/common/drs.go deleted file mode 100644 index 0db9944b..00000000 --- a/common/drs.go +++ /dev/null @@ -1,125 +0,0 @@ -package common - -import ( - "fmt" - - drsapi "github.com/calypr/syfon/apigen/client/drs" - "github.com/google/uuid" -) - -// drsUUIDNamespace is a fixed namespace UUID for deterministic DRS ID generation. -var drsUUIDNamespace = uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - -// DrsUUID generates a deterministic UUID for a DRS object from org, project, and hash. -func DrsUUID(org, project, hash string) string { - hash = NormalizeOid(hash) - resource, err := ResourcePath(org, project) - if err != nil || hash == "" || project == "" || resource == "" { - return "" - } - name := "sha256:" + hash + "|" + resource - return uuid.NewSHA1(drsUUIDNamespace, []byte(name)).String() -} - -// ObjectBuilder constructs DRS objects for a given bucket and project scope. -type ObjectBuilder struct { - Bucket string - Project string - Organization string - StoragePrefix string - PathStyle string -} - -// NewObjectBuilder creates an ObjectBuilder for the given bucket and project. -func NewObjectBuilder(bucket, project string) ObjectBuilder { - return ObjectBuilder{ - Bucket: bucket, - Project: project, - } -} - -// Build constructs a DRS object with the given file metadata. -func (b ObjectBuilder) Build(fileName string, checksum string, size int64, drsId string) (*drsapi.DrsObject, error) { - prefix := b.StoragePrefix - return BuildDrsObjWithPrefix(fileName, checksum, size, drsId, b.Bucket, b.Organization, b.Project, prefix) -} - -// BuildDrsObjWithPrefix builds a DRS object with an S3 access URL derived from -// the provided bucket, org, project, and storage prefix. -func BuildDrsObjWithPrefix(fileName string, checksum string, size int64, drsId string, bucket string, org string, project string, prefix string) (*drsapi.DrsObject, error) { - if checksum == "" { - return nil, fmt.Errorf("checksum is required") - } - - obj := &drsapi.DrsObject{ - Id: drsId, - SelfUri: "drs://" + drsId, - Size: size, - Name: &fileName, - Checksums: []drsapi.Checksum{ - {Type: "sha256", Checksum: checksum}, - }, - } - - if bucket == "" { - return obj, nil - } - - var accessURL string - if prefix != "" { - accessURL = fmt.Sprintf("s3://%s/%s/%s", bucket, prefix, checksum) - } else { - accessURL = fmt.Sprintf("s3://%s/%s", bucket, checksum) - } - - am := drsapi.AccessMethod{ - Type: drsapi.AccessMethodTypeS3, - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: accessURL}, - } - - if authzMap := AuthzMapFromScope(org, project); authzMap != nil { - controlled := AuthzMapToControlledAccess(authzMap) - obj.ControlledAccess = &controlled - } - - ams := []drsapi.AccessMethod{am} - obj.AccessMethods = &ams - return obj, nil -} - -// ConvertToCandidate converts a DRS object to a registration candidate, -// stripping server-assigned fields (Id, SelfUri, timestamps). -func ConvertToCandidate(obj *drsapi.DrsObject) drsapi.DrsObjectCandidate { - if obj == nil { - return drsapi.DrsObjectCandidate{} - } - return drsapi.DrsObjectCandidate{ - AccessMethods: obj.AccessMethods, - Aliases: obj.Aliases, - Checksums: obj.Checksums, - Contents: obj.Contents, - ControlledAccess: obj.ControlledAccess, - Description: obj.Description, - MimeType: obj.MimeType, - Name: obj.Name, - Size: obj.Size, - Version: obj.Version, - } -} - -func DrsObjectMatchesScope(obj *drsapi.DrsObject, org, project string) bool { - if obj == nil { - return false - } - return AuthzMapMatchesScope(ControlledAccessToAuthzMap(derefStringSlice(obj.ControlledAccess)), org, project) -} - -func derefStringSlice(ptr *[]string) []string { - if ptr == nil { - return nil - } - return append([]string(nil), (*ptr)...) -} diff --git a/common/drs_test.go b/common/drs_test.go deleted file mode 100644 index 073d5249..00000000 --- a/common/drs_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package common - -import ( - "testing" - - drsapi "github.com/calypr/syfon/apigen/client/drs" -) - -func TestDrsObjectMatchesScope(t *testing.T) { - t.Run("controlled access", func(t *testing.T) { - obj := &drsapi.DrsObject{ - ControlledAccess: &[]string{"/organization/org/project/proj"}, - } - if !DrsObjectMatchesScope(obj, "org", "proj") { - t.Fatal("expected controlled-access match") - } - }) - - t.Run("access method authz ignored", func(t *testing.T) { - authz := []string{"/programs/org/projects/proj"} - obj := &drsapi.DrsObject{ - AccessMethods: &[]drsapi.AccessMethod{{ - Type: drsapi.AccessMethodTypeS3, - Authorizations: &struct { - BearerAuthIssuers *[]string `json:"bearer_auth_issuers,omitempty"` - DrsObjectId *string `json:"drs_object_id,omitempty"` - PassportAuthIssuers *[]string `json:"passport_auth_issuers,omitempty"` - SupportedTypes *[]drsapi.AccessMethodAuthorizationsSupportedTypes `json:"supported_types,omitempty"` - }{BearerAuthIssuers: &authz}, - }}, - } - if DrsObjectMatchesScope(obj, "org", "proj") { - t.Fatal("access method authorizations should not define scope") - } - }) -} diff --git a/common/resource.go b/common/resource.go index ad4abcd0..c8ea5e88 100644 --- a/common/resource.go +++ b/common/resource.go @@ -23,13 +23,7 @@ func ResourcePath(org, project string) (string, error) { return "/organization/" + org + "/project/" + project, nil } -// StoragePrefix returns the storage path prefix for an org/project scope -// (no leading slash), e.g. "programs/myorg/projects/myproject". -func StoragePrefix(org, project string) string { - _ = org - _ = project - return "" -} + // NormalizeChecksum trims whitespace and an optional sha256: prefix. func NormalizeChecksum(raw string) string { @@ -161,28 +155,7 @@ func AuthzMapToControlledAccess(authzMap map[string][]string) []string { return NormalizeAccessResources(AuthzMapToList(authzMap)) } -// AuthzMapMatchesScope reports whether the authz map grants access for the given -// org and project. An empty project list in the map means org-wide access. -func AuthzMapMatchesScope(authzMap map[string][]string, org, project string) bool { - org = strings.TrimSpace(org) - project = strings.TrimSpace(project) - if len(authzMap) == 0 || org == "" { - return false - } - projects, ok := authzMap[org] - if !ok { - return false - } - if len(projects) == 0 { - return true - } - for _, candidate := range projects { - if strings.TrimSpace(candidate) == project { - return true - } - } - return false -} + // ResourceScope parses a canonical or raw access resource into // (organization, project). It accepts "/organization/org", diff --git a/common/resource_test.go b/common/resource_test.go index 9bcbbc1c..d5f875a1 100644 --- a/common/resource_test.go +++ b/common/resource_test.go @@ -41,17 +41,7 @@ func TestResourceAndAuthzHelpers(t *testing.T) { } }) - t.Run("storage prefix", func(t *testing.T) { - if got := StoragePrefix("", ""); got != "" { - t.Fatalf("expected empty prefix, got %q", got) - } - if got := StoragePrefix("syfon", ""); got != "" { - t.Fatalf("unexpected org prefix: %q", got) - } - if got := StoragePrefix("syfon", "e2e"); got != "" { - t.Fatalf("unexpected project prefix: %q", got) - } - }) + t.Run("checksum normalization", func(t *testing.T) { if got := NormalizeChecksum(" sha256:ABC123 "); got != "ABC123" { @@ -122,21 +112,5 @@ func TestResourceAndAuthzHelpers(t *testing.T) { } }) - t.Run("authz map matches scope", func(t *testing.T) { - projectScoped := map[string][]string{"syfon": {"e2e"}} - if !AuthzMapMatchesScope(projectScoped, "syfon", "e2e") { - t.Fatal("expected project-scoped match") - } - if AuthzMapMatchesScope(projectScoped, "syfon", "other") { - t.Fatal("expected project-scoped miss") - } - orgWide := map[string][]string{"syfon": {}} - if !AuthzMapMatchesScope(orgWide, "syfon", "anything") { - t.Fatal("expected org-wide match") - } - if AuthzMapMatchesScope(nil, "syfon", "e2e") { - t.Fatal("expected nil map miss") - } - }) } diff --git a/internal/api/middleware/middleware_test.go b/internal/api/middleware/middleware_test.go index 93126598..0e274768 100644 --- a/internal/api/middleware/middleware_test.go +++ b/internal/api/middleware/middleware_test.go @@ -276,7 +276,7 @@ func TestGen3ModePopulatesUserPrivilegesFromFence(t *testing.T) { t.Fatalf("sign token: %v", err) } - m := NewAuthzMiddleware(newTestLogger(), "gen3", "", "") + m := NewAuthzMiddleware(slog.Default(), "gen3", "", "") injectDummyAuthenticationPluginManager(m, true) app := fiber.New() app.Use(m.FiberMiddleware()) diff --git a/internal/api/middleware/plugin_clients.go b/internal/api/middleware/plugin_clients.go index 9374f4b2..7eb6b616 100644 --- a/internal/api/middleware/plugin_clients.go +++ b/internal/api/middleware/plugin_clients.go @@ -148,12 +148,5 @@ func (a *AuthzRPC) Authorize(ctx context.Context, in *plugin.AuthorizationInput) return &out, err } -// DummyPluginManager implements the same interface as PluginManager for testing. -type DummyPluginManager struct{} - -func (d *DummyPluginManager) Authorize(ctx context.Context, in *plugin.AuthorizationInput) (*plugin.AuthorizationOutput, error) { - return &plugin.AuthorizationOutput{Allow: true}, nil -} - // Ensure PluginManager implements pluginManagerInterface var _ pluginManagerInterface = (*PluginManager)(nil) diff --git a/internal/api/middleware/plugin_clients_test.go b/internal/api/middleware/plugin_clients_test.go index 5bb5599a..b6927316 100644 --- a/internal/api/middleware/plugin_clients_test.go +++ b/internal/api/middleware/plugin_clients_test.go @@ -2,215 +2,13 @@ package middleware import ( "context" - "errors" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "testing" "github.com/calypr/syfon/plugin" - "github.com/gofiber/fiber/v3" ) -type mockPlugin struct { - output *plugin.AuthorizationOutput - err error -} - -func (m *mockPlugin) Authorize(ctx context.Context, in *plugin.AuthorizationInput) (*plugin.AuthorizationOutput, error) { - return m.output, m.err -} - -func addRecovery(app *fiber.App, t *testing.T) { - app.Use(func(c fiber.Ctx) error { - defer func() { - if r := recover(); r != nil { - t.Fatalf("panic: %v", r) - } - }() - return c.Next() - }) -} - -func newTestLogger() *slog.Logger { - return slog.New(slog.NewTextHandler(io.Discard, nil)) -} - -func TestPluginIntegrationAllow(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{ - Allow: true, - Obligations: map[string]interface{}{ - "resources": []interface{}{"/foo", "/bar"}, - "privileges": map[string]interface{}{ - "/foo": []interface{}{"read", "update"}, - "/bar": []interface{}{"read"}, - }, - }, - }, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } -} - -func TestPluginIntegrationDeny(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{Allow: false, Reason: "denied"}, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 403 { - t.Fatalf("expected 403, got %d", resp.StatusCode) - } -} - -func TestPluginIntegrationError(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: nil, - err: errors.New("plugin error"), - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 401 { - t.Fatalf("expected 401, got %d", resp.StatusCode) - } -} - -func TestPluginIntegration_MissingClaims(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{Allow: true, Obligations: map[string]interface{}{}}, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer invalidtoken") // Not a real JWT - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } -} - -func TestPluginIntegration_PluginUnavailable(t *testing.T) { - mw := &AuthzMiddleware{pluginManager: nil, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 401 { - t.Fatalf("expected 401, got %d", resp.StatusCode) - } -} - -func TestPluginIntegration_UnexpectedObligations(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{ - Allow: true, - Obligations: map[string]interface{}{ - "resources": 123, // Not a []interface{} - "privileges": "not a map", - }, - }, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } -} +// DummyPluginManager implements the same interface as PluginManager for testing. +type DummyPluginManager struct{} -func TestPluginIntegration_NoAuthHeader(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{Allow: true, Obligations: map[string]interface{}{}}, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - // No Authorization header - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } +func (d *DummyPluginManager) Authorize(ctx context.Context, in *plugin.AuthorizationInput) (*plugin.AuthorizationOutput, error) { + return &plugin.AuthorizationOutput{Allow: true}, nil } diff --git a/internal/common/checksum.go b/internal/common/checksum.go index 827818d0..9e993b6d 100644 --- a/internal/common/checksum.go +++ b/internal/common/checksum.go @@ -106,23 +106,7 @@ func CanonicalSHA256(checksums []drs.Checksum) (string, bool) { return "", false } -// NormalizeSHA256 cleans and dedups a list of potential SHA256 hashes. -func NormalizeSHA256(values []string) []string { - targets := make([]string, 0, len(values)) - seen := make(map[string]struct{}, len(values)) - for _, raw := range values { - sha := strings.ToLower(strings.TrimSpace(NormalizeChecksum(strings.TrimSpace(raw)))) - if sha == "" { - continue - } - if _, ok := seen[sha]; ok { - continue - } - seen[sha] = struct{}{} - targets = append(targets, sha) - } - return targets -} + // ParseS3URL extracts bucket/key pairs from an s3:// URL. func ParseS3URL(raw string) (bucket string, key string, ok bool) { diff --git a/internal/common/checksum_test.go b/internal/common/checksum_test.go index 9d0aff0a..77e558b2 100644 --- a/internal/common/checksum_test.go +++ b/internal/common/checksum_test.go @@ -57,10 +57,7 @@ func TestSHAHelpers(t *testing.T) { t.Fatalf("CanonicalSHA256 mismatch: ok=%v sha=%q", ok, sha) } - norm := NormalizeSHA256([]string{"sha256:ABC", "abc", "abc", ""}) - if len(norm) != 1 || norm[0] != "abc" { - t.Fatalf("NormalizeSHA256 unexpected output: %+v", norm) - } + obj := models.InternalObject{DrsObject: drs.DrsObject{Checksums: []drs.Checksum{{Type: "md5", Checksum: "m"}}}} if !ObjectHasChecksumTypeAndValue(obj, "md5", "m") { diff --git a/internal/common/cloud.go b/internal/common/cloud.go index 61cc9f75..0b7b53f2 100644 --- a/internal/common/cloud.go +++ b/internal/common/cloud.go @@ -4,11 +4,8 @@ import ( "fmt" "net" "net/url" - "path/filepath" "regexp" "strings" - - "github.com/calypr/syfon/internal/models" ) const ( @@ -77,33 +74,7 @@ func ProviderToScheme(p string) string { } } -func ObjectURLForCredential(cred *models.S3Credential, key string) (string, error) { - if cred == nil { - return "", fmt.Errorf("credential is required") - } - cleanKey := strings.TrimPrefix(strings.TrimSpace(key), "/") - provider := NormalizeProvider(cred.Provider, S3Provider) - switch provider { - case S3Provider: - return fmt.Sprintf("%s%s/%s", S3Prefix, cred.Bucket, cleanKey), nil - case GCSProvider: - return fmt.Sprintf("%s%s/%s", GCSPrefix, cred.Bucket, cleanKey), nil - case AzureProvider: - return fmt.Sprintf("%s%s/%s", AzurePrefix, cred.Bucket, cleanKey), nil - case FileProvider: - root := filepath.Clean(strings.TrimSpace(cred.Endpoint)) - if root == "." || root == "" { - root = strings.TrimPrefix(strings.TrimSpace(cred.Bucket), "/") - } - if root == "" { - return "", fmt.Errorf("file provider requires an endpoint or bucket root") - } - return filepath.ToSlash(filepath.Join(root, cleanKey)), nil - default: - return "", fmt.Errorf("unsupported provider: %s", provider) - } -} func NormalizeStoragePath(rawPath, bucket string) (string, error) { p := strings.TrimSpace(rawPath) @@ -149,9 +120,7 @@ func ParseBucketProvider(raw string) (string, error) { // The rules are intentionally provider-specific: // - s3 and azure share the stricter DNS-style naming rules. // - gcs permits dots and underscores but still requires a DNS-safe shape. -func ValidateBucketName(providerName, bucketName string) error { - return ValidateBucketNameWithEndpoint(providerName, bucketName, "") -} + // ValidateBucketNameWithEndpoint validates a bucket/container name for the // given provider and endpoint. S3-compatible backends with a custom endpoint diff --git a/internal/common/cloud_test.go b/internal/common/cloud_test.go index 4c8ff68c..cba03f36 100644 --- a/internal/common/cloud_test.go +++ b/internal/common/cloud_test.go @@ -61,7 +61,7 @@ func TestValidateBucketName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := ValidateBucketName(tt.provider, tt.bucket) + err := ValidateBucketNameWithEndpoint(tt.provider, tt.bucket, "") if tt.wantErr && err == nil { t.Fatalf("expected error for provider=%q bucket=%q", tt.provider, tt.bucket) } diff --git a/internal/common/errors.go b/internal/common/errors.go index 79b8c816..58276b74 100644 --- a/internal/common/errors.go +++ b/internal/common/errors.go @@ -90,6 +90,3 @@ func IsNotFoundError(err error) bool { return errors.Is(err, ErrNotFound) } -func IsUnauthorizedError(err error) bool { - return errors.Is(err, ErrUnauthorized) -} diff --git a/internal/common/errors_test.go b/internal/common/errors_test.go index c37ac9b1..2f82c5e5 100644 --- a/internal/common/errors_test.go +++ b/internal/common/errors_test.go @@ -11,9 +11,6 @@ func TestAuthorizationError_UnwrapAndClassifiers(t *testing.T) { if !errors.Is(err, ErrUnauthorized) { t.Fatalf("expected AuthorizationError to unwrap to ErrUnauthorized") } - if !IsUnauthorizedError(err) { - t.Fatalf("expected IsUnauthorizedError=true") - } } func TestAuthorizationError_PublicMessage(t *testing.T) { diff --git a/internal/common/ptr.go b/internal/common/ptr.go index c8624842..dd119734 100644 --- a/internal/common/ptr.go +++ b/internal/common/ptr.go @@ -7,13 +7,7 @@ func Ptr[T any](v T) *T { return &v } -// Val returns the value of the pointer if not nil, otherwise the default value. -func Val[T any](p *T, def T) T { - if p == nil { - return def - } - return *p -} + // TimeVal returns the time value if not nil, otherwise zero time. func TimeVal(p *time.Time) time.Time { @@ -44,13 +38,7 @@ func DerefStringSlice(p *[]string) []string { return append([]string(nil), (*p)...) } -// IntVal returns the int value if not nil, otherwise 0. -func IntVal(p *int) int { - if p == nil { - return 0 - } - return *p -} + // Int64Val returns the int64 value if not nil, otherwise 0. func Int64Val(p *int64) int64 { @@ -60,10 +48,4 @@ func Int64Val(p *int64) int64 { return *p } -// FloatVal returns the float64 value if not nil, otherwise 0.0. -func FloatVal(p *float64) float64 { - if p == nil { - return 0.0 - } - return *p -} + diff --git a/internal/common/util.go b/internal/common/util.go index c2e72b77..d251cc39 100644 --- a/internal/common/util.go +++ b/internal/common/util.go @@ -5,40 +5,7 @@ import ( "strings" ) -// UniqueStrings returns a deduped slice of strings, preserving order. -func UniqueStrings(values []string) []string { - seen := make(map[string]struct{}, len(values)) - out := make([]string, 0, len(values)) - for _, v := range values { - if v == "" { - continue - } - if _, exists := seen[v]; exists { - continue - } - seen[v] = struct{}{} - out = append(out, v) - } - return out -} -// UniqueStringsCaseInsensitive returns a deduped slice of strings based on lowercase comparison, preserving the first-seen original string. -func UniqueStringsCaseInsensitive(values []string) []string { - seen := make(map[string]struct{}, len(values)) - out := make([]string, 0, len(values)) - for _, v := range values { - normalized := strings.ToLower(strings.TrimSpace(v)) - if normalized == "" { - continue - } - if _, exists := seen[normalized]; exists { - continue - } - seen[normalized] = struct{}{} - out = append(out, v) - } - return out -} // SchemeFromURL extracts the scheme from a URL string. func SchemeFromURL(raw string) string { @@ -48,14 +15,7 @@ func SchemeFromURL(raw string) string { return "" } -// NormalizeUploadKey ensures a key is valid for upload and defaults to ID if empty. -func NormalizeUploadKey(inputKey, id string) string { - k := strings.TrimSpace(inputKey) - if k == "" { - return id - } - return k -} + // BucketToURL converts a bucket and key to an s3:// URL. func BucketToURL(bucket, key string) string { diff --git a/internal/common/util_test.go b/internal/common/util_test.go index cfbc6c88..5d463ead 100644 --- a/internal/common/util_test.go +++ b/internal/common/util_test.go @@ -1,52 +1,10 @@ package common import ( - "reflect" "testing" ) -func TestUniqueStrings(t *testing.T) { - tests := []struct { - name string - input []string - expected []string - }{ - {"Empty", []string{}, []string{}}, - {"No Duplicates", []string{"a", "b", "c"}, []string{"a", "b", "c"}}, - {"With Duplicates", []string{"a", "b", "a", "c", "b"}, []string{"a", "b", "c"}}, - {"With Empty Strings", []string{"a", "", "b", ""}, []string{"a", "b"}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := UniqueStrings(tt.input); !reflect.DeepEqual(got, tt.expected) { - t.Errorf("UniqueStrings() = %v, want %v", got, tt.expected) - } - }) - } -} - -func TestUniqueStringsCaseInsensitive(t *testing.T) { - tests := []struct { - name string - input []string - expected []string - }{ - {"Empty", []string{}, []string{}}, - {"No Duplicates", []string{"a", "b", "c"}, []string{"a", "b", "c"}}, - {"With Duplicates Case", []string{"A", "a", "B", "b"}, []string{"A", "B"}}, - {"With Spacing", []string{" a ", "a", " B"}, []string{" a ", " B"}}, - {"With Empty Strings", []string{" ", ""}, []string{}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := UniqueStringsCaseInsensitive(tt.input); !reflect.DeepEqual(got, tt.expected) { - t.Errorf("UniqueStringsCaseInsensitive() = %v, want %v", got, tt.expected) - } - }) - } -} func TestSchemeFromURL(t *testing.T) { tests := []struct { @@ -70,27 +28,7 @@ func TestSchemeFromURL(t *testing.T) { } } -func TestNormalizeUploadKey(t *testing.T) { - tests := []struct { - name string - key string - id string - expected string - }{ - {"Normal Key", "my-key", "my-id", "my-key"}, - {"Empty Key", "", "my-id", "my-id"}, - {"Space Key", " ", "my-id", "my-id"}, - {"Untrimmed Key", " my-key ", "my-id", "my-key"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := NormalizeUploadKey(tt.key, tt.id); got != tt.expected { - t.Errorf("NormalizeUploadKey() = %v, want %v", got, tt.expected) - } - }) - } -} func TestBucketToURL(t *testing.T) { tests := []struct { diff --git a/internal/converters/converters.go b/internal/converters/converters.go deleted file mode 100644 index 907dc8b0..00000000 --- a/internal/converters/converters.go +++ /dev/null @@ -1,167 +0,0 @@ -package converters - -import ( - "fmt" - "path/filepath" - "strings" - "time" - - "github.com/calypr/syfon/apigen/server/drs" - "github.com/calypr/syfon/apigen/server/lfsapi" - syfoncommon "github.com/calypr/syfon/common" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" -) - -func CandidateToInternalObject(c drs.DrsObjectCandidate, now time.Time) (models.InternalObject, error) { - oid, ok := common.CanonicalSHA256(c.Checksums) - if !ok { - return models.InternalObject{}, fmt.Errorf("candidate must include sha256 checksum") - } - if c.AccessMethods == nil || len(*c.AccessMethods) == 0 { - return models.InternalObject{}, common.ErrAccessMethodsRequired - } - authzMap := syfoncommon.ControlledAccessToAuthzMap(common.DerefStringSlice(c.ControlledAccess)) - id, err := common.MintObjectIDFromChecksum(oid, syfoncommon.AuthzMapToList(authzMap)) - if err != nil { - return models.InternalObject{}, err - } - obj := drs.DrsObject{ - Id: id, - Size: c.Size, - CreatedTime: now, - UpdatedTime: &now, - Version: common.Ptr("1"), - MimeType: c.MimeType, - Description: c.Description, - Aliases: common.Ptr(append([]string(nil), common.DerefStringSlice(c.Aliases)...)), - Checksums: []drs.Checksum{{Type: "sha256", Checksum: oid}}, - } - if c.ControlledAccess != nil { - controlled := syfoncommon.NormalizeAccessResources(*c.ControlledAccess) - obj.ControlledAccess = &controlled - } - if c.Name != nil { - obj.Name = normalizedObjectNamePtr(c.Name) - } - obj.SelfUri = "drs://" + obj.Id - if obj.Name == nil || strings.TrimSpace(*obj.Name) == "" { - obj.Name = &oid - } - - seenAccess := make(map[string]struct{}) - if c.AccessMethods != nil { - for _, am := range *c.AccessMethods { - url := "" - if am.AccessUrl != nil { - url = strings.TrimSpace(am.AccessUrl.Url) - } - if url == "" { - continue - } - key := string(am.Type) + "|" + url - if _, ok := seenAccess[key]; ok { - continue - } - seenAccess[key] = struct{}{} - accessID := am.AccessId - if accessID == nil || strings.TrimSpace(*accessID) == "" { - accessID = common.Ptr(string(am.Type)) - } - if obj.AccessMethods == nil { - obj.AccessMethods = &[]drs.AccessMethod{} - } - - newMethod := drs.AccessMethod{ - Type: drs.AccessMethodType(am.Type), - AccessId: accessID, - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: url}, - } - *obj.AccessMethods = append(*obj.AccessMethods, newMethod) - } - } - if obj.AccessMethods == nil || len(*obj.AccessMethods) == 0 { - return models.InternalObject{}, common.ErrAccessMethodsRequired - } - return models.InternalObject{ - DrsObject: obj, - Authorizations: authzMap, - }, nil -} - -func normalizedObjectNamePtr(name *string) *string { - if name == nil { - return nil - } - trimmed := strings.TrimSpace(*name) - if trimmed == "" { - return nil - } - trimmed = strings.ReplaceAll(trimmed, "\\", "/") - base := filepath.Base(trimmed) - if base == "." || base == "/" || base == "" { - base = trimmed - } - return common.Ptr(base) -} - -func LFSCandidateToDRS(in lfsapi.DrsObjectCandidate) drs.DrsObjectCandidate { - var size int64 - if in.Size != nil { - size = *in.Size - } - aliases := append([]string(nil), common.DerefStringSlice(in.Aliases)...) - explicitID := strings.TrimSpace(common.DerefString(in.Id)) - out := drs.DrsObjectCandidate{ - Name: in.Name, - Size: size, - MimeType: in.MimeType, - Description: in.Description, - } - if in.Checksums != nil { - out.Checksums = make([]drs.Checksum, 0, len(*in.Checksums)) - for _, c := range *in.Checksums { - out.Checksums = append(out.Checksums, drs.Checksum{ - Type: c.Type, - Checksum: c.Checksum, - }) - } - if explicitID == "" { - for _, c := range out.Checksums { - if strings.EqualFold(strings.TrimSpace(c.Type), "sha256") { - explicitID = syfoncommon.NormalizeOid(c.Checksum) - break - } - } - } - } - if explicitID != "" { - aliases = append([]string{"id:" + explicitID}, aliases...) - } - out.Aliases = common.Ptr(aliases) - if in.AccessMethods != nil { - ams := make([]drs.AccessMethod, 0, len(*in.AccessMethods)) - for _, am := range *in.AccessMethods { - drsMethod := drs.AccessMethod{ - Type: drs.AccessMethodType(common.DerefString(am.Type)), - Region: am.Region, - AccessId: am.AccessId, - } - if am.AccessUrl != nil { - drsMethod.AccessUrl = &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{} - if am.AccessUrl.Url != nil { - drsMethod.AccessUrl.Url = *am.AccessUrl.Url - } - } - ams = append(ams, drsMethod) - } - out.AccessMethods = &ams - } - return out -} diff --git a/internal/converters/converters_test.go b/internal/converters/converters_test.go deleted file mode 100644 index df1e8a05..00000000 --- a/internal/converters/converters_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package converters - -import ( - "strings" - "testing" - "time" - - "github.com/calypr/syfon/apigen/server/drs" - "github.com/calypr/syfon/apigen/server/lfsapi" -) - -func TestCandidateToInternalObjectAndLFSCandidateToDRS(t *testing.T) { - t.Run("candidate to internal object", func(t *testing.T) { - authz := []string{"/programs/syfon/projects/e2e"} - url := "https://storage.example/object.bin" - name := "object.bin" - size := int64(42) - candidate := drs.DrsObjectCandidate{ - Name: &name, - Size: size, - Checksums: []drs.Checksum{{Type: "sha256", Checksum: strings.Repeat("a", 64)}}, - AccessMethods: &[]drs.AccessMethod{{ - Type: "https", - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: url}, - }}, - ControlledAccess: &authz, - } - - obj, err := CandidateToInternalObject(candidate, time.Unix(123, 0)) - if err != nil { - t.Fatalf("CandidateToInternalObject returned error: %v", err) - } - if got := obj.Authorizations["syfon"]; len(got) != 1 || got[0] != "e2e" { - t.Fatalf("unexpected internal authz map: %+v", obj.Authorizations) - } - if obj.DrsObject.AccessMethods == nil || len(*obj.DrsObject.AccessMethods) != 1 { - t.Fatalf("expected one access method, got %+v", obj.DrsObject.AccessMethods) - } - if (*obj.DrsObject.AccessMethods)[0].Authorizations != nil { - t.Fatalf("did not expect per-method authz, got %+v", (*obj.DrsObject.AccessMethods)[0].Authorizations) - } - }) - - t.Run("candidate without access methods fails", func(t *testing.T) { - candidate := drs.DrsObjectCandidate{ - Size: 42, - Checksums: []drs.Checksum{{Type: "sha256", Checksum: strings.Repeat("b", 64)}}, - } - if _, err := CandidateToInternalObject(candidate, time.Unix(123, 0)); err == nil || !strings.Contains(err.Error(), "access method") { - t.Fatalf("expected access-method validation error, got %v", err) - } - }) - - t.Run("lfs candidate to drs", func(t *testing.T) { - url := "https://storage.example/object.bin" - size := int64(42) - candidate := lfsapi.DrsObjectCandidate{ - Name: strPtr("object.bin"), - Size: &size, - Checksums: &[]lfsapi.Checksum{{ - Type: "sha256", Checksum: "abc123", - }}, - AccessMethods: &[]lfsapi.AccessMethod{{ - Type: strPtr("https"), - AccessUrl: &lfsapi.AccessMethodAccessUrl{ - Url: &url, - }, - }}, - } - - got := LFSCandidateToDRS(candidate) - if got.AccessMethods == nil || len(*got.AccessMethods) != 1 { - t.Fatalf("expected one access method, got %+v", got.AccessMethods) - } - if (*got.AccessMethods)[0].Authorizations != nil { - t.Fatalf("did not expect authz map on access method, got %+v", (*got.AccessMethods)[0].Authorizations) - } - }) -} - -func strPtr(s string) *string { return &s } diff --git a/internal/converters/util.go b/internal/converters/util.go deleted file mode 100644 index 6064ead1..00000000 --- a/internal/converters/util.go +++ /dev/null @@ -1,13 +0,0 @@ -package converters - -import ( - "sort" - - "github.com/calypr/syfon/internal/models" -) - -func SortInternalObjects(objects []models.InternalObject) { - sort.Slice(objects, func(i, j int) bool { - return objects[i].Id < objects[j].Id - }) -} diff --git a/internal/core/bucket_catalog.go b/internal/core/bucket_catalog.go index b081acc8..3fc895f3 100644 --- a/internal/core/bucket_catalog.go +++ b/internal/core/bucket_catalog.go @@ -294,20 +294,7 @@ func (m *ObjectManager) listBucketsVisibleObjects(ctx context.Context) ([]models return m.filterObjectsByMethod(ctx, objects, objectMethodRead), nil } -func bucketReferencedByPublicObject(objects []models.InternalObject, cred models.S3Credential, creds []models.S3Credential) bool { - credentialID := credentialIDForCredential(cred) - for _, obj := range objects { - if len(ObjectAccessResources(&obj)) != 0 || obj.AccessMethods == nil { - continue - } - for _, method := range *obj.AccessMethods { - if id, ok := credentialIDForAccessMethod(method, creds); ok && id == credentialID { - return true - } - } - } - return false -} + func credentialIDForCredential(cred models.S3Credential) string { if credentialID := strings.TrimSpace(cred.CredentialID); credentialID != "" { diff --git a/internal/core/object_storage_inspect.go b/internal/core/object_storage_inspect.go index e1b95e2d..94dd4cfa 100644 --- a/internal/core/object_storage_inspect.go +++ b/internal/core/object_storage_inspect.go @@ -83,14 +83,7 @@ func (e *StorageInspectError) Error() string { return fmt.Sprintf("storage inspect failed: %s", e.Kind) } -func WithStorageInspectCache(ctx context.Context) context.Context { - if cache, _ := ctx.Value(storageInspectCacheKey).(*storageInspectRequestCache); cache != nil { - return ctx - } - return context.WithValue(ctx, storageInspectCacheKey, &storageInspectRequestCache{ - credentials: map[string]storageInspectCredentialCacheEntry{}, - }) -} + func storageInspectCacheFromContext(ctx context.Context) *storageInspectRequestCache { cache, _ := ctx.Value(storageInspectCacheKey).(*storageInspectRequestCache) diff --git a/internal/core/service_test.go b/internal/core/service_test.go index 75402911..18e9b9f6 100644 --- a/internal/core/service_test.go +++ b/internal/core/service_test.go @@ -991,59 +991,5 @@ func TestObjectManagerDeleteResolveAndSignDelegation(t *testing.T) { } }) - t.Run("storage inspect cache reuses bucket credential and visibility lookups", func(t *testing.T) { - db := &coreTestDB{ - MockDatabase: &testutils.MockDatabase{ - Credentials: map[string]models.S3Credential{ - "gdcdata": { - CredentialID: "gdcdata", - Bucket: "gdcdata", - Provider: "s3", - Region: "us-east-1", - AccessKey: "test-key", - SecretKey: "test-secret", - }, - }, - Objects: map[string]*drs.DrsObject{ - "obj-1": { - Id: "obj-1", - AccessMethods: &[]drs.AccessMethod{ - { - Type: drs.AccessMethodTypeS3, - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{ - Url: "s3://gdcdata/existing/path-a", - }, - }, - }, - }, - }, - }, - } - om := NewObjectManager(db, &capturingURLManager{}) - om.SetS3ObjectInspector(func(ctx context.Context, cred models.S3Credential, bucket string, key string) (*StorageObjectMetadata, error) { - return &StorageObjectMetadata{ - ObjectURL: "s3://" + bucket + "/" + key, - Bucket: bucket, - Key: key, - }, nil - }) - ctx := WithStorageInspectCache(context.Background()) - if _, err := om.InspectStorageObject(ctx, InspectStorageRequest{ObjectURL: "s3://gdcdata/probe/path-a"}); err != nil { - t.Fatalf("first InspectStorageObject failed: %v", err) - } - if _, err := om.InspectStorageObject(ctx, InspectStorageRequest{ObjectURL: "s3://gdcdata/probe/path-b"}); err != nil { - t.Fatalf("second InspectStorageObject failed: %v", err) - } - - if db.getS3CredentialCalls != 1 { - t.Fatalf("expected one GetS3Credential call, got %d", db.getS3CredentialCalls) - } - if db.listS3CredentialsCalls != 1 { - t.Fatalf("expected one ListS3Credentials call, got %d", db.listS3CredentialsCalls) - } - }) } diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 482cec58..69cc4b89 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -169,19 +169,7 @@ func newAWSKMSKeyManagerFromEnv() (CredentialKeyManager, error) { }, nil } -func RegisterCredentialKeyManager(name string, factory func() (CredentialKeyManager, error)) error { - name = strings.ToLower(strings.TrimSpace(name)) - if name == "" { - return errors.New("credential key manager name is required") - } - if factory == nil { - return errors.New("credential key manager factory is required") - } - credentialKeyManagerRegistryMu.Lock() - defer credentialKeyManagerRegistryMu.Unlock() - credentialKeyManagerRegistry[name] = factory - return nil -} + func resolveCredentialKeyManager(name string) (CredentialKeyManager, error) { managerName := strings.ToLower(strings.TrimSpace(name)) diff --git a/internal/db/postgres/metrics.go b/internal/db/postgres/metrics.go index 8e7138b0..67cd5c1c 100644 --- a/internal/db/postgres/metrics.go +++ b/internal/db/postgres/metrics.go @@ -615,12 +615,7 @@ func nullableInt64(v *int64) any { return *v } -func nullableTime(v *time.Time) any { - if v == nil { - return nil - } - return v.UTC() -} + func (db *PostgresDB) backfillAccessGrants(ctx context.Context) error { tx, err := db.db.BeginTx(ctx, nil) @@ -1033,78 +1028,7 @@ func postgresTransferResourceClause(resources []string, startIndex int) (string, return strings.Join(clauses, " OR "), args } -func providerTransferWhere(filter models.TransferAttributionFilter) (string, []any) { - parts := make([]string, 0) - args := make([]any, 0) - add := func(column string, value any) { - args = append(args, value) - parts = append(parts, fmt.Sprintf("%s = $%d", column, len(args))) - } - status := strings.TrimSpace(filter.ReconciliationStatus) - if status == "" { - status = models.ProviderTransferMatched - } - if status != "all" { - add("reconciliation_status", status) - } - if strings.TrimSpace(filter.Organization) != "" { - add("organization", strings.TrimSpace(filter.Organization)) - } - if strings.TrimSpace(filter.Project) != "" { - add("project", strings.TrimSpace(filter.Project)) - } - direction := strings.TrimSpace(filter.Direction) - if direction == "" { - switch strings.TrimSpace(filter.EventType) { - case models.ProviderTransferDirectionDownload: - direction = models.ProviderTransferDirectionDownload - case models.ProviderTransferDirectionUpload: - direction = models.ProviderTransferDirectionUpload - } - } - if direction != "" && direction != "all" { - add("direction", direction) - } - if filter.From != nil { - args = append(args, filter.From.UTC()) - parts = append(parts, fmt.Sprintf("event_time >= $%d", len(args))) - } - if filter.To != nil { - args = append(args, filter.To.UTC()) - parts = append(parts, fmt.Sprintf("event_time <= $%d", len(args))) - } - if strings.TrimSpace(filter.Provider) != "" { - add("provider", strings.TrimSpace(filter.Provider)) - } - if strings.TrimSpace(filter.Bucket) != "" { - add("bucket", strings.TrimSpace(filter.Bucket)) - } - if strings.TrimSpace(filter.SHA256) != "" { - add("sha256", strings.TrimSpace(filter.SHA256)) - } - if strings.TrimSpace(filter.User) != "" { - user := strings.TrimSpace(filter.User) - args = append(args, user, user) - parts = append(parts, fmt.Sprintf("(actor_email = $%d OR actor_subject = $%d)", len(args)-1, len(args))) - } - if len(parts) == 0 { - return "", args - } - return " WHERE " + strings.Join(parts, " AND "), args -} -func providerTransferGroupExpr(groupBy string) (string, string) { - switch strings.ToLower(strings.TrimSpace(groupBy)) { - case "user": - return "COALESCE(NULLIF(actor_email, ''), actor_subject), actor_email, actor_subject", "COALESCE(NULLIF(actor_email, ''), actor_subject) AS key, '' AS organization, '' AS project, '' AS provider, '' AS bucket, '' AS sha256, actor_email, actor_subject" - case "provider": - return "provider, bucket", "provider || ':' || bucket AS key, '' AS organization, '' AS project, provider, bucket, '' AS sha256, '' AS actor_email, '' AS actor_subject" - case "object": - return "sha256", "sha256 AS key, '' AS organization, '' AS project, '' AS provider, '' AS bucket, sha256, '' AS actor_email, '' AS actor_subject" - default: - return "organization, project", "organization || '/' || project AS key, organization, project, '' AS provider, '' AS bucket, '' AS sha256, '' AS actor_email, '' AS actor_subject" - } -} func transferAttributionGroupExpr(groupBy string) (string, string) { switch strings.ToLower(strings.TrimSpace(groupBy)) { diff --git a/internal/db/postgres/metrics_sqlmock_test.go b/internal/db/postgres/metrics_sqlmock_test.go index ca42a4fc..4b43cd62 100644 --- a/internal/db/postgres/metrics_sqlmock_test.go +++ b/internal/db/postgres/metrics_sqlmock_test.go @@ -70,17 +70,6 @@ func TestTransferAttributionWhereHelpers(t *testing.T) { } func TestProviderTransferHelpers(t *testing.T) { - where, args := providerTransferWhere(models.TransferAttributionFilter{User: "alice", ReconciliationStatus: "all"}) - if !strings.Contains(where, "actor_email") { - t.Fatalf("expected user filter in provider where clause: %q", where) - } - if len(args) < 2 { - t.Fatalf("expected user args in provider where clause") - } - - if key, _ := providerTransferGroupExpr("user"); !strings.Contains(key, "actor_email") { - t.Fatalf("unexpected provider group expression for user: %q", key) - } if key, _ := transferAttributionGroupExpr("provider"); !strings.Contains(key, "provider") { t.Fatalf("unexpected attribution group expression for provider: %q", key) } diff --git a/internal/db/sqlite/metrics.go b/internal/db/sqlite/metrics.go index b331a43f..4e7032df 100644 --- a/internal/db/sqlite/metrics.go +++ b/internal/db/sqlite/metrics.go @@ -677,12 +677,7 @@ func nullableInt64(v *int64) any { return *v } -func nullableTime(v *time.Time) any { - if v == nil { - return nil - } - return v.UTC() -} + func (db *SqliteDB) backfillAccessGrants(ctx context.Context) error { tx, err := db.db.BeginTx(ctx, nil) @@ -1073,76 +1068,7 @@ func sqliteTransferResourceClause(resources []string) (string, []any) { return strings.Join(clauses, " OR "), args } -func providerTransferWhere(filter models.TransferAttributionFilter) (string, []any) { - parts := make([]string, 0) - args := make([]any, 0) - add := func(clause string, value any) { - parts = append(parts, clause) - args = append(args, value) - } - status := strings.TrimSpace(filter.ReconciliationStatus) - if status == "" { - status = models.ProviderTransferMatched - } - if status != "all" { - add("reconciliation_status = ?", status) - } - if strings.TrimSpace(filter.Organization) != "" { - add("organization = ?", strings.TrimSpace(filter.Organization)) - } - if strings.TrimSpace(filter.Project) != "" { - add("project = ?", strings.TrimSpace(filter.Project)) - } - direction := strings.TrimSpace(filter.Direction) - if direction == "" { - switch strings.TrimSpace(filter.EventType) { - case models.ProviderTransferDirectionDownload: - direction = models.ProviderTransferDirectionDownload - case models.ProviderTransferDirectionUpload: - direction = models.ProviderTransferDirectionUpload - } - } - if direction != "" && direction != "all" { - add("direction = ?", direction) - } - if filter.From != nil { - add("event_time >= ?", filter.From.UTC()) - } - if filter.To != nil { - add("event_time <= ?", filter.To.UTC()) - } - if strings.TrimSpace(filter.Provider) != "" { - add("provider = ?", strings.TrimSpace(filter.Provider)) - } - if strings.TrimSpace(filter.Bucket) != "" { - add("bucket = ?", strings.TrimSpace(filter.Bucket)) - } - if strings.TrimSpace(filter.SHA256) != "" { - add("sha256 = ?", strings.TrimSpace(filter.SHA256)) - } - if strings.TrimSpace(filter.User) != "" { - user := strings.TrimSpace(filter.User) - parts = append(parts, "(actor_email = ? OR actor_subject = ?)") - args = append(args, user, user) - } - if len(parts) == 0 { - return "", args - } - return " WHERE " + strings.Join(parts, " AND "), args -} -func providerTransferGroupExpr(groupBy string) (string, string) { - switch strings.ToLower(strings.TrimSpace(groupBy)) { - case "user": - return "COALESCE(NULLIF(actor_email, ''), actor_subject), actor_email, actor_subject", "COALESCE(NULLIF(actor_email, ''), actor_subject) AS key, '' AS organization, '' AS project, '' AS provider, '' AS bucket, '' AS sha256, actor_email, actor_subject" - case "provider": - return "provider, bucket", "provider || ':' || bucket AS key, '' AS organization, '' AS project, provider, bucket, '' AS sha256, '' AS actor_email, '' AS actor_subject" - case "object": - return "sha256", "sha256 AS key, '' AS organization, '' AS project, '' AS provider, '' AS bucket, sha256, '' AS actor_email, '' AS actor_subject" - default: - return "organization, project", "organization || '/' || project AS key, organization, project, '' AS provider, '' AS bucket, '' AS sha256, '' AS actor_email, '' AS actor_subject" - } -} func transferAttributionGroupExpr(groupBy string) (string, string) { switch strings.ToLower(strings.TrimSpace(groupBy)) { diff --git a/internal/db/sqlite/objects.go b/internal/db/sqlite/objects.go index e3a30834..73c09ee9 100644 --- a/internal/db/sqlite/objects.go +++ b/internal/db/sqlite/objects.go @@ -1413,9 +1413,6 @@ func (db *SqliteDB) RemoveObjectControlledAccess(ctx context.Context, objectID, if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_controlled_access WHERE object_id = ? AND resource = ?`, objectID, resource); err != nil { return err } - if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = ? AND resource = ?`, objectID, resource); err != nil { - return err - } return tx.Commit() } diff --git a/internal/testutils/mocks.go b/internal/testutils/mocks.go index 29b21465..a1bbb261 100644 --- a/internal/testutils/mocks.go +++ b/internal/testutils/mocks.go @@ -764,64 +764,6 @@ func (m *MockDatabase) GetTransferAttributionBreakdown(ctx context.Context, filt return out, nil } -func providerTransferEventMatchesFilter(ev models.ProviderTransferEvent, filter models.TransferAttributionFilter) bool { - status := filter.ReconciliationStatus - if status == "" { - status = models.ProviderTransferMatched - } - if status != "all" && ev.ReconciliationStatus != status { - return false - } - if filter.Organization != "" && ev.Organization != filter.Organization { - return false - } - if filter.Project != "" && ev.Project != filter.Project { - return false - } - direction := filter.Direction - if direction == "" { - direction = filter.EventType - } - if direction != "" && direction != "all" && ev.Direction != direction { - return false - } - if filter.From != nil && ev.EventTime.Before(*filter.From) { - return false - } - if filter.To != nil && ev.EventTime.After(*filter.To) { - return false - } - if filter.Provider != "" && ev.Provider != filter.Provider { - return false - } - if filter.Bucket != "" && ev.Bucket != filter.Bucket { - return false - } - if filter.SHA256 != "" && ev.SHA256 != filter.SHA256 { - return false - } - if filter.User != "" && ev.ActorEmail != filter.User && ev.ActorSubject != filter.User { - return false - } - return true -} - -func providerTransferBreakdownKey(ev models.ProviderTransferEvent, groupBy string) string { - switch groupBy { - case "user": - if ev.ActorEmail != "" { - return ev.ActorEmail - } - return ev.ActorSubject - case "provider": - return ev.Provider + ":" + ev.Bucket - case "object": - return ev.SHA256 - default: - return ev.Organization + "/" + ev.Project - } -} - func transferEventMatchesFilter(ev models.TransferAttributionEvent, filter models.TransferAttributionFilter) bool { if filter.Organization != "" && ev.Organization != filter.Organization { return false @@ -881,6 +823,8 @@ func transferBreakdownKey(ev models.TransferAttributionEvent, groupBy string) st } } + + func cloneAuthzMap(in map[string][]string) map[string][]string { if len(in) == 0 { return nil From b9d9d964eb8bfdfea6aa94e133a73441a96270c6 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 29 Jun 2026 11:51:06 -0700 Subject: [PATCH 08/24] fix gh review issues --- internal/api/internaldrs/repair_cleanup.go | 5 ++- .../api/internaldrs/repair_cleanup_test.go | 30 ++++++++++++++ internal/repair/service.go | 15 +++++-- internal/repair/service_test.go | 39 +++++++++++++++++++ 4 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 internal/api/internaldrs/repair_cleanup_test.go diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go index 127dfd0d..851980fe 100644 --- a/internal/api/internaldrs/repair_cleanup.go +++ b/internal/api/internaldrs/repair_cleanup.go @@ -79,7 +79,10 @@ func handleInternalScopeRepairApplyFiber(om *core.ObjectManager) fiber.Handler { if req.Organization == "" || req.Project == "" { return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") } - if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read", "update"); err != nil { + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read"); err != nil { + return apiutil.HandleError(c, err) + } + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "update"); err != nil { return apiutil.HandleError(c, err) } result, err := svc.Apply(c.Context(), req) diff --git a/internal/api/internaldrs/repair_cleanup_test.go b/internal/api/internaldrs/repair_cleanup_test.go new file mode 100644 index 00000000..1ef62b05 --- /dev/null +++ b/internal/api/internaldrs/repair_cleanup_test.go @@ -0,0 +1,30 @@ +package internaldrs + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + sycommon "github.com/calypr/syfon/common" + "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/testutils" +) + +func TestScopeRepairApplyRejectsReadOnlyProjectUser(t *testing.T) { + resource, err := sycommon.ResourcePath("org", "proj") + if err != nil { + t.Fatalf("resource path: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/data/repair/project-scope/apply", strings.NewReader(`{"organization":"org","project":"proj"}`)) + req.Header.Set("Content-Type", "application/json") + req = req.WithContext(policyTestContext("gen3", true, map[string]map[string]bool{ + resource: {"read": true}, + })) + + rr := doInternalDRSTestRequest(req, core.NewObjectManager(&testutils.MockDatabase{}, &testutils.MockUrlManager{})) + if rr.Code != http.StatusForbidden { + t.Fatalf("expected 403 for read-only caller, got %d body=%s", rr.Code, rr.Body.String()) + } +} diff --git a/internal/repair/service.go b/internal/repair/service.go index 765401d8..967bea6a 100644 --- a/internal/repair/service.go +++ b/internal/repair/service.go @@ -146,8 +146,13 @@ func (s *Service) auditRecord(ctx context.Context, rec internalapi.InternalRecor obj.scopeAmbiguous = ambiguous obj.inferredScope = resource if known { - obj.scope = scopes[resource][0] - obj.canonicalURL = canonicalAccessURL(obj.scope, rec.Did, sha) + targets := scopes[resource] + if len(targets) == 0 { + obj.scopeKnown = false + } else { + obj.scope = targets[0] + obj.canonicalURL = canonicalAccessURL(obj.scope, rec.Did, sha) + } } targetResource := "" @@ -315,7 +320,11 @@ func (s *Service) addDuplicateFindings(objects []*auditedObject) { func inferRecordResource(rec internalapi.InternalRecord, sha string, scopes map[string][]scopeTarget) (string, bool, bool) { projectResources := recordProjectResources(rec, "") if len(projectResources) == 1 { - return projectResources[0], true, false + resource := projectResources[0] + if len(scopes[resource]) == 0 { + return resource, false, false + } + return resource, true, false } if len(projectResources) > 1 { return "", false, true diff --git a/internal/repair/service_test.go b/internal/repair/service_test.go index 9ca2ca11..5635b170 100644 --- a/internal/repair/service_test.go +++ b/internal/repair/service_test.go @@ -172,6 +172,45 @@ func TestCheckStorageAddsMissingObjectFinding(t *testing.T) { } } +func TestSingleControlledAccessWithoutScopeTargetDoesNotPanicOrMutate(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + rec := recordWithMethods(t, "did-1", "abc", []string{resource}, []string{"s3://bforepc/legacy/file.ndjson"}) + idx := &fakeIndex{records: []internalapi.InternalRecord{rec}} + svc := NewService( + idx, + &fakeBuckets{ + list: bucketapi.BucketsResponse{ + S3BUCKETS: map[string]bucketapi.BucketMetadata{ + "other-bucket": {}, + }, + }, + scopes: map[string][]bucketapi.BucketScopeResponse{ + "other-bucket": nil, + }, + }, + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 0 { + t.Fatalf("expected no findings without a canonical target, got %d", len(report.Objects)) + } + + result, err := svc.Apply(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if result.Mutated != 0 || result.AutoFixable != 0 { + t.Fatalf("expected no mutations without a canonical target, got mutated=%d autofixable=%d", result.Mutated, result.AutoFixable) + } + if len(idx.updated) != 0 { + t.Fatalf("expected no updates, got %d", len(idx.updated)) + } +} + type fakeIndex struct { records []internalapi.InternalRecord updated []internalapi.InternalRecord From f20e0a2c59463f0b58eb42d3f5c7c61b7ff03dee Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 1 Jul 2026 14:26:22 -0700 Subject: [PATCH 09/24] implement bulk HEAD operations for files in bucket --- internal/api/drsapi/mutations.go | 5 +- internal/api/internaldrs/inspect.go | 397 +++++++++++++++++- .../api/internaldrs/inspect_routes_test.go | 90 ++++ internal/api/internaldrs/repair_cleanup.go | 52 ++- internal/api/internaldrs/transfer.go | 5 + internal/common/routes.go | 35 +- internal/common/util.go | 15 + internal/common/util_test.go | 24 ++ internal/core/object_mutations.go | 15 + internal/core/object_reads.go | 24 ++ internal/core/object_storage_inspect.go | 296 ++++++++++++- internal/core/object_storage_inspect_test.go | 23 + internal/core/object_storage_inventory.go | 342 +++++++++++++++ internal/core/service.go | 2 + internal/db/postgres/objects.go | 4 +- internal/db/sqlite/objects.go | 4 +- internal/db/sqlite/sqlite_test.go | 90 ++++ internal/repair/service.go | 84 ++-- internal/repair/service_test.go | 63 +++ internal/repair/types.go | 6 +- 20 files changed, 1507 insertions(+), 69 deletions(-) create mode 100644 internal/core/object_storage_inspect_test.go create mode 100644 internal/core/object_storage_inventory.go diff --git a/internal/api/drsapi/mutations.go b/internal/api/drsapi/mutations.go index 657a1f6c..db31be4d 100644 --- a/internal/api/drsapi/mutations.go +++ b/internal/api/drsapi/mutations.go @@ -134,7 +134,10 @@ func handleBulkDeleteObjectsFiber(om *core.ObjectManager) fiber.Handler { ids = append(ids, id) } - if err := om.BulkDeleteObjects(c.Context(), ids); err != nil { + opts := core.DeleteOptions{ + DeleteStorageData: body.DeleteStorageData != nil && *body.DeleteStorageData, + } + if err := om.BulkDeleteObjectsWithOptions(c.Context(), ids, opts); err != nil { return apiutil.HandleError(c, err) } return c.SendStatus(fiber.StatusNoContent) diff --git a/internal/api/internaldrs/inspect.go b/internal/api/internaldrs/inspect.go index f8ab4104..fedb2877 100644 --- a/internal/api/internaldrs/inspect.go +++ b/internal/api/internaldrs/inspect.go @@ -2,19 +2,52 @@ package internaldrs import ( "strings" + "time" + "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/api/apiutil" apimiddleware "github.com/calypr/syfon/internal/api/middleware" + "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/models" "github.com/gofiber/fiber/v3" ) type internalInspectObjectRequest struct { + ID string `json:"id,omitempty"` + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + Key string `json:"key,omitempty"` + Scheme string `json:"scheme,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` +} + +type internalInspectObjectsBulkRequest struct { + Items []internalInspectObjectRequest `json:"items"` +} + +type internalInspectProjectBucketRequest struct { Organization string `json:"organization,omitempty"` Project string `json:"project,omitempty"` - Key string `json:"key,omitempty"` - Scheme string `json:"scheme,omitempty"` - ObjectURL string `json:"object_url,omitempty"` + IncludeHead bool `json:"include_head,omitempty"` +} + +type internalInspectProjectRecordsRequest struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` +} + +type internalInspectProjectScopesRequest struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` +} + +type internalDeleteProjectBucketObjectsRequest struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + ObjectURLs []string `json:"object_urls"` } type internalInspectObjectResponse struct { @@ -29,6 +62,91 @@ type internalInspectObjectResponse struct { LastModTime string `json:"last_modified,omitempty"` } +type internalInspectObjectBulkResponse struct { + Items []internalInspectObjectBulkItem `json:"items"` +} + +type internalInspectProjectBucketResponse struct { + Items []internalInspectProjectBucketItem `json:"items"` +} + +type internalInspectProjectRecordsResponse struct { + Items []internalInspectProjectRecordItem `json:"items"` +} + +type internalInspectProjectScopesResponse struct { + Items []internalInspectProjectScopeItem `json:"items"` +} + +type internalDeleteProjectBucketObjectsResponse struct { + Items []internalDeleteProjectBucketObjectsItem `json:"items"` +} + +type internalInspectObjectBulkItem struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + Provider string `json:"provider,omitempty"` + Bucket string `json:"bucket,omitempty"` + Key string `json:"key,omitempty"` + Path string `json:"path,omitempty"` + Exists bool `json:"exists"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + MetaSHA256 string `json:"meta_sha256,omitempty"` + ETag string `json:"etag,omitempty"` + LastModTime string `json:"last_modified,omitempty"` + ValidationStatus string `json:"validation_status"` + SizeMatch *bool `json:"size_match,omitempty"` + SHA256Match *bool `json:"sha256_match,omitempty"` + ValidationMismatches []string `json:"validation_mismatches,omitempty"` +} + +type internalInspectProjectBucketItem struct { + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + MetaSHA256 string `json:"meta_sha256,omitempty"` + ETag string `json:"etag,omitempty"` + LastModTime string `json:"last_modified,omitempty"` +} + +type internalInspectProjectRecordItem struct { + ObjectID string `json:"object_id"` + Checksum string `json:"checksum"` + Organization string `json:"organization"` + Project string `json:"project"` + Size int64 `json:"size"` + CreatedTime string `json:"created_time,omitempty"` + UpdatedTime string `json:"updated_time,omitempty"` + AccessURLs []string `json:"access_urls"` + AccessMethods []internalProjectAccessMethod `json:"access_methods"` +} + +type internalProjectAccessMethod struct { + AccessID string `json:"access_id,omitempty"` + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Headers []string `json:"headers,omitempty"` +} + +type internalInspectProjectScopeItem struct { + Bucket string `json:"bucket"` + Organization string `json:"organization"` + ProjectID string `json:"project_id,omitempty"` + Path string `json:"path,omitempty"` +} + +type internalDeleteProjectBucketObjectsItem struct { + ObjectURL string `json:"object_url"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} + func handleInternalInspectObjectFiber(om *core.ObjectManager) fiber.Handler { return func(c fiber.Ctx) error { if apimiddleware.MissingGen3AuthHeader(c.Context()) { @@ -39,11 +157,14 @@ func handleInternalInspectObjectFiber(om *core.ObjectManager) fiber.Handler { return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) } resp, err := om.InspectStorageObject(c.Context(), core.InspectStorageRequest{ - Organization: strings.TrimSpace(req.Organization), - Project: strings.TrimSpace(req.Project), - Key: strings.TrimSpace(req.Key), - Scheme: strings.TrimSpace(req.Scheme), - ObjectURL: strings.TrimSpace(req.ObjectURL), + ID: strings.TrimSpace(req.ID), + Organization: strings.TrimSpace(req.Organization), + Project: strings.TrimSpace(req.Project), + Key: strings.TrimSpace(req.Key), + Scheme: strings.TrimSpace(req.Scheme), + ObjectURL: strings.TrimSpace(req.ObjectURL), + ExpectedSizeBytes: req.ExpectedSizeBytes, + ExpectedSHA256: strings.TrimSpace(req.ExpectedSHA256), }) if err != nil { return handleInspectStorageError(c, err) @@ -65,6 +186,180 @@ func handleInternalInspectObjectFiber(om *core.ObjectManager) fiber.Handler { } } +func handleInternalInspectObjectBulkFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectObjectsBulkRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + if len(req.Items) == 0 { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: items are required") + } + items := make([]core.InspectStorageRequest, 0, len(req.Items)) + for _, item := range req.Items { + items = append(items, core.InspectStorageRequest{ + ID: strings.TrimSpace(item.ID), + Organization: strings.TrimSpace(item.Organization), + Project: strings.TrimSpace(item.Project), + Key: strings.TrimSpace(item.Key), + Scheme: strings.TrimSpace(item.Scheme), + ObjectURL: strings.TrimSpace(item.ObjectURL), + ExpectedSizeBytes: item.ExpectedSizeBytes, + ExpectedSHA256: strings.TrimSpace(item.ExpectedSHA256), + }) + } + results := om.InspectStorageObjects(c.Context(), items) + out := internalInspectObjectBulkResponse{Items: make([]internalInspectObjectBulkItem, 0, len(results))} + for _, result := range results { + out.Items = append(out.Items, bulkInspectItemFromCore(result)) + } + return c.JSON(out) + } +} + +func handleInternalInspectProjectBucketFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectProjectBucketRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + items, err := om.ListProjectStorageObjects(c.Context(), strings.TrimSpace(req.Organization), strings.TrimSpace(req.Project), req.IncludeHead) + if err != nil { + return handleInspectStorageError(c, err) + } + out := internalInspectProjectBucketResponse{Items: make([]internalInspectProjectBucketItem, 0, len(items))} + for _, item := range items { + row := internalInspectProjectBucketItem{ + ObjectURL: item.ObjectURL, + Provider: item.Provider, + Bucket: item.Bucket, + Key: item.Key, + Path: item.Path, + SizeBytes: item.SizeBytes, + MetaSHA256: item.MetaSHA256, + ETag: item.ETag, + } + if !item.LastModTime.IsZero() { + row.LastModTime = item.LastModTime.Format(time.RFC3339) + } + out.Items = append(out.Items, row) + } + return c.JSON(out) + } +} + +func handleInternalInspectProjectRecordsFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectProjectRecordsRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + organization := strings.TrimSpace(req.Organization) + project := strings.TrimSpace(req.Project) + if organization == "" || project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + objects, err := om.ListObjectsByScope(c.Context(), organization, project, "read") + if err != nil { + return apiutil.HandleError(c, err) + } + out := internalInspectProjectRecordsResponse{Items: make([]internalInspectProjectRecordItem, 0, len(objects))} + for _, obj := range objects { + record, ok := projectRecordAuditItemFromObject(obj, organization, project) + if !ok { + continue + } + out.Items = append(out.Items, record) + } + return c.JSON(out) + } +} + +func handleInternalInspectProjectScopesFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectProjectScopesRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + organization := strings.TrimSpace(req.Organization) + project := strings.TrimSpace(req.Project) + if organization == "" || project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + scopes, err := om.ListBucketScopes(c.Context()) + if err != nil { + return apiutil.HandleError(c, err) + } + out := internalInspectProjectScopesResponse{Items: make([]internalInspectProjectScopeItem, 0)} + for _, scope := range scopes { + if !strings.EqualFold(strings.TrimSpace(scope.Organization), organization) { + continue + } + scopeProject := strings.TrimSpace(scope.ProjectID) + if scopeProject != "" && !strings.EqualFold(scopeProject, project) { + continue + } + if !bucketScopeAllowed(c.Context(), scope, "read") { + continue + } + row := internalInspectProjectScopeItem{ + Bucket: strings.TrimSpace(scope.Bucket), + Organization: organization, + ProjectID: scopeProject, + } + scheme := "s3" + if cred, err := om.GetS3Credential(c.Context(), scope.CredentialID); err == nil && cred != nil { + scheme = common.ProviderToScheme(cred.Provider) + } + row.Path = scheme + "://" + strings.TrimSpace(scope.Bucket) + if scope.PathPrefix != "" { + row.Path += "/" + strings.Trim(strings.TrimSpace(scope.PathPrefix), "/") + } + out.Items = append(out.Items, row) + } + return c.JSON(out) + } +} + +func handleInternalDeleteProjectBucketObjectsFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalDeleteProjectBucketObjectsRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + if len(req.ObjectURLs) == 0 { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: object_urls are required") + } + results := om.DeleteProjectStorageObjects(c.Context(), strings.TrimSpace(req.Organization), strings.TrimSpace(req.Project), req.ObjectURLs) + out := internalDeleteProjectBucketObjectsResponse{ + Items: make([]internalDeleteProjectBucketObjectsItem, 0, len(results)), + } + for _, result := range results { + out.Items = append(out.Items, internalDeleteProjectBucketObjectsItem{ + ObjectURL: result.ObjectURL, + Status: result.Status, + Error: result.Error, + }) + } + return c.JSON(out) + } +} + func handleInspectStorageError(c fiber.Ctx, err error) error { if inspectErr, ok := err.(*core.StorageInspectError); ok { switch inspectErr.Kind { @@ -72,9 +367,95 @@ func handleInspectStorageError(c fiber.Ctx, err error) error { return apiutil.Reject(c, fiber.StatusBadRequest, inspectErr.Error()) case core.StorageInspectScopeNotFound, core.StorageInspectCredentialMissing, core.StorageInspectObjectNotFound: return apiutil.Reject(c, fiber.StatusNotFound, inspectErr.Error()) + case core.StorageInspectBucketUnavailable: + return apiutil.Reject(c, fiber.StatusConflict, inspectErr.Error()) case core.StorageInspectPermissionDenied: return apiutil.Reject(c, fiber.StatusForbidden, inspectErr.Error()) } } return apiutil.HandleError(c, err) } + +func bulkInspectItemFromCore(result core.StorageProbeResult) internalInspectObjectBulkItem { + out := internalInspectObjectBulkItem{ + ID: result.ID, + ObjectURL: result.ObjectURL, + Provider: result.Provider, + Bucket: result.Bucket, + Key: result.Key, + Path: result.Path, + Exists: result.Exists, + Status: string(result.Status), + Error: result.Error, + ErrorKind: result.ErrorKind, + SizeBytes: result.SizeBytes, + MetaSHA256: result.MetaSHA256, + ETag: result.ETag, + ValidationStatus: string(result.ValidationStatus), + SizeMatch: result.SizeMatch, + SHA256Match: result.SHA256Match, + ValidationMismatches: append([]string(nil), result.ValidationMismatches...), + } + if !result.LastModTime.IsZero() { + out.LastModTime = result.LastModTime.Format(time.RFC3339) + } + return out +} + +func projectRecordAuditItemFromObject(obj models.InternalObject, organization, project string) (internalInspectProjectRecordItem, bool) { + checksum := primarySHA256Checksum(obj.Checksums) + if checksum == "" { + return internalInspectProjectRecordItem{}, false + } + accessURLs := make([]string, 0) + accessMethods := make([]internalProjectAccessMethod, 0) + if obj.AccessMethods != nil { + for _, method := range *obj.AccessMethods { + row := internalProjectAccessMethod{ + Type: strings.TrimSpace(string(method.Type)), + Headers: []string{}, + } + if method.AccessId != nil { + row.AccessID = strings.TrimSpace(*method.AccessId) + } + if method.AccessUrl != nil { + row.URL = strings.TrimSpace(method.AccessUrl.Url) + if row.URL != "" { + accessURLs = append(accessURLs, row.URL) + } + if method.AccessUrl.Headers != nil { + row.Headers = append([]string(nil), (*method.AccessUrl.Headers)...) + } + } + accessMethods = append(accessMethods, row) + } + } + item := internalInspectProjectRecordItem{ + ObjectID: strings.TrimSpace(obj.Id), + Checksum: checksum, + Organization: organization, + Project: project, + Size: obj.Size, + AccessURLs: accessURLs, + AccessMethods: accessMethods, + } + if !obj.CreatedTime.IsZero() { + item.CreatedTime = obj.CreatedTime.Format(time.RFC3339Nano) + } + if obj.UpdatedTime != nil && !obj.UpdatedTime.IsZero() { + item.UpdatedTime = obj.UpdatedTime.Format(time.RFC3339Nano) + } + return item, true +} + +func primarySHA256Checksum(checksums []drs.Checksum) string { + for _, checksum := range checksums { + if strings.EqualFold(strings.TrimSpace(checksum.Type), "sha256") { + value := strings.TrimSpace(strings.TrimPrefix(checksum.Checksum, "sha256:")) + if value != "" { + return value + } + } + } + return "" +} diff --git a/internal/api/internaldrs/inspect_routes_test.go b/internal/api/internaldrs/inspect_routes_test.go index 68d32b17..cbd25ad1 100644 --- a/internal/api/internaldrs/inspect_routes_test.go +++ b/internal/api/internaldrs/inspect_routes_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/core" "github.com/calypr/syfon/internal/models" "github.com/calypr/syfon/internal/testutils" @@ -113,3 +114,92 @@ func TestHandleInternalInspectObjectMalformedURL(t *testing.T) { t.Fatalf("expected 400, got %d body=%s", rr.Code, rr.Body.String()) } } + +func TestHandleInternalInspectProjectRecords(t *testing.T) { + body, _ := json.Marshal(internalInspectProjectRecordsRequest{Organization: "syfon", Project: "e2e"}) + name := "example.bin" + created := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + updated := time.Date(2026, 7, 1, 12, 30, 0, 0, time.UTC) + db := &testutils.MockDatabase{ + Objects: map[string]*drs.DrsObject{ + "obj-1": { + Id: "obj-1", + Name: &name, + Checksums: []drs.Checksum{ + {Type: "sha256", Checksum: "abc123"}, + }, + CreatedTime: created, + UpdatedTime: &updated, + Size: 17, + AccessMethods: &[]drs.AccessMethod{ + { + Type: "s3", + AccessId: ptr("acc-1"), + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket-a/prefix/example.bin"}, + }, + }, + }, + }, + ObjectAuthz: map[string]map[string][]string{ + "obj-1": {"syfon": {"e2e"}}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-records", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{"/organization/syfon/project/e2e": {"read": true}}) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectProjectRecordsResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Items) != 1 { + t.Fatalf("expected 1 item, got %d", len(resp.Items)) + } + item := resp.Items[0] + if item.ObjectID != "obj-1" || item.Checksum != "abc123" { + t.Fatalf("unexpected item identity: %+v", item) + } + if item.Organization != "syfon" || item.Project != "e2e" { + t.Fatalf("unexpected scope: %+v", item) + } + if len(item.AccessURLs) != 1 || item.AccessURLs[0] != "s3://bucket-a/prefix/example.bin" { + t.Fatalf("unexpected access urls: %+v", item.AccessURLs) + } +} + +func TestHandleInternalInspectProjectScopes(t *testing.T) { + body, _ := json.Marshal(internalInspectProjectScopesRequest{Organization: "syfon", Project: "e2e"}) + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|": {Organization: "syfon", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "program-root"}, + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-scopes", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon": {"read": true}, + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectProjectScopesResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Items) != 2 { + t.Fatalf("expected 2 items, got %d", len(resp.Items)) + } + if resp.Items[0].Bucket != "bucket-a" || !strings.HasPrefix(resp.Items[0].Path, "s3://bucket-a/") { + t.Fatalf("unexpected first scope: %+v", resp.Items[0]) + } +} diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go index 851980fe..64fc3781 100644 --- a/internal/api/internaldrs/repair_cleanup.go +++ b/internal/api/internaldrs/repair_cleanup.go @@ -2,7 +2,10 @@ package internaldrs import ( "context" + "encoding/json" + "errors" "fmt" + "net/http" "strings" "time" @@ -11,6 +14,7 @@ import ( clientinternalapi "github.com/calypr/syfon/apigen/client/internalapi" serverdrs "github.com/calypr/syfon/apigen/server/drs" serverinternalapi "github.com/calypr/syfon/apigen/server/internalapi" + "github.com/calypr/syfon/client/request" sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/api/apiutil" apimiddleware "github.com/calypr/syfon/internal/api/middleware" @@ -37,7 +41,7 @@ func authorizeStorageCleanupScope(ctx context.Context, organization, project str } func handleInternalScopeRepairAuditFiber(om *core.ObjectManager) fiber.Handler { - svc := repair.NewService(scopeRepairIndexAdapter{om: om}, scopeRepairBucketsAdapter{om: om}, nil) + svc := repair.NewService(scopeRepairIndexAdapter{om: om}, scopeRepairBucketsAdapter{om: om}, &srvRequester{om: om}) return func(c fiber.Ctx) error { if apimiddleware.MissingGen3AuthHeader(c.Context()) { return c.SendStatus(fiber.StatusUnauthorized) @@ -48,7 +52,7 @@ func handleInternalScopeRepairAuditFiber(om *core.ObjectManager) fiber.Handler { } req.Organization = strings.TrimSpace(req.Organization) req.Project = strings.TrimSpace(req.Project) - req.CheckStorage = false + req.CheckStorage = true if req.Organization == "" || req.Project == "" { return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") } @@ -64,7 +68,7 @@ func handleInternalScopeRepairAuditFiber(om *core.ObjectManager) fiber.Handler { } func handleInternalScopeRepairApplyFiber(om *core.ObjectManager) fiber.Handler { - svc := repair.NewService(scopeRepairIndexAdapter{om: om}, scopeRepairBucketsAdapter{om: om}, nil) + svc := repair.NewService(scopeRepairIndexAdapter{om: om}, scopeRepairBucketsAdapter{om: om}, &srvRequester{om: om}) return func(c fiber.Ctx) error { if apimiddleware.MissingGen3AuthHeader(c.Context()) { return c.SendStatus(fiber.StatusUnauthorized) @@ -75,7 +79,7 @@ func handleInternalScopeRepairApplyFiber(om *core.ObjectManager) fiber.Handler { } req.Organization = strings.TrimSpace(req.Organization) req.Project = strings.TrimSpace(req.Project) - req.CheckStorage = false + req.CheckStorage = true if req.Organization == "" || req.Project == "" { return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") } @@ -98,7 +102,7 @@ type scopeRepairIndexAdapter struct { } func (a scopeRepairIndexAdapter) List(ctx context.Context, opts repair.ListRecordsOptions) (clientinternalapi.ListRecordsResponse, error) { - ids, err := a.om.ListObjectIDsPageByScope(ctx, "", "", "read", strings.TrimSpace(opts.Start), opts.Limit, 0) + ids, err := a.om.ListObjectIDsPageByScope(ctx, strings.TrimSpace(opts.Organization), strings.TrimSpace(opts.Project), "read", strings.TrimSpace(opts.Start), opts.Limit, 0) if err != nil { return clientinternalapi.ListRecordsResponse{}, err } @@ -284,3 +288,41 @@ func serverAccessMethodsFromClient(in *[]clientdrs.AccessMethod) *[]serverdrs.Ac } return &out } + +type srvRequester struct { + om *core.ObjectManager +} + +func (r *srvRequester) Do(ctx context.Context, method, path string, body, out any, opts ...request.RequestOption) error { + if path == "/data/inspect" { + data, err := json.Marshal(body) + if err != nil { + return err + } + var inspectReq core.InspectStorageRequest + if err := json.Unmarshal(data, &inspectReq); err != nil { + return err + } + meta, err := r.om.InspectStorageObject(ctx, inspectReq) + if err != nil { + var inspectErr *core.StorageInspectError + if errors.As(err, &inspectErr) { + if inspectErr.Kind == core.StorageInspectObjectNotFound { + return &request.ResponseError{ + Method: method, + URL: path, + Status: http.StatusNotFound, + Body: "storage object not found", + } + } + } + return err + } + if out != nil { + respData, _ := json.Marshal(map[string]string{"object_url": meta.ObjectURL}) + _ = json.Unmarshal(respData, out) + } + return nil + } + return fmt.Errorf("unsupported service call: %s", path) +} diff --git a/internal/api/internaldrs/transfer.go b/internal/api/internaldrs/transfer.go index 6d7cde18..ff269056 100644 --- a/internal/api/internaldrs/transfer.go +++ b/internal/api/internaldrs/transfer.go @@ -32,6 +32,11 @@ func registerInternalTransferRoutes(router fiber.Router, om *core.ObjectManager) router.Post(common.RouteInternalUpload, handleInternalUploadBlankFiber(om)) router.Get(routeutil.FiberPath(common.RouteInternalUploadURL), handleInternalUploadURLFiber(om)) router.Post(common.RouteInternalInspectObject, handleInternalInspectObjectFiber(om)) + router.Post(common.RouteInternalInspectObjectBulk, handleInternalInspectObjectBulkFiber(om)) + router.Post(common.RouteInternalInspectProjectBucket, handleInternalInspectProjectBucketFiber(om)) + router.Post(common.RouteInternalInspectProjectRecords, handleInternalInspectProjectRecordsFiber(om)) + router.Post(common.RouteInternalInspectProjectScopes, handleInternalInspectProjectScopesFiber(om)) + router.Post(common.RouteInternalDeleteProjectBucketObjects, handleInternalDeleteProjectBucketObjectsFiber(om)) router.Post(common.RouteInternalUploadBulk, handleInternalUploadBulkFiber(om)) router.Post(common.RouteInternalMultipartInit, handleInternalMultipartInitFiber(om)) router.Post(common.RouteInternalMultipartUpload, handleInternalMultipartUploadFiber(om)) diff --git a/internal/common/routes.go b/internal/common/routes.go index 8b875b5c..20ddfe2f 100644 --- a/internal/common/routes.go +++ b/internal/common/routes.go @@ -1,21 +1,26 @@ package common const ( - RouteInternalDownload = "/data/download/{file_id}" - RouteInternalDownloadPart = "/data/download/{file_id}/part" - RouteInternalUpload = "/data/upload" - RouteInternalUploadURL = "/data/upload/{file_id}" - RouteInternalUploadBulk = "/data/upload/bulk" - RouteInternalMultipartInit = "/data/multipart/init" - RouteInternalMultipartUpload = "/data/multipart/upload" - RouteInternalMultipartComplete = "/data/multipart/complete" - RouteInternalInspectObject = "/data/inspect" - RouteInternalBuckets = "/data/buckets" - RouteInternalBucketDetail = "/data/buckets/{bucket}" - RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" - RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" - RouteInternalRepairScopeAudit = "/data/repair/project-scope/audit" - RouteInternalRepairScopeApply = "/data/repair/project-scope/apply" + RouteInternalDownload = "/data/download/{file_id}" + RouteInternalDownloadPart = "/data/download/{file_id}/part" + RouteInternalUpload = "/data/upload" + RouteInternalUploadURL = "/data/upload/{file_id}" + RouteInternalUploadBulk = "/data/upload/bulk" + RouteInternalMultipartInit = "/data/multipart/init" + RouteInternalMultipartUpload = "/data/multipart/upload" + RouteInternalMultipartComplete = "/data/multipart/complete" + RouteInternalInspectObject = "/data/inspect" + RouteInternalInspectObjectBulk = "/data/inspect/bulk" + RouteInternalInspectProjectBucket = "/data/inspect/project-bucket" + RouteInternalInspectProjectRecords = "/data/inspect/project-records" + RouteInternalInspectProjectScopes = "/data/inspect/project-scopes" + RouteInternalDeleteProjectBucketObjects = "/data/inspect/project-bucket/delete" + RouteInternalBuckets = "/data/buckets" + RouteInternalBucketDetail = "/data/buckets/{bucket}" + RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" + RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" + RouteInternalRepairScopeAudit = "/data/repair/project-scope/audit" + RouteInternalRepairScopeApply = "/data/repair/project-scope/apply" RouteInternalIndex = "/index" RouteInternalIndexDetail = "/index/{id}" diff --git a/internal/common/util.go b/internal/common/util.go index d251cc39..99f48528 100644 --- a/internal/common/util.go +++ b/internal/common/util.go @@ -2,9 +2,24 @@ package common import ( "fmt" + "path/filepath" "strings" ) +// CleanToBasename extracts the basename from a path (handling both windows and unix separators). +func CleanToBasename(name string) string { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return "" + } + trimmed = strings.ReplaceAll(trimmed, "\\", "/") + base := filepath.Base(trimmed) + if base == "." || base == "/" || base == "" { + base = trimmed + } + return base +} + // SchemeFromURL extracts the scheme from a URL string. diff --git a/internal/common/util_test.go b/internal/common/util_test.go index 5d463ead..92cf10ef 100644 --- a/internal/common/util_test.go +++ b/internal/common/util_test.go @@ -51,3 +51,27 @@ func TestBucketToURL(t *testing.T) { }) } } + +func TestCleanToBasename(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + {"Empty", "", ""}, + {"Spaces", " ", ""}, + {"Unix path", "/foo/bar/baz.txt", "baz.txt"}, + {"Windows path", `C:\foo\bar\baz.txt`, "baz.txt"}, + {"Relative path", "foo/bar.txt", "bar.txt"}, + {"No path", "baz.txt", "baz.txt"}, + {"Slash end", "foo/bar/", "bar"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CleanToBasename(tt.input); got != tt.expected { + t.Errorf("CleanToBasename(%q) = %v, want %v", tt.input, got, tt.expected) + } + }) + } +} diff --git a/internal/core/object_mutations.go b/internal/core/object_mutations.go index fe8bc417..3c26a0a0 100644 --- a/internal/core/object_mutations.go +++ b/internal/core/object_mutations.go @@ -92,6 +92,10 @@ func (m *ObjectManager) DeleteObjectWithOptions(ctx context.Context, id string, } func (m *ObjectManager) BulkDeleteObjects(ctx context.Context, ids []string) error { + return m.BulkDeleteObjectsWithOptions(ctx, ids, DeleteOptions{}) +} + +func (m *ObjectManager) BulkDeleteObjectsWithOptions(ctx context.Context, ids []string, opts DeleteOptions) error { toDelete, err := m.deletableObjectIDs(ctx, ids) if err != nil { return err @@ -99,6 +103,17 @@ func (m *ObjectManager) BulkDeleteObjects(ctx context.Context, ids []string) err if len(toDelete) == 0 { return nil } + if opts.DeleteStorageData { + objects, err := m.db.GetBulkObjects(ctx, toDelete) + if err != nil { + return err + } + for i := range objects { + if err := m.deleteObjectStorage(ctx, &objects[i]); err != nil { + return err + } + } + } return m.db.BulkDeleteObjects(ctx, toDelete) } diff --git a/internal/core/object_reads.go b/internal/core/object_reads.go index 8e520e42..b310b01a 100644 --- a/internal/core/object_reads.go +++ b/internal/core/object_reads.go @@ -304,6 +304,30 @@ func (m *ObjectManager) ListObjectIDsByScope(ctx context.Context, organization, return out, nil } +func (m *ObjectManager) ListObjectsByScope(ctx context.Context, organization, project, requiredMethod string) ([]models.InternalObject, error) { + if strings.TrimSpace(organization) == "" && strings.EqualFold(strings.TrimSpace(requiredMethod), objectMethodRead) { + if ids, ok, err := m.listReadableObjectIDs(ctx); ok { + if err != nil { + return nil, err + } + objects, err := m.db.GetBulkObjects(ctx, ids) + if err != nil { + return nil, err + } + return m.filterObjectsByMethod(ctx, objects, requiredMethod), nil + } + } + ids, err := m.db.ListObjectIDsByScope(ctx, organization, project) + if err != nil { + return nil, err + } + objects, err := m.db.GetBulkObjects(ctx, ids) + if err != nil { + return nil, err + } + return m.filterObjectsByMethod(ctx, objects, requiredMethod), nil +} + func objectHasAccessURL(obj *models.InternalObject, objectURL string) bool { if obj == nil || obj.AccessMethods == nil { return false diff --git a/internal/core/object_storage_inspect.go b/internal/core/object_storage_inspect.go index 94dd4cfa..eae8c6e0 100644 --- a/internal/core/object_storage_inspect.go +++ b/internal/core/object_storage_inspect.go @@ -22,11 +22,14 @@ import ( ) type InspectStorageRequest struct { - Organization string - Project string - Key string - Scheme string - ObjectURL string + ID string + Organization string + Project string + Key string + Scheme string + ObjectURL string + ExpectedSizeBytes *int64 + ExpectedSHA256 string } type StorageObjectMetadata struct { @@ -41,6 +44,47 @@ type StorageObjectMetadata struct { LastModTime time.Time } +type StorageProbeStatus string + +const ( + StorageProbeStatusPresent StorageProbeStatus = "present" + StorageProbeStatusNotFound StorageProbeStatus = "not_found" + StorageProbeStatusForbidden StorageProbeStatus = "forbidden" + StorageProbeStatusInvalid StorageProbeStatus = "invalid" + StorageProbeStatusUnsupported StorageProbeStatus = "unsupported" + StorageProbeStatusError StorageProbeStatus = "error" +) + +type StorageValidationStatus string + +const ( + StorageValidationNotRequested StorageValidationStatus = "not_requested" + StorageValidationMatched StorageValidationStatus = "matched" + StorageValidationMismatched StorageValidationStatus = "mismatched" + StorageValidationUnverifiable StorageValidationStatus = "unverifiable" +) + +type StorageProbeResult struct { + ID string + ObjectURL string + Provider string + Bucket string + Key string + Path string + Exists bool + Status StorageProbeStatus + Error string + ErrorKind string + SizeBytes *int64 + MetaSHA256 string + ETag string + LastModTime time.Time + ValidationStatus StorageValidationStatus + SizeMatch *bool + SHA256Match *bool + ValidationMismatches []string +} + type StorageInspectErrorKind string const ( @@ -49,9 +93,12 @@ const ( StorageInspectCredentialMissing StorageInspectErrorKind = "credential_missing" StorageInspectPermissionDenied StorageInspectErrorKind = "permission_denied" StorageInspectObjectNotFound StorageInspectErrorKind = "object_not_found" + StorageInspectBucketUnavailable StorageInspectErrorKind = "bucket_unavailable" StorageInspectUnsupported StorageInspectErrorKind = "unsupported" ) +const maxStorageInspectWorkers = 16 + type StorageInspectError struct { Kind StorageInspectErrorKind Message string @@ -68,6 +115,7 @@ type storageInspectRequestCache struct { mu sync.Mutex credentials map[string]storageInspectCredentialCacheEntry + s3Clients map[string]*awss3.Client visible map[string]VisibleBucket visibleErr error visibleLoaded bool @@ -83,7 +131,15 @@ func (e *StorageInspectError) Error() string { return fmt.Sprintf("storage inspect failed: %s", e.Kind) } - +func WithStorageInspectCache(ctx context.Context) context.Context { + if storageInspectCacheFromContext(ctx) != nil { + return ctx + } + return context.WithValue(ctx, storageInspectCacheKey, &storageInspectRequestCache{ + credentials: map[string]storageInspectCredentialCacheEntry{}, + s3Clients: map[string]*awss3.Client{}, + }) +} func storageInspectCacheFromContext(ctx context.Context) *storageInspectRequestCache { cache, _ := ctx.Value(storageInspectCacheKey).(*storageInspectRequestCache) @@ -105,6 +161,188 @@ func (m *ObjectManager) InspectStorageObject(ctx context.Context, req InspectSto return m.inspectScopedStorageObject(ctx, req) } +func (m *ObjectManager) InspectStorageObjects(ctx context.Context, items []InspectStorageRequest) []StorageProbeResult { + ctx = WithStorageInspectCache(ctx) + if len(items) == 0 { + return []StorageProbeResult{} + } + results := make([]StorageProbeResult, len(items)) + workers := len(items) + if workers > maxStorageInspectWorkers { + workers = maxStorageInspectWorkers + } + type probeWork struct { + index int + req InspectStorageRequest + } + workCh := make(chan probeWork) + cache := newStorageProbeResultCache() + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for work := range workCh { + results[work.index] = cache.probe(ctx, m, work.req) + } + }() + } + for i, item := range items { + workCh <- probeWork{index: i, req: item} + } + close(workCh) + wg.Wait() + return results +} + +type storageProbeResultCache struct { + mu sync.Mutex + results map[string]StorageProbeResult +} + +func newStorageProbeResultCache() *storageProbeResultCache { + return &storageProbeResultCache{results: map[string]StorageProbeResult{}} +} + +func (c *storageProbeResultCache) probe(ctx context.Context, m *ObjectManager, req InspectStorageRequest) StorageProbeResult { + key := storageProbeCacheKey(req) + c.mu.Lock() + if result, ok := c.results[key]; ok { + c.mu.Unlock() + return cloneStorageProbeResultForRequest(result, req) + } + c.mu.Unlock() + + result := m.inspectStorageProbe(ctx, req) + + c.mu.Lock() + c.results[key] = result + c.mu.Unlock() + return result +} + +func storageProbeCacheKey(req InspectStorageRequest) string { + return strings.TrimSpace(req.ObjectURL) + "|" + + strings.TrimSpace(req.Organization) + "|" + + strings.TrimSpace(req.Project) + "|" + + strings.TrimSpace(req.Key) + "|" + + strings.TrimSpace(req.Scheme) + "|" + + normalizeExpectedSize(req.ExpectedSizeBytes) + "|" + + strings.ToLower(strings.TrimSpace(strings.TrimPrefix(req.ExpectedSHA256, "sha256:"))) +} + +func normalizeExpectedSize(v *int64) string { + if v == nil { + return "" + } + return fmt.Sprintf("%d", *v) +} + +func cloneStorageProbeResultForRequest(result StorageProbeResult, req InspectStorageRequest) StorageProbeResult { + clone := result + clone.ID = strings.TrimSpace(req.ID) + if clone.ID == "" { + clone.ID = strings.TrimSpace(result.ID) + } + if clone.ObjectURL == "" { + clone.ObjectURL = strings.TrimSpace(req.ObjectURL) + } + clone.ValidationMismatches = append([]string(nil), result.ValidationMismatches...) + return clone +} + +func (m *ObjectManager) inspectStorageProbe(ctx context.Context, req InspectStorageRequest) StorageProbeResult { + result := StorageProbeResult{ + ID: strings.TrimSpace(req.ID), + ObjectURL: strings.TrimSpace(req.ObjectURL), + Status: StorageProbeStatusError, + ValidationStatus: StorageValidationNotRequested, + } + meta, err := m.InspectStorageObject(ctx, req) + if err != nil { + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = strings.TrimSpace(err.Error()) + result.Exists = false + result.ValidationStatus = storageValidationStatusForError(req) + return result + } + result.ObjectURL = strings.TrimSpace(meta.ObjectURL) + result.Provider = strings.TrimSpace(meta.Provider) + result.Bucket = strings.TrimSpace(meta.Bucket) + result.Key = strings.TrimSpace(meta.Key) + result.Path = strings.TrimSpace(meta.Path) + result.Exists = true + result.Status = StorageProbeStatusPresent + result.SizeBytes = ptrInt64(meta.SizeBytes) + result.MetaSHA256 = strings.TrimSpace(meta.MetaSHA256) + result.ETag = strings.TrimSpace(meta.ETag) + result.LastModTime = meta.LastModTime + result.ValidationStatus, result.SizeMatch, result.SHA256Match, result.ValidationMismatches = validateStorageProbe(req, meta) + return result +} + +func classifyStorageProbeError(err error) (StorageProbeStatus, string) { + var inspectErr *StorageInspectError + if errors.As(err, &inspectErr) { + switch inspectErr.Kind { + case StorageInspectObjectNotFound: + return StorageProbeStatusNotFound, string(inspectErr.Kind) + case StorageInspectPermissionDenied, StorageInspectBucketUnavailable: + return StorageProbeStatusForbidden, string(inspectErr.Kind) + case StorageInspectInvalidInput, StorageInspectScopeNotFound, StorageInspectCredentialMissing: + return StorageProbeStatusInvalid, string(inspectErr.Kind) + case StorageInspectUnsupported: + return StorageProbeStatusUnsupported, string(inspectErr.Kind) + default: + return StorageProbeStatusError, string(inspectErr.Kind) + } + } + return StorageProbeStatusError, "error" +} + +func storageValidationStatusForError(req InspectStorageRequest) StorageValidationStatus { + if req.ExpectedSizeBytes == nil && strings.TrimSpace(req.ExpectedSHA256) == "" { + return StorageValidationNotRequested + } + return StorageValidationUnverifiable +} + +func validateStorageProbe(req InspectStorageRequest, meta *StorageObjectMetadata) (StorageValidationStatus, *bool, *bool, []string) { + if req.ExpectedSizeBytes == nil && strings.TrimSpace(req.ExpectedSHA256) == "" { + return StorageValidationNotRequested, nil, nil, nil + } + mismatches := make([]string, 0, 2) + var sizeMatch *bool + if req.ExpectedSizeBytes != nil { + matched := meta.SizeBytes == *req.ExpectedSizeBytes + sizeMatch = &matched + if !matched { + mismatches = append(mismatches, "size_mismatch") + } + } + var shaMatch *bool + expectedSHA := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(req.ExpectedSHA256, "sha256:"))) + if expectedSHA != "" { + if strings.TrimSpace(meta.MetaSHA256) == "" { + return StorageValidationUnverifiable, sizeMatch, nil, append(mismatches, "missing_remote_sha256") + } + matched := strings.EqualFold(strings.TrimSpace(meta.MetaSHA256), expectedSHA) + shaMatch = &matched + if !matched { + mismatches = append(mismatches, "sha256_mismatch") + } + } + if len(mismatches) > 0 { + return StorageValidationMismatched, sizeMatch, shaMatch, mismatches + } + return StorageValidationMatched, sizeMatch, shaMatch, nil +} + +func ptrInt64(v int64) *int64 { + copy := v + return © +} + func (m *ObjectManager) inspectScopedStorageObject(ctx context.Context, req InspectStorageRequest) (*StorageObjectMetadata, error) { organization := strings.TrimSpace(req.Organization) project := strings.TrimSpace(req.Project) @@ -297,6 +535,22 @@ func (c *storageInspectRequestCache) setVisible(visible map[string]VisibleBucket c.visibleLoaded = true } +func (c *storageInspectRequestCache) getS3Client(key string) (*awss3.Client, bool) { + c.mu.Lock() + defer c.mu.Unlock() + client, ok := c.s3Clients[key] + return client, ok +} + +func (c *storageInspectRequestCache) setS3Client(key string, client *awss3.Client) { + if client == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.s3Clients[key] = client +} + func cloneVisibleBuckets(in map[string]VisibleBucket) map[string]VisibleBucket { if in == nil { return nil @@ -313,7 +567,13 @@ func cloneVisibleBuckets(in map[string]VisibleBucket) map[string]VisibleBucket { return out } -func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, bucket string, key string) (*StorageObjectMetadata, error) { +func s3ClientFromContext(ctx context.Context, cred models.S3Credential) (*awss3.Client, error) { + cacheKey := s3ClientCacheKey(cred) + if cache := storageInspectCacheFromContext(ctx); cache != nil { + if client, ok := cache.getS3Client(cacheKey); ok { + return client, nil + } + } loadOpts := []func(*awsconfig.LoadOptions) error{ awsconfig.WithRegion(cred.Region), awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cred.AccessKey, cred.SecretKey, "")), @@ -337,6 +597,26 @@ func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, buc o.UsePathStyle = true } }) + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setS3Client(cacheKey, client) + } + return client, nil +} + +func s3ClientCacheKey(cred models.S3Credential) string { + return strings.ToLower(strings.TrimSpace(cred.Provider)) + "|" + + strings.TrimSpace(cred.Endpoint) + "|" + + strings.TrimSpace(cred.Region) + "|" + + strings.TrimSpace(cred.AccessKey) + "|" + + strings.TrimSpace(cred.Bucket) + "|" + + strings.TrimSpace(cred.CredentialID) +} + +func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, bucket string, key string) (*StorageObjectMetadata, error) { + client, err := s3ClientFromContext(ctx, cred) + if err != nil { + return nil, err + } out, err := client.HeadObject(ctx, &awss3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), @@ -346,7 +626,7 @@ func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, buc if errors.As(err, &apiErr) { switch strings.ToLower(strings.TrimSpace(apiErr.ErrorCode())) { case "forbidden", "accessdenied", "permissiondenied": - return nil, &StorageInspectError{Kind: StorageInspectPermissionDenied, Message: fmt.Sprintf("provider denied access to s3://%s/%s", bucket, key)} + return nil, &StorageInspectError{Kind: StorageInspectBucketUnavailable, Message: fmt.Sprintf("provider rejected object probe for s3://%s/%s; mapped bucket target may be missing or inaccessible", bucket, key)} case "notfound", "nosuchkey", "nosuchbucket": return nil, &StorageInspectError{Kind: StorageInspectObjectNotFound, Message: fmt.Sprintf("provider could not find s3://%s/%s", bucket, key)} } diff --git a/internal/core/object_storage_inspect_test.go b/internal/core/object_storage_inspect_test.go new file mode 100644 index 00000000..e443f312 --- /dev/null +++ b/internal/core/object_storage_inspect_test.go @@ -0,0 +1,23 @@ +package core + +import "testing" + +func TestClassifyProbeErrorCredentialMissingIsNotBucketMiss(t *testing.T) { + status, kind := classifyStorageProbeError(&StorageInspectError{ + Kind: StorageInspectCredentialMissing, + Message: `no stored bucket credential found for bucket "missing"`, + }) + if status != StorageProbeStatusInvalid || kind != "credential_missing" { + t.Fatalf("expected credential_missing to classify as invalid status, got status=%q kind=%q", status, kind) + } +} + +func TestClassifyProbeErrorObjectNotFoundRemainsBucketMiss(t *testing.T) { + status, kind := classifyStorageProbeError(&StorageInspectError{ + Kind: StorageInspectObjectNotFound, + Message: "provider could not find s3://bucket/key", + }) + if status != StorageProbeStatusNotFound || kind != "object_not_found" { + t.Fatalf("expected object_not_found to classify as not_found, got status=%q kind=%q", status, kind) + } +} diff --git a/internal/core/object_storage_inventory.go b/internal/core/object_storage_inventory.go new file mode 100644 index 00000000..a7354a37 --- /dev/null +++ b/internal/core/object_storage_inventory.go @@ -0,0 +1,342 @@ +package core + +import ( + "context" + "errors" + "fmt" + "path" + "sort" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + syfoncommon "github.com/calypr/syfon/common" + "github.com/calypr/syfon/internal/authz" + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/models" +) + +type StorageBucketObject struct { + ObjectURL string + Provider string + Bucket string + Key string + Path string + SizeBytes int64 + MetaSHA256 string + ETag string + LastModTime time.Time +} + +type resolvedStorageScopeTarget struct { + provider string + bucket string + prefix string + cred models.S3Credential +} + +type ProjectStorageDeleteResult struct { + ObjectURL string + Status string + Error string +} + +func (m *ObjectManager) SetS3PrefixLister(fn func(context.Context, models.S3Credential, string, string, bool) ([]StorageBucketObject, error)) { + if fn == nil { + m.listS3Prefix = defaultS3PrefixLister + return + } + m.listS3Prefix = fn +} + +func (m *ObjectManager) ListProjectStorageObjects(ctx context.Context, organization, project string, includeHead bool) ([]StorageBucketObject, error) { + ctx = WithStorageInspectCache(ctx) + target, err := m.resolveProjectStorageScopeTarget(ctx, organization, project) + if err != nil { + return nil, err + } + items, err := m.listS3Prefix(ctx, target.cred, target.bucket, target.prefix, includeHead) + if err != nil { + return nil, err + } + out := make([]StorageBucketObject, 0, len(items)) + for _, item := range items { + entry := item + entry.Provider = common.S3Provider + entry.Bucket = target.bucket + entry.Key = strings.Trim(strings.TrimSpace(entry.Key), "/") + entry.ObjectURL = common.BucketToURL(target.bucket, entry.Key) + if strings.TrimSpace(entry.Path) == "" { + entry.Path = path.Base(entry.Key) + } + out = append(out, entry) + } + sort.Slice(out, func(i, j int) bool { + return out[i].Key < out[j].Key + }) + return out, nil +} + +func (m *ObjectManager) resolveProjectStorageScopeTarget(ctx context.Context, organization, project string) (*resolvedStorageScopeTarget, error) { + return m.resolveProjectStorageScopeTargetForMethod(ctx, organization, project, objectMethodRead) +} + +func (m *ObjectManager) resolveProjectStorageScopeTargetForMethod(ctx context.Context, organization, project, method string) (*resolvedStorageScopeTarget, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" { + return nil, &StorageInspectError{Kind: StorageInspectInvalidInput, Message: "organization is required"} + } + resource, err := syfoncommon.ResourcePath(organization, project) + if err != nil { + return nil, &StorageInspectError{Kind: StorageInspectInvalidInput, Message: err.Error()} + } + if authz.IsAuthzEnforced(ctx) && !authz.HasMethodAccess(ctx, method, []string{resource}) { + return nil, &common.AuthorizationError{Method: method, Resources: []string{resource}} + } + + scopes := make([]models.BucketScope, 0, 2) + if scope, found, err := m.lookupBucketScope(ctx, organization, ""); err != nil { + return nil, err + } else if found { + scopes = append(scopes, scope) + } + if project != "" { + if scope, found, err := m.lookupBucketScope(ctx, organization, project); err != nil { + return nil, err + } else if found { + scopes = append(scopes, scope) + } + } + if len(scopes) == 0 { + if project != "" { + return nil, &StorageInspectError{Kind: StorageInspectScopeNotFound, Message: fmt.Sprintf("no bucket scope configured for organization %q project %q", organization, project)} + } + return nil, &StorageInspectError{Kind: StorageInspectScopeNotFound, Message: fmt.Sprintf("no bucket scope configured for organization %q", organization)} + } + + bucket := "" + for _, scope := range scopes { + if strings.TrimSpace(scope.Bucket) != "" { + bucket = strings.TrimSpace(scope.Bucket) + } + } + if bucket == "" { + return nil, &StorageInspectError{Kind: StorageInspectInvalidInput, Message: fmt.Sprintf("unable to resolve scoped storage bucket for organization %q project %q", organization, project)} + } + cred, err := m.credentialForBucket(ctx, bucket) + if err != nil { + return nil, err + } + if common.NormalizeProvider(cred.Provider, common.S3Provider) != common.S3Provider { + return nil, &StorageInspectError{Kind: StorageInspectUnsupported, Message: fmt.Sprintf("provider %q is not supported for scoped bucket listing", cred.Provider)} + } + return &resolvedStorageScopeTarget{ + provider: common.S3Provider, + bucket: bucket, + prefix: strings.Join(normalizedScopePrefixes(scopes), "/"), + cred: *cred, + }, nil +} + +func (m *ObjectManager) DeleteProjectStorageObjects(ctx context.Context, organization, project string, objectURLs []string) []ProjectStorageDeleteResult { + ctx = WithStorageInspectCache(ctx) + normalized := uniqueStorageObjectURLs(objectURLs) + if len(normalized) == 0 { + return []ProjectStorageDeleteResult{} + } + results := make([]ProjectStorageDeleteResult, 0, len(normalized)) + target, err := m.resolveProjectStorageScopeTargetForMethod(ctx, organization, project, objectMethodDelete) + if err != nil { + for _, objectURL := range normalized { + results = append(results, ProjectStorageDeleteResult{ + ObjectURL: objectURL, + Status: "error", + Error: err.Error(), + }) + } + return results + } + for _, objectURL := range normalized { + result := ProjectStorageDeleteResult{ + ObjectURL: objectURL, + Status: "deleted", + } + storageTarget, ok, targetErr := m.storageTargetFromURL(ctx, objectURL) + switch { + case targetErr != nil: + result.Status = "error" + result.Error = targetErr.Error() + case !ok: + result.Status = "invalid" + result.Error = "object_url must resolve to a deletable storage target" + case !projectStorageTargetAllowed(storageTarget, target): + result.Status = "forbidden" + result.Error = fmt.Sprintf("object_url %q is outside configured project bucket scope", objectURL) + default: + err := m.deleteStorageTarget(ctx, storageTarget) + if err == nil { + break + } + result.Status = "error" + result.Error = err.Error() + } + results = append(results, result) + } + return results +} + +func projectStorageTargetAllowed(candidate storageTarget, target *resolvedStorageScopeTarget) bool { + if target == nil { + return false + } + if common.NormalizeProvider(candidate.provider, common.S3Provider) != common.S3Provider { + return false + } + if strings.TrimSpace(candidate.bucket) != strings.TrimSpace(target.bucket) { + return false + } + return projectStorageKeyWithinPrefix(candidate.key, target.prefix) +} + +func projectStorageKeyWithinPrefix(key, prefix string) bool { + cleanKey := strings.Trim(strings.TrimSpace(key), "/") + cleanPrefix := strings.Trim(strings.TrimSpace(prefix), "/") + if cleanPrefix == "" { + return cleanKey != "" + } + return cleanKey == cleanPrefix || strings.HasPrefix(cleanKey, cleanPrefix+"/") +} + +func uniqueStorageObjectURLs(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket string, prefix string, includeHead bool) ([]StorageBucketObject, error) { + client, err := s3ClientFromContext(ctx, cred) + if err != nil { + return nil, err + } + input := &awss3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + } + if trimmedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); trimmedPrefix != "" { + input.Prefix = aws.String(trimmedPrefix + "/") + } + paginator := awss3.NewListObjectsV2Paginator(client, input) + out := make([]StorageBucketObject, 0) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return nil, classifyS3ListError(bucket, prefix, err) + } + for _, item := range page.Contents { + key := strings.Trim(strings.TrimSpace(aws.ToString(item.Key)), "/") + if key == "" { + continue + } + var size int64 + if item.Size != nil { + size = *item.Size + } + lastMod := time.Time{} + if item.LastModified != nil { + lastMod = *item.LastModified + } + out = append(out, StorageBucketObject{ + Provider: common.S3Provider, + Bucket: bucket, + Key: key, + ObjectURL: common.BucketToURL(bucket, key), + Path: path.Base(key), + SizeBytes: size, + ETag: strings.Trim(strings.TrimSpace(aws.ToString(item.ETag)), "\""), + LastModTime: lastMod, + }) + } + } + if !includeHead || len(out) == 0 { + return out, nil + } + + workers := len(out) + if workers > maxStorageInspectWorkers { + workers = maxStorageInspectWorkers + } + if workers == 0 { + return out, nil + } + indexCh := make(chan int) + var wg sync.WaitGroup + var firstErr error + var errMu sync.Mutex + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for index := range indexCh { + meta, err := defaultS3ObjectInspector(ctx, cred, bucket, out[index].Key) + if err != nil { + errMu.Lock() + if firstErr == nil { + firstErr = err + } + errMu.Unlock() + continue + } + out[index].MetaSHA256 = strings.TrimSpace(meta.MetaSHA256) + if out[index].ETag == "" { + out[index].ETag = strings.TrimSpace(meta.ETag) + } + if out[index].LastModTime.IsZero() { + out[index].LastModTime = meta.LastModTime + } + } + }() + } + for index := range out { + indexCh <- index + } + close(indexCh) + wg.Wait() + if firstErr != nil { + return nil, firstErr + } + return out, nil +} + +func classifyS3ListError(bucket string, prefix string, err error) error { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch strings.ToLower(strings.TrimSpace(apiErr.ErrorCode())) { + case "forbidden", "accessdenied", "permissiondenied": + return &StorageInspectError{ + Kind: StorageInspectBucketUnavailable, + Message: fmt.Sprintf( + "provider rejected bucket inventory request for s3://%s/%s; mapped bucket target may be missing or inaccessible", + bucket, + strings.Trim(strings.TrimSpace(prefix), "/"), + ), + } + case "notfound", "nosuchbucket": + return &StorageInspectError{Kind: StorageInspectObjectNotFound, Message: fmt.Sprintf("provider could not find bucket %q", bucket)} + } + } + return fmt.Errorf("list s3 objects for %s/%s: %w", bucket, strings.Trim(strings.TrimSpace(prefix), "/"), err) +} diff --git a/internal/core/service.go b/internal/core/service.go index cb218b8b..7f5e4d3a 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -37,6 +37,7 @@ type ObjectManager struct { uM urlmanager.UrlManager bucketScopeCache *bucketScopeCache inspectS3Object func(context.Context, models.S3Credential, string, string) (*StorageObjectMetadata, error) + listS3Prefix func(context.Context, models.S3Credential, string, string, bool) ([]StorageBucketObject, error) } type VisibleBucket struct { @@ -50,5 +51,6 @@ func NewObjectManager(db db.DatabaseInterface, uM urlmanager.UrlManager) *Object uM: uM, bucketScopeCache: newBucketScopeCache(30 * time.Second), inspectS3Object: defaultS3ObjectInspector, + listS3Prefix: defaultS3PrefixLister, } } diff --git a/internal/db/postgres/objects.go b/internal/db/postgres/objects.go index 2c9337eb..1173c498 100644 --- a/internal/db/postgres/objects.go +++ b/internal/db/postgres/objects.go @@ -228,7 +228,7 @@ func (db *PostgresDB) CreateObject(ctx context.Context, obj *models.InternalObje _, err = tx.ExecContext(ctx, ` INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) VALUES ($1, $2, $3, $4, $5, $6, $7)`, - obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), common.StringVal(obj.Version), common.StringVal(obj.Description), + obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.CleanToBasename(common.StringVal(obj.Name)), common.StringVal(obj.Version), common.StringVal(obj.Description), ) if err != nil { return fmt.Errorf("failed to insert drs_object: %w", err) @@ -303,7 +303,7 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte sizes = append(sizes, obj.Size) createdTimes = append(createdTimes, obj.CreatedTime) updatedTimes = append(updatedTimes, common.TimeVal(obj.UpdatedTime)) - names = append(names, common.StringVal(obj.Name)) + names = append(names, common.CleanToBasename(common.StringVal(obj.Name))) versions = append(versions, common.StringVal(obj.Version)) descriptions = append(descriptions, common.StringVal(obj.Description)) diff --git a/internal/db/sqlite/objects.go b/internal/db/sqlite/objects.go index 73c09ee9..c9774a68 100644 --- a/internal/db/sqlite/objects.go +++ b/internal/db/sqlite/objects.go @@ -230,7 +230,7 @@ func (db *SqliteDB) CreateObject(ctx context.Context, obj *models.InternalObject _, err = tx.ExecContext(ctx, ` INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) VALUES (?, ?, ?, ?, ?, ?, ?)`, - obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), common.StringVal(obj.Version), common.StringVal(obj.Description), + obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.CleanToBasename(common.StringVal(obj.Name)), common.StringVal(obj.Version), common.StringVal(obj.Description), ) if err != nil { return fmt.Errorf("failed to insert drs_object: %w", err) @@ -295,7 +295,7 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern for _, obj := range objects { ids = append(ids, obj.Id) - mainArgs = append(mainArgs, obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), common.StringVal(obj.Version), common.StringVal(obj.Description)) + mainArgs = append(mainArgs, obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.CleanToBasename(common.StringVal(obj.Name)), common.StringVal(obj.Version), common.StringVal(obj.Description)) seenAccess := make(map[string]struct{}) for _, resource := range objectAccessResources(&obj) { diff --git a/internal/db/sqlite/sqlite_test.go b/internal/db/sqlite/sqlite_test.go index ca3d9c3d..ff6f7641 100644 --- a/internal/db/sqlite/sqlite_test.go +++ b/internal/db/sqlite/sqlite_test.go @@ -275,6 +275,96 @@ func TestSqliteDB_GetObjectPreservesStoredName(t *testing.T) { } } +func TestSqliteDB_NormalizeNameToBasenameOnInsert(t *testing.T) { + ctx := context.Background() + db, err := NewSqliteDB(":memory:") + if err != nil { + t.Fatalf("failed to create db: %v", err) + } + + // 1. Test CreateObject with Unix and Windows paths + now := time.Now().UTC() + objUnix := &models.InternalObject{ + DrsObject: drs.DrsObject{ + Id: "unix-1", + Size: 100, + CreatedTime: now, + Name: common.Ptr("/path/to/some/unix_file.txt"), + }, + } + if err := db.CreateObject(ctx, objUnix); err != nil { + t.Fatalf("CreateObject failed: %v", err) + } + + gotUnix, err := db.GetObject(ctx, "unix-1") + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + if got := common.StringVal(gotUnix.Name); got != "unix_file.txt" { + t.Fatalf("expected name to be normalized to unix_file.txt, got %q", got) + } + + objWin := &models.InternalObject{ + DrsObject: drs.DrsObject{ + Id: "win-1", + Size: 200, + CreatedTime: now, + Name: common.Ptr(`C:\Windows\System32\win_file.txt`), + }, + } + if err := db.CreateObject(ctx, objWin); err != nil { + t.Fatalf("CreateObject failed: %v", err) + } + + gotWin, err := db.GetObject(ctx, "win-1") + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + if got := common.StringVal(gotWin.Name); got != "win_file.txt" { + t.Fatalf("expected name to be normalized to win_file.txt, got %q", got) + } + + // 2. Test RegisterObjects with paths + bulkObjs := []models.InternalObject{ + { + DrsObject: drs.DrsObject{ + Id: "bulk-unix", + Size: 300, + CreatedTime: now, + Name: common.Ptr("/var/log/syslog.log"), + }, + }, + { + DrsObject: drs.DrsObject{ + Id: "bulk-win", + Size: 400, + CreatedTime: now, + Name: common.Ptr(`D:\Data\config.json`), + }, + }, + } + + if err := db.RegisterObjects(ctx, bulkObjs); err != nil { + t.Fatalf("RegisterObjects failed: %v", err) + } + + gotBulkUnix, err := db.GetObject(ctx, "bulk-unix") + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + if got := common.StringVal(gotBulkUnix.Name); got != "syslog.log" { + t.Fatalf("expected name syslog.log, got %q", got) + } + + gotBulkWin, err := db.GetObject(ctx, "bulk-win") + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + if got := common.StringVal(gotBulkWin.Name); got != "config.json" { + t.Fatalf("expected name config.json, got %q", got) + } +} + func TestSqliteDB_ObjectAliasLifecycle(t *testing.T) { ctx := context.Background() db, err := NewSqliteDB(":memory:") diff --git a/internal/repair/service.go b/internal/repair/service.go index 967bea6a..f4ecdd5b 100644 --- a/internal/repair/service.go +++ b/internal/repair/service.go @@ -112,7 +112,12 @@ func (s *Service) listRecords(ctx context.Context, opts Options) ([]internalapi. if limit <= 0 && opts.Limit > 0 { break } - resp, err := s.index.List(ctx, ListRecordsOptions{Limit: limit, Start: start}) + resp, err := s.index.List(ctx, ListRecordsOptions{ + Limit: limit, + Start: start, + Organization: opts.Organization, + Project: opts.Project, + }) if err != nil { return nil, scanned, err } @@ -170,7 +175,7 @@ func (s *Service) auditRecord(ctx context.Context, rec internalapi.InternalRecor obj.updated = &updated } if obj.scopeKnown && obj.canonicalURL != "" { - s.classifyAccessMethods(obj) + s.classifyAccessMethods(ctx, obj, opts.CheckStorage) } if opts.CheckStorage { s.addStorageFindings(ctx, obj) @@ -178,15 +183,30 @@ func (s *Service) auditRecord(ctx context.Context, rec internalapi.InternalRecor return obj, true } -func (s *Service) classifyAccessMethods(obj *auditedObject) { +func (s *Service) classifyAccessMethods(ctx context.Context, obj *auditedObject, checkStorage bool) { if obj.record.AccessMethods == nil { return } methods := cloneAccessMethods(*obj.record.AccessMethods) - hasCanonical := false + + // Construct path-style URL + pathStyleURL := pathStyleAccessURL(obj.scope, intcommon.StringVal(obj.record.Name)) + + // Determine correct target URL based on storage checks (if checkStorage is enabled) + targetURL := obj.canonicalURL + if checkStorage { + canonicalExists := s.checkURLExists(ctx, obj, obj.canonicalURL) + pathStyleExists := s.checkURLExists(ctx, obj, pathStyleURL) + if !canonicalExists && pathStyleExists { + targetURL = pathStyleURL + } + } + + // Check if the record already has the target URL + hasTarget := false for _, raw := range obj.currentURLs { - if raw == obj.canonicalURL { - hasCanonical = true + if raw == targetURL { + hasTarget = true break } } @@ -194,22 +214,21 @@ func (s *Service) classifyAccessMethods(obj *auditedObject) { changed := false for i := range methods { raw := accessMethodURL(methods[i]) - if raw == "" || raw == obj.canonicalURL { + if raw == "" || raw == targetURL { continue } - if isLegacyPathStyleURL(raw) { - if hasCanonical { - obj.findings = append(obj.findings, newFinding(FindingLegacyAccessURLRemovable, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, obj.canonicalURL, true, "legacy META/CONFIG URL has canonical sibling")) - methods[i].AccessUrl = nil - changed = true - continue - } - obj.findings = append(obj.findings, newFinding(FindingLegacyAccessURLRewritable, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, obj.canonicalURL, true, "legacy META/CONFIG URL can be rewritten to canonical scoped URL")) - setAccessMethodURL(&methods[i], obj.canonicalURL) + + // Any URL that does not match the target URL is considered legacy/incorrect. + if hasTarget { + obj.findings = append(obj.findings, newFinding(FindingLegacyAccessURLRemovable, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, targetURL, true, fmt.Sprintf("redundant URL %q has target sibling %q", raw, targetURL))) + methods[i].AccessUrl = nil changed = true - continue + } else { + obj.findings = append(obj.findings, newFinding(FindingLegacyAccessURLRewritable, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, targetURL, true, fmt.Sprintf("URL %q can be rewritten to target URL %q", raw, targetURL))) + setAccessMethodURL(&methods[i], targetURL) + changed = true + hasTarget = true } - obj.findings = append(obj.findings, newFinding(FindingNonCanonicalAccessURL, SeverityInfo, obj.record, obj.sha256, obj.currentURLs, obj.canonicalURL, false, "access URL does not match canonical scoped target")) } if !changed { @@ -473,17 +492,30 @@ func canonicalAccessURL(target scopeTarget, did, sha string) string { return "s3://" + strings.TrimSpace(target.Bucket) + "/" + strings.Join(parts, "/") } -func isLegacyPathStyleURL(raw string) bool { - parsed, err := url.Parse(strings.TrimSpace(raw)) - if err != nil || !strings.EqualFold(parsed.Scheme, "s3") { - return false +func pathStyleAccessURL(target scopeTarget, name string) string { + if strings.TrimSpace(target.Bucket) == "" || strings.TrimSpace(name) == "" { + return "" } - path := strings.Trim(strings.TrimSpace(parsed.Path), "/") - if path == "" { + parts := []string{} + if prefix := strings.Trim(target.Prefix, "/"); prefix != "" { + parts = append(parts, prefix) + } + parts = append(parts, strings.Trim(name, "/")) + return "s3://" + strings.TrimSpace(target.Bucket) + "/" + strings.Join(parts, "/") +} + +func (s *Service) checkURLExists(ctx context.Context, obj *auditedObject, url string) bool { + if s.requestor == nil { return false } - head := strings.ToUpper(strings.Split(path, "/")[0]) - return head == "META" || head == "CONFIG" + req := storageInspectRequest{ + Organization: obj.scope.Organization, + Project: obj.scope.Project, + ObjectURL: url, + } + var resp storageInspectResponse + err := s.requestor.Do(ctx, http.MethodPost, "/data/inspect", req, &resp) + return err == nil } func newFinding(kind FindingKind, severity Severity, rec internalapi.InternalRecord, sha string, currentURLs []string, canonical string, autoFixable bool, msg string) Finding { diff --git a/internal/repair/service_test.go b/internal/repair/service_test.go index 5635b170..b4aedbf0 100644 --- a/internal/repair/service_test.go +++ b/internal/repair/service_test.go @@ -61,6 +61,69 @@ func TestLegacyAccessURLWithoutCanonicalSiblingIsRewritable(t *testing.T) { } } +func TestPathStyleAccessURLWithCanonicalSiblingIsRemovable(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + // did is a real UUID + did := "01208c9f-e188-5aec-a026-51cc31fabd23" + // sha is a 64-char hex string + sha := "52aa452e5a90a2c641f20fffbac0e899d79d9cc40a560fe76e931b7f37566bb2" + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{ + recordWithMethods(t, did, sha, []string{resource}, []string{ + "s3://bforepc-prod/JHU/ashley_kiemen/hematoxylin_eosin_stain/Level_2/HTA201_3/HTA201_3_ndpis/HTA201_3_1_0145.ndpi", + "s3://bforepc-prod/" + did + "/" + sha, + }), + }}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 1 { + t.Fatalf("expected one object report, got %d", len(report.Objects)) + } + obj := report.Objects[0] + if !obj.AutoFixable { + t.Fatal("expected autofixable object") + } + if obj.Findings[0].Kind != FindingLegacyAccessURLRemovable { + t.Fatalf("expected removable finding, got %s", obj.Findings[0].Kind) + } +} + +func TestPathStyleAccessURLWithoutCanonicalSiblingIsRewritable(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + did := "01208c9f-e188-5aec-a026-51cc31fabd23" + sha := "52aa452e5a90a2c641f20fffbac0e899d79d9cc40a560fe76e931b7f37566bb2" + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{ + recordWithMethods(t, did, sha, []string{resource}, []string{ + "s3://bforepc-prod/JHU/ashley_kiemen/hematoxylin_eosin_stain/Level_2/HTA201_3/HTA201_3_ndpis/HTA201_3_1_0145.ndpi", + }), + }}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 1 { + t.Fatalf("expected one object report, got %d", len(report.Objects)) + } + obj := report.Objects[0] + if !obj.AutoFixable { + t.Fatal("expected autofixable object") + } + if obj.Findings[0].Kind != FindingLegacyAccessURLRewritable { + t.Fatalf("expected rewritable finding, got %s", obj.Findings[0].Kind) + } +} + func TestMissingControlledAccessRecoverableFromDeterministicScope(t *testing.T) { resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") did, err := intcommon.MintObjectIDFromChecksum("abc", []string{resource}) diff --git a/internal/repair/types.go b/internal/repair/types.go index dff0a3b1..f63a6065 100644 --- a/internal/repair/types.go +++ b/internal/repair/types.go @@ -14,8 +14,10 @@ type IndexAPI interface { } type ListRecordsOptions struct { - Limit int - Start string + Limit int + Start string + Organization string + Project string } type BucketsAPI interface { From ba2c0281745febb02d0e44bef080085ed18e3148 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 2 Jul 2026 13:40:28 -0700 Subject: [PATCH 10/24] update syfon to correctly return distinct records --- apigen/client/internalapi/internal.gen.go | 2 + apigen/server/internalapi/internal.gen.go | 2 + .../api/internaldrs/bucket_routes_test.go | 60 +++ internal/api/internaldrs/buckets.go | 20 +- internal/api/internaldrs/index.go | 8 + internal/api/internaldrs/index_test.go | 173 +++++++++ internal/api/internaldrs/inspect.go | 124 +++++- .../api/internaldrs/inspect_routes_test.go | 175 +++++++++ internal/api/internaldrs/repair_cleanup.go | 6 + internal/api/internaldrs/transfer.go | 1 + internal/common/routes.go | 1 + internal/common/util.go | 22 +- internal/core/converters.go | 4 + internal/core/object_canonicalization.go | 340 +++++++++++++++++ internal/core/object_mutations.go | 187 ++++++++- internal/core/object_mutations_test.go | 88 +++++ internal/core/object_reads.go | 180 ++++++--- internal/core/object_reads_test.go | 102 +++++ internal/core/object_storage_inventory.go | 360 +++++++++++++++++- internal/core/service.go | 2 +- internal/db/interface.go | 1 + internal/db/postgres/object_schema.sql | 10 + internal/db/postgres/objects.go | 170 ++++++++- internal/db/postgres/objects_sqlmock_test.go | 34 ++ internal/db/sqlite/objects.go | 183 ++++++++- internal/db/sqlite/sqlite.go | 7 + internal/db/sqlite/sqlite_test.go | 77 ++++ internal/models/object.go | 56 ++- internal/testutils/mocks.go | 50 ++- 29 files changed, 2354 insertions(+), 91 deletions(-) create mode 100644 internal/core/object_canonicalization.go diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index ea0f532d..e50169cc 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -111,6 +111,7 @@ type InternalRecord struct { // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` Name *string `json:"name,omitempty"` + NameAliases *[]string `json:"name_aliases,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` Size *int64 `json:"size,omitempty"` @@ -131,6 +132,7 @@ type InternalRecordResponse struct { // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` Name *string `json:"name,omitempty"` + NameAliases *[]string `json:"name_aliases,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` Rev *string `json:"rev,omitempty"` diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 5e5d2956..8ffa2d3d 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -109,6 +109,7 @@ type InternalRecord struct { // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` Name *string `json:"name,omitempty"` + NameAliases *[]string `json:"name_aliases,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` Size *int64 `json:"size,omitempty"` @@ -129,6 +130,7 @@ type InternalRecordResponse struct { // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` Name *string `json:"name,omitempty"` + NameAliases *[]string `json:"name_aliases,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` Rev *string `json:"rev,omitempty"` diff --git a/internal/api/internaldrs/bucket_routes_test.go b/internal/api/internaldrs/bucket_routes_test.go index 1a75ab45..fe14d42e 100644 --- a/internal/api/internaldrs/bucket_routes_test.go +++ b/internal/api/internaldrs/bucket_routes_test.go @@ -199,6 +199,66 @@ func TestHandleInternalPutBucket_RejectsInvalidGeneratedPayloads(t *testing.T) { } } +func TestHandleInternalPutBucket_ReusesExistingPhysicalBucketCredential(t *testing.T) { + mockDB := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cbdscollab_1c102e76761b": { + CredentialID: "cbdscollab_1c102e76761b", + Bucket: "cbdscollab", + Provider: "s3", + Region: "us-east-1", + AccessKey: "old-ak", + SecretKey: "old-sk", + Endpoint: "https://old-endpoint.example.org", + }, + }, + } + provider := "s3" + region := "us-east-1" + accessKey := "new-ak" + secretKey := "new-sk" + endpoint := "https://fortera-object.ohsu.edu" + path := "s3://cbdscollab/Lab_Projects/Embedding_Rotation" + putBody, _ := json.Marshal(bucketapi.PutBucketRequest{ + Bucket: "cbdscollab", + Provider: &provider, + Region: ®ion, + AccessKey: &accessKey, + SecretKey: &secretKey, + Endpoint: &endpoint, + Organization: "Ellrott_Lab", + ProjectId: "embedding_rotation", + Path: &path, + }) + req := httptest.NewRequest(http.MethodPut, "/data/buckets", bytes.NewBuffer(putBody)) + req = req.WithContext(dataTestAuthContext(req.Context(), "gen3", true, map[string]map[string]bool{ + "/programs/Ellrott_Lab/projects/embedding_rotation": {"create": true, "update": true}, + })) + + rr := doInternalDRSTestRequest(req, core.NewObjectManager(mockDB, &testutils.MockUrlManager{})) + if rr.Code != http.StatusCreated { + t.Fatalf("expected 201, got %d body=%s", rr.Code, rr.Body.String()) + } + + updated, ok := mockDB.Credentials["cbdscollab_1c102e76761b"] + if !ok { + t.Fatalf("expected existing credential id to be preserved, got %+v", mockDB.Credentials) + } + if updated.AccessKey != accessKey || updated.SecretKey != secretKey || updated.Endpoint != endpoint { + t.Fatalf("expected existing credential to be updated, got %+v", updated) + } + if _, exists := mockDB.Credentials[common.DeriveCredentialID("cbdscollab", provider, region, endpoint, accessKey)]; exists { + t.Fatalf("expected no replacement credential to be created, got %+v", mockDB.Credentials) + } + scope, ok := mockDB.BucketScopes["Ellrott_Lab|embedding_rotation"] + if !ok { + t.Fatalf("expected bucket scope to be created") + } + if scope.CredentialID != "cbdscollab_1c102e76761b" || scope.Bucket != "cbdscollab" || scope.PathPrefix != "Lab_Projects/Embedding_Rotation" { + t.Fatalf("unexpected scope saved: %+v", scope) + } +} + func TestRegisterInternalRoutes_Smoke(t *testing.T) { app := fiber.New() om := core.NewObjectManager(&testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}, Credentials: map[string]models.S3Credential{"b1": {Bucket: "b1"}}}, &testutils.MockUrlManager{}) diff --git a/internal/api/internaldrs/buckets.go b/internal/api/internaldrs/buckets.go index 7aa600a5..64d3eac0 100644 --- a/internal/api/internaldrs/buckets.go +++ b/internal/api/internaldrs/buckets.go @@ -93,15 +93,15 @@ func handleInternalPutBucketFiber(c fiber.Ctx, om *core.ObjectManager) error { return apiutil.Reject(c, fiber.StatusBadRequest, err.Error()) } - hasCredentialMaterial := accessKey != "" || secretKey != "" || endpoint != "" || region != "" || rawProvider != "" credentialID := "" var existingCred *models.S3Credential var credErr error - if !hasCredentialMaterial { - existingCred, credErr = om.GetS3Credential(c.Context(), req.Bucket) - if credErr == nil && existingCred != nil { - credentialID = existingCred.CredentialID - } + existingCred, credErr = om.GetS3Credential(c.Context(), req.Bucket) + if credErr != nil && !isCredentialNotFoundError(credErr) { + return apiutil.HandleError(c, credErr) + } + if credErr == nil && existingCred != nil { + credentialID = existingCred.CredentialID } if credentialID == "" { credentialID = common.DeriveCredentialID(req.Bucket, bucketProvider, region, endpoint, accessKey) @@ -110,6 +110,10 @@ func handleInternalPutBucketFiber(c fiber.Ctx, om *core.ObjectManager) error { existingCred, credErr = om.GetS3Credential(c.Context(), credentialID) } hasExistingCred := credErr == nil && existingCred != nil + if hasExistingCred && rawProvider == "" { + bucketProvider = existingCred.Provider + req.Provider = common.Ptr(bucketProvider) + } scopeOnly := hasExistingCred && accessKey == "" && secretKey == "" && @@ -174,6 +178,10 @@ func handleInternalPutBucketFiber(c fiber.Ctx, om *core.ObjectManager) error { return c.SendStatus(fiber.StatusCreated) } +func isCredentialNotFoundError(err error) bool { + return err != nil && strings.EqualFold(strings.TrimSpace(err.Error()), "credential not found") +} + func handleInternalDeleteBucketFiber(c fiber.Ctx, om *core.ObjectManager) error { credentialID := strings.TrimSpace(c.Params("bucket")) if credentialID == "" { diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index 6b995731..b4e2c9de 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -76,6 +76,10 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { if err != nil { return apiutil.HandleError(c, err) } + objs, err = om.PrepareScopedObjects(c.Context(), objs, filterOrg, filterProject, "read") + if err != nil { + return apiutil.HandleError(c, err) + } records := make([]internalapi.InternalRecord, 0, len(objs)) for _, o := range objs { records = append(records, core.InternalObjectToInternalRecord(o)) @@ -109,6 +113,10 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { if err != nil { return apiutil.HandleError(c, err) } + objs, err = om.PrepareScopedObjects(c.Context(), objs, filterOrg, filterProject, "read") + if err != nil { + return apiutil.HandleError(c, err) + } records := make([]internalapi.InternalRecord, 0, len(objs)) for _, obj := range objs { records = append(records, core.InternalObjectToInternalRecord(obj)) diff --git a/internal/api/internaldrs/index_test.go b/internal/api/internaldrs/index_test.go index a77c9146..e827550b 100644 --- a/internal/api/internaldrs/index_test.go +++ b/internal/api/internaldrs/index_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "slices" "strings" "testing" "time" @@ -17,6 +18,7 @@ import ( internalauth "github.com/calypr/syfon/internal/auth" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/db" "github.com/calypr/syfon/internal/models" "github.com/calypr/syfon/internal/testutils" "github.com/gofiber/fiber/v3" @@ -112,6 +114,177 @@ func TestHandleInternalList_ExactScopeListingDoesNotDependOnBrowseRows(t *testin } } +func TestHandleInternalList_CanonicalizesProjectChecksumDuplicates(t *testing.T) { + database := db.NewInMemoryDB() + om := core.NewObjectManager(database, &testutils.MockUrlManager{}) + now := time.Now().UTC() + later := now.Add(time.Minute) + sha := "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + oldURL := "s3://ellrottlab/old/" + sha + newURL := "s3://EllrottLab/new/" + sha + + for _, obj := range []models.InternalObject{ + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-1", + Name: stringPtr("older.tsv"), + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: sha}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: oldURL}, + }}, + }, + }, + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-2", + Name: stringPtr("newer.tsv"), + CreatedTime: later, + UpdatedTime: &later, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: sha}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: newURL}, + }}, + }, + }, + } { + if err := om.RegisterObjects(context.Background(), []models.InternalObject{obj}); err != nil { + t.Fatalf("RegisterObjects failed: %v", err) + } + } + + req := httptest.NewRequest(http.MethodGet, "/index?organization=org&project=p1", nil) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var payload struct { + Records []internalapi.InternalRecord `json:"records"` + } + if err := json.NewDecoder(rr.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload.Records) != 1 { + t.Fatalf("expected 1 canonical record, got %d", len(payload.Records)) + } + if payload.Records[0].Did != "did-1" { + t.Fatalf("expected canonical did-1, got %q", payload.Records[0].Did) + } + if payload.Records[0].Name == nil || *payload.Records[0].Name != "newer.tsv" { + t.Fatalf("expected latest name newer.tsv, got %+v", payload.Records[0].Name) + } + if payload.Records[0].NameAliases == nil || !slices.Equal(*payload.Records[0].NameAliases, []string{"older.tsv"}) { + t.Fatalf("unexpected name aliases: %#v", payload.Records[0].NameAliases) + } + if payload.Records[0].AccessMethods == nil || len(*payload.Records[0].AccessMethods) != 2 { + t.Fatalf("expected 2 merged access methods, got %#v", payload.Records[0].AccessMethods) + } + gotURLs := make([]string, 0, len(*payload.Records[0].AccessMethods)) + for _, method := range *payload.Records[0].AccessMethods { + if method.AccessUrl != nil { + gotURLs = append(gotURLs, method.AccessUrl.Url) + } + } + slices.Sort(gotURLs) + if !slices.Equal(gotURLs, []string{newURL, oldURL}) { + t.Fatalf("unexpected merged access method urls: %#v", gotURLs) + } +} + +func TestHandleInternalList_MergesSiblingAccessMethodsFromLegacyDuplicateRows(t *testing.T) { + database := db.NewInMemoryDB() + om := core.NewObjectManager(database, &testutils.MockUrlManager{}) + now := time.Now().UTC() + later := now.Add(time.Minute) + sha := "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + oldURL := "s3://ellrottlab/fa4ee697-f689-5291-a7cc-8ebe2f3ea8b9/" + sha + newURL := "s3://EllrottLab/" + sha + controlled := []string{"/organization/org/project/p1"} + + for _, obj := range []models.InternalObject{ + { + DrsObject: drs.DrsObject{ + Id: "did-legacy-1", + Name: stringPtr("legacy.tsv"), + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: sha}}, + ControlledAccess: &controlled, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: oldURL}, + }}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "did-legacy-2", + Name: stringPtr("canonical.tsv"), + CreatedTime: later, + UpdatedTime: &later, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: sha}}, + ControlledAccess: &controlled, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: newURL}, + }}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + } { + candidate := obj + if err := database.CreateObject(context.Background(), &candidate); err != nil { + t.Fatalf("CreateObject failed: %v", err) + } + } + + req := httptest.NewRequest(http.MethodGet, "/index?organization=org&project=p1", nil) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var payload struct { + Records []internalapi.InternalRecord `json:"records"` + } + if err := json.NewDecoder(rr.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload.Records) != 1 { + t.Fatalf("expected 1 canonical record, got %d", len(payload.Records)) + } + if payload.Records[0].AccessMethods == nil || len(*payload.Records[0].AccessMethods) != 2 { + t.Fatalf("expected 2 merged access methods, got %#v", payload.Records[0].AccessMethods) + } + gotURLs := make([]string, 0, len(*payload.Records[0].AccessMethods)) + for _, method := range *payload.Records[0].AccessMethods { + if method.AccessUrl != nil { + gotURLs = append(gotURLs, method.AccessUrl.Url) + } + } + slices.Sort(gotURLs) + if !slices.Equal(gotURLs, []string{newURL, oldURL}) { + t.Fatalf("unexpected merged access method urls: %#v", gotURLs) + } +} + func TestHandleInternalList_PaginatesIDs(t *testing.T) { now := time.Now().UTC() mockDB := &testutils.MockDatabase{ diff --git a/internal/api/internaldrs/inspect.go b/internal/api/internaldrs/inspect.go index fedb2877..11cd4294 100644 --- a/internal/api/internaldrs/inspect.go +++ b/internal/api/internaldrs/inspect.go @@ -1,6 +1,7 @@ package internaldrs import ( + "log" "strings" "time" @@ -22,6 +23,7 @@ type internalInspectObjectRequest struct { ObjectURL string `json:"object_url,omitempty"` ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` ExpectedSHA256 string `json:"expected_sha256,omitempty"` + ExpectedName string `json:"expected_name,omitempty"` } type internalInspectObjectsBulkRequest struct { @@ -32,6 +34,8 @@ type internalInspectProjectBucketRequest struct { Organization string `json:"organization,omitempty"` Project string `json:"project,omitempty"` IncludeHead bool `json:"include_head,omitempty"` + Mode string `json:"mode,omitempty"` + PathPrefix string `json:"path_prefix,omitempty"` } type internalInspectProjectRecordsRequest struct { @@ -67,7 +71,8 @@ type internalInspectObjectBulkResponse struct { } type internalInspectProjectBucketResponse struct { - Items []internalInspectProjectBucketItem `json:"items"` + Summary *internalInspectProjectBucketSummary `json:"summary,omitempty"` + Items []internalInspectProjectBucketItem `json:"items"` } type internalInspectProjectRecordsResponse struct { @@ -99,10 +104,23 @@ type internalInspectObjectBulkItem struct { LastModTime string `json:"last_modified,omitempty"` ValidationStatus string `json:"validation_status"` SizeMatch *bool `json:"size_match,omitempty"` + NameMatch *bool `json:"name_match,omitempty"` SHA256Match *bool `json:"sha256_match,omitempty"` ValidationMismatches []string `json:"validation_mismatches,omitempty"` } +type internalInspectProjectBucketSummary struct { + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + Exists bool `json:"exists"` + ObjectCount int `json:"object_count"` + TotalBytes int64 `json:"total_bytes"` + ComputedAt string `json:"computed_at"` + Mode string `json:"mode"` +} + type internalInspectProjectBucketItem struct { ObjectURL string `json:"object_url"` Provider string `json:"provider"` @@ -117,6 +135,7 @@ type internalInspectProjectBucketItem struct { type internalInspectProjectRecordItem struct { ObjectID string `json:"object_id"` + Name string `json:"name,omitempty"` Checksum string `json:"checksum"` Organization string `json:"organization"` Project string `json:"project"` @@ -220,8 +239,41 @@ func handleInternalInspectObjectBulkFiber(om *core.ObjectManager) fiber.Handler } } +func handleInternalInspectObjectBulkListFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + started := time.Now() + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectObjectsBulkRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + if len(req.Items) == 0 { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: items are required") + } + items := make([]core.StorageListValidationRequest, 0, len(req.Items)) + for _, item := range req.Items { + items = append(items, core.StorageListValidationRequest{ + ID: strings.TrimSpace(item.ID), + ObjectURL: strings.TrimSpace(item.ObjectURL), + ExpectedSizeBytes: item.ExpectedSizeBytes, + ExpectedName: strings.TrimSpace(item.ExpectedName), + }) + } + results := om.ListValidateStorageObjects(c.Context(), items) + out := internalInspectObjectBulkResponse{Items: make([]internalInspectObjectBulkItem, 0, len(results))} + for _, result := range results { + out.Items = append(out.Items, bulkListInspectItemFromCore(result)) + } + log.Printf("INFO: syfon_inspect_bulk_list_handler items=%d results=%d duration_ms=%d", len(items), len(out.Items), time.Since(started).Milliseconds()) + return c.JSON(out) + } +} + func handleInternalInspectProjectBucketFiber(om *core.ObjectManager) fiber.Handler { return func(c fiber.Ctx) error { + started := time.Now() if apimiddleware.MissingGen3AuthHeader(c.Context()) { return c.SendStatus(fiber.StatusUnauthorized) } @@ -229,12 +281,20 @@ func handleInternalInspectProjectBucketFiber(om *core.ObjectManager) fiber.Handl if err := decodeStrictJSON(c.Body(), &req); err != nil { return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) } - items, err := om.ListProjectStorageObjects(c.Context(), strings.TrimSpace(req.Organization), strings.TrimSpace(req.Project), req.IncludeHead) + result, err := om.InspectProjectStorage(c.Context(), strings.TrimSpace(req.Organization), strings.TrimSpace(req.Project), core.ProjectStorageInspectOptions{ + Mode: core.ProjectStorageInspectMode(strings.TrimSpace(req.Mode)), + IncludeHead: req.IncludeHead, + PathPrefix: strings.TrimSpace(req.PathPrefix), + }) if err != nil { + log.Printf("INFO: syfon_project_bucket_handler organization=%s project=%s mode=%s path_prefix=%q include_head=%t duration_ms=%d error=%q", req.Organization, req.Project, req.Mode, req.PathPrefix, req.IncludeHead, time.Since(started).Milliseconds(), err.Error()) return handleInspectStorageError(c, err) } - out := internalInspectProjectBucketResponse{Items: make([]internalInspectProjectBucketItem, 0, len(items))} - for _, item := range items { + out := internalInspectProjectBucketResponse{ + Summary: projectBucketSummaryFromCore(result.Summary), + Items: make([]internalInspectProjectBucketItem, 0, len(result.Items)), + } + for _, item := range result.Items { row := internalInspectProjectBucketItem{ ObjectURL: item.ObjectURL, Provider: item.Provider, @@ -250,6 +310,17 @@ func handleInternalInspectProjectBucketFiber(om *core.ObjectManager) fiber.Handl } out.Items = append(out.Items, row) } + exists := false + objectCount := 0 + totalBytes := int64(0) + mode := strings.TrimSpace(req.Mode) + if out.Summary != nil { + exists = out.Summary.Exists + objectCount = out.Summary.ObjectCount + totalBytes = out.Summary.TotalBytes + mode = out.Summary.Mode + } + log.Printf("INFO: syfon_project_bucket_handler organization=%s project=%s mode=%s path_prefix=%q include_head=%t exists=%t object_count=%d returned_items=%d total_bytes=%d duration_ms=%d", req.Organization, req.Project, mode, req.PathPrefix, req.IncludeHead, exists, objectCount, len(out.Items), totalBytes, time.Since(started).Milliseconds()) return c.JSON(out) } } @@ -402,6 +473,48 @@ func bulkInspectItemFromCore(result core.StorageProbeResult) internalInspectObje return out } +func bulkListInspectItemFromCore(result core.StorageListValidationResult) internalInspectObjectBulkItem { + out := internalInspectObjectBulkItem{ + ID: result.ID, + ObjectURL: result.ObjectURL, + Provider: result.Provider, + Bucket: result.Bucket, + Key: result.Key, + Path: result.Path, + Exists: result.Exists, + Status: string(result.Status), + Error: result.Error, + ErrorKind: result.ErrorKind, + SizeBytes: result.SizeBytes, + ETag: result.ETag, + ValidationStatus: string(result.ValidationStatus), + SizeMatch: result.SizeMatch, + NameMatch: result.NameMatch, + ValidationMismatches: append([]string(nil), result.ValidationMismatches...), + } + if !result.LastModTime.IsZero() { + out.LastModTime = result.LastModTime.Format(time.RFC3339) + } + return out +} + +func projectBucketSummaryFromCore(summary core.ProjectStorageSummary) *internalInspectProjectBucketSummary { + out := &internalInspectProjectBucketSummary{ + Provider: summary.Provider, + Bucket: summary.Bucket, + Prefix: summary.Prefix, + ObjectURL: summary.ObjectURL, + Exists: summary.Exists, + ObjectCount: summary.ObjectCount, + TotalBytes: summary.TotalBytes, + Mode: string(summary.Mode), + } + if !summary.ComputedAt.IsZero() { + out.ComputedAt = summary.ComputedAt.Format(time.RFC3339) + } + return out +} + func projectRecordAuditItemFromObject(obj models.InternalObject, organization, project string) (internalInspectProjectRecordItem, bool) { checksum := primarySHA256Checksum(obj.Checksums) if checksum == "" { @@ -439,6 +552,9 @@ func projectRecordAuditItemFromObject(obj models.InternalObject, organization, p AccessURLs: accessURLs, AccessMethods: accessMethods, } + if obj.Name != nil { + item.Name = strings.TrimSpace(*obj.Name) + } if !obj.CreatedTime.IsZero() { item.CreatedTime = obj.CreatedTime.Format(time.RFC3339Nano) } diff --git a/internal/api/internaldrs/inspect_routes_test.go b/internal/api/internaldrs/inspect_routes_test.go index cbd25ad1..d50150d3 100644 --- a/internal/api/internaldrs/inspect_routes_test.go +++ b/internal/api/internaldrs/inspect_routes_test.go @@ -164,6 +164,9 @@ func TestHandleInternalInspectProjectRecords(t *testing.T) { if item.ObjectID != "obj-1" || item.Checksum != "abc123" { t.Fatalf("unexpected item identity: %+v", item) } + if item.Name != "example.bin" { + t.Fatalf("expected project record name, got %q", item.Name) + } if item.Organization != "syfon" || item.Project != "e2e" { t.Fatalf("unexpected scope: %+v", item) } @@ -203,3 +206,175 @@ func TestHandleInternalInspectProjectScopes(t *testing.T) { t.Fatalf("unexpected first scope: %+v", resp.Items[0]) } } + +func TestHandleInternalInspectProjectBucketModesUsePrefixList(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|": {Organization: "syfon", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "program-root"}, + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + inspectCalls := 0 + om.SetS3ObjectInspector(func(ctx context.Context, cred models.S3Credential, bucket string, key string) (*core.StorageObjectMetadata, error) { + inspectCalls++ + return nil, nil + }) + var listCalls []core.StoragePrefixListOptions + var listedPrefixes []string + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + if bucket != "bucket-a" { + t.Fatalf("unexpected list target bucket=%q prefix=%q", bucket, prefix) + } + listedPrefixes = append(listedPrefixes, prefix) + listCalls = append(listCalls, options) + if prefix == "program-root/project-root/CONFIG" { + return []core.StorageBucketObject{ + {Provider: "s3", Bucket: bucket, Key: prefix + "/a.bin", Path: "CONFIG/a.bin", SizeBytes: 10}, + {Provider: "s3", Bucket: bucket, Key: prefix + "/nested/b.bin", Path: "CONFIG/nested/b.bin", SizeBytes: 15}, + }, nil + } + if prefix != "program-root/project-root" { + t.Fatalf("unexpected list prefix=%q", prefix) + } + return []core.StorageBucketObject{ + {Provider: "s3", Bucket: bucket, Key: prefix + "/a.bin", Path: "a.bin", SizeBytes: 10}, + {Provider: "s3", Bucket: bucket, Key: prefix + "/b.bin", Path: "b.bin", SizeBytes: 15}, + }, nil + }) + + existsBody, _ := json.Marshal(internalInspectProjectBucketRequest{Organization: "syfon", Project: "e2e", Mode: "exists"}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-bucket", bytes.NewBuffer(existsBody)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var existsResp internalInspectProjectBucketResponse + if err := json.NewDecoder(rr.Body).Decode(&existsResp); err != nil { + t.Fatalf("decode exists response: %v", err) + } + if existsResp.Summary == nil || !existsResp.Summary.Exists || existsResp.Summary.Mode != "exists" { + t.Fatalf("unexpected exists summary: %+v", existsResp.Summary) + } + if len(existsResp.Items) != 0 { + t.Fatalf("exists mode should not return item rows, got %+v", existsResp.Items) + } + if len(listCalls) != 1 || listCalls[0].MaxKeys != 1 || listCalls[0].IncludeHead { + t.Fatalf("exists mode should use MaxKeys=1 without HEAD, got %+v", listCalls) + } + + summaryBody, _ := json.Marshal(internalInspectProjectBucketRequest{Organization: "syfon", Project: "e2e", Mode: "summary"}) + req = withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-bucket", bytes.NewBuffer(summaryBody)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr = doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var summaryResp internalInspectProjectBucketResponse + if err := json.NewDecoder(rr.Body).Decode(&summaryResp); err != nil { + t.Fatalf("decode summary response: %v", err) + } + if summaryResp.Summary == nil || summaryResp.Summary.Mode != "summary" || summaryResp.Summary.ObjectCount != 2 || summaryResp.Summary.TotalBytes != 25 { + t.Fatalf("unexpected summary response: %+v", summaryResp.Summary) + } + if len(summaryResp.Items) != 0 { + t.Fatalf("summary mode should not return item rows, got %+v", summaryResp.Items) + } + if len(listCalls) != 2 || listCalls[1].MaxKeys != 0 || listCalls[1].IncludeHead { + t.Fatalf("summary mode should list recursively without HEAD, got %+v", listCalls) + } + + itemsBody, _ := json.Marshal(internalInspectProjectBucketRequest{Organization: "syfon", Project: "e2e", Mode: "items", PathPrefix: "CONFIG"}) + req = withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-bucket", bytes.NewBuffer(itemsBody)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr = doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var itemsResp internalInspectProjectBucketResponse + if err := json.NewDecoder(rr.Body).Decode(&itemsResp); err != nil { + t.Fatalf("decode items response: %v", err) + } + if itemsResp.Summary == nil || itemsResp.Summary.Mode != "items" || itemsResp.Summary.Prefix != "program-root/project-root/CONFIG" { + t.Fatalf("unexpected items summary: %+v", itemsResp.Summary) + } + if len(itemsResp.Items) != 2 { + t.Fatalf("items mode should return recursive rows, got %+v", itemsResp.Items) + } + if len(listedPrefixes) != 3 || listedPrefixes[2] != "program-root/project-root/CONFIG" { + t.Fatalf("expected path_prefix to scope recursive LIST, got %+v", listedPrefixes) + } + if inspectCalls != 0 { + t.Fatalf("expected project-bucket modes not to HEAD objects, got %d inspector calls", inspectCalls) + } +} + +func TestHandleInternalInspectObjectBulkListValidatesExactKeyWithoutHead(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + inspectCalls := 0 + om.SetS3ObjectInspector(func(ctx context.Context, cred models.S3Credential, bucket string, key string) (*core.StorageObjectMetadata, error) { + inspectCalls++ + return nil, nil + }) + var listedPrefixes []string + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + if bucket != "bucket-a" || !options.ExactPrefix || options.MaxKeys != 1 || options.IncludeHead { + t.Fatalf("unexpected bulk-list options bucket=%q options=%+v", bucket, options) + } + listedPrefixes = append(listedPrefixes, prefix) + switch prefix { + case "project-root/file.bin": + return []core.StorageBucketObject{{Provider: "s3", Bucket: bucket, Key: "project-root/file.bin", Path: "file.bin", SizeBytes: 17, ETag: "etag-1", LastModTime: time.Date(2026, 7, 1, 1, 2, 3, 0, time.UTC)}}, nil + case "project-root/dir": + return []core.StorageBucketObject{{Provider: "s3", Bucket: bucket, Key: "project-root/dir/child.bin", Path: "dir/child.bin", SizeBytes: 19}}, nil + default: + return nil, nil + } + }) + expectedSize := int64(17) + body, _ := json.Marshal(internalInspectObjectsBulkRequest{Items: []internalInspectObjectRequest{ + {ID: "present", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &expectedSize, ExpectedName: "file.bin"}, + {ID: "prefix-child", ObjectURL: "s3://bucket-a/project-root/dir", ExpectedSizeBytes: &expectedSize, ExpectedName: "dir"}, + }}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/bulk-list", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectObjectBulkResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Items) != 2 { + t.Fatalf("expected 2 items, got %+v", resp.Items) + } + if !resp.Items[0].Exists || resp.Items[0].Status != "present" || resp.Items[0].ValidationStatus != "matched" { + t.Fatalf("unexpected present item: %+v", resp.Items[0]) + } + if resp.Items[1].Exists || resp.Items[1].Status != "not_found" { + t.Fatalf("expected prefix child not to count as exact key, got %+v", resp.Items[1]) + } + if len(listedPrefixes) != 2 { + t.Fatalf("expected two LIST calls, got %+v", listedPrefixes) + } + if inspectCalls != 0 { + t.Fatalf("expected bulk-list not to HEAD objects, got %d inspector calls", inspectCalls) + } +} diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go index 64fc3781..7e1506a0 100644 --- a/internal/api/internaldrs/repair_cleanup.go +++ b/internal/api/internaldrs/repair_cleanup.go @@ -89,6 +89,9 @@ func handleInternalScopeRepairApplyFiber(om *core.ObjectManager) fiber.Handler { if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "update"); err != nil { return apiutil.HandleError(c, err) } + if _, err := om.CollapseProjectChecksumDuplicates(c.Context(), req.Organization, req.Project); err != nil { + return apiutil.HandleError(c, err) + } result, err := svc.Apply(c.Context(), req) if err != nil { return apiutil.HandleError(c, err) @@ -186,6 +189,7 @@ func clientRecordFromServer(rec serverinternalapi.InternalRecord) clientinternal CreatedTime: rec.CreatedTime, Description: rec.Description, Name: rec.Name, + NameAliases: rec.NameAliases, Version: rec.Version, UpdatedTime: rec.UpdatedTime, Hashes: (*clientinternalapi.HashInfo)(rec.Hashes), @@ -204,6 +208,7 @@ func serverRecordFromClient(rec clientinternalapi.InternalRecord) serverinternal CreatedTime: rec.CreatedTime, Description: rec.Description, Name: rec.Name, + NameAliases: rec.NameAliases, Version: rec.Version, UpdatedTime: rec.UpdatedTime, Hashes: (*serverinternalapi.HashInfo)(rec.Hashes), @@ -222,6 +227,7 @@ func clientRecordResponseFromServer(rec serverinternalapi.InternalRecordResponse CreatedTime: rec.CreatedTime, Description: rec.Description, Name: rec.Name, + NameAliases: rec.NameAliases, Version: rec.Version, UpdatedTime: rec.UpdatedTime, Hashes: (*clientinternalapi.HashInfo)(rec.Hashes), diff --git a/internal/api/internaldrs/transfer.go b/internal/api/internaldrs/transfer.go index ff269056..50b6b779 100644 --- a/internal/api/internaldrs/transfer.go +++ b/internal/api/internaldrs/transfer.go @@ -33,6 +33,7 @@ func registerInternalTransferRoutes(router fiber.Router, om *core.ObjectManager) router.Get(routeutil.FiberPath(common.RouteInternalUploadURL), handleInternalUploadURLFiber(om)) router.Post(common.RouteInternalInspectObject, handleInternalInspectObjectFiber(om)) router.Post(common.RouteInternalInspectObjectBulk, handleInternalInspectObjectBulkFiber(om)) + router.Post(common.RouteInternalInspectObjectBulkList, handleInternalInspectObjectBulkListFiber(om)) router.Post(common.RouteInternalInspectProjectBucket, handleInternalInspectProjectBucketFiber(om)) router.Post(common.RouteInternalInspectProjectRecords, handleInternalInspectProjectRecordsFiber(om)) router.Post(common.RouteInternalInspectProjectScopes, handleInternalInspectProjectScopesFiber(om)) diff --git a/internal/common/routes.go b/internal/common/routes.go index 20ddfe2f..d61051e4 100644 --- a/internal/common/routes.go +++ b/internal/common/routes.go @@ -11,6 +11,7 @@ const ( RouteInternalMultipartComplete = "/data/multipart/complete" RouteInternalInspectObject = "/data/inspect" RouteInternalInspectObjectBulk = "/data/inspect/bulk" + RouteInternalInspectObjectBulkList = "/data/inspect/bulk-list" RouteInternalInspectProjectBucket = "/data/inspect/project-bucket" RouteInternalInspectProjectRecords = "/data/inspect/project-records" RouteInternalInspectProjectScopes = "/data/inspect/project-scopes" diff --git a/internal/common/util.go b/internal/common/util.go index 99f48528..7f4786b7 100644 --- a/internal/common/util.go +++ b/internal/common/util.go @@ -3,6 +3,7 @@ package common import ( "fmt" "path/filepath" + "sort" "strings" ) @@ -20,7 +21,24 @@ func CleanToBasename(name string) string { return base } - +func NormalizeNameAliases(primary string, aliases []string) []string { + primary = CleanToBasename(primary) + seen := make(map[string]struct{}, len(aliases)+1) + out := make([]string, 0, len(aliases)) + for _, alias := range aliases { + name := CleanToBasename(alias) + if name == "" || name == primary { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + out = append(out, name) + } + sort.Strings(out) + return out +} // SchemeFromURL extracts the scheme from a URL string. func SchemeFromURL(raw string) string { @@ -30,8 +48,6 @@ func SchemeFromURL(raw string) string { return "" } - - // BucketToURL converts a bucket and key to an s3:// URL. func BucketToURL(bucket, key string) string { return fmt.Sprintf("s3://%s/%s", strings.TrimPrefix(bucket, "s3://"), strings.TrimPrefix(key, "/")) diff --git a/internal/core/converters.go b/internal/core/converters.go index ec8dee23..c97b6824 100644 --- a/internal/core/converters.go +++ b/internal/core/converters.go @@ -253,6 +253,7 @@ func InternalRecordToInternalObject(r internalapi.InternalRecord, now time.Time) if r.Name != nil && strings.TrimSpace(*r.Name) != "" { obj.Name = normalizedObjectNamePtr(r.Name) } + objectName := common.StringVal(obj.Name) if v := r.Version; v != nil { obj.Version = v } @@ -277,6 +278,7 @@ func InternalRecordToInternalObject(r internalapi.InternalRecord, now time.Time) } internalObj := models.InternalObject{ DrsObject: obj, + NameAliases: common.NormalizeNameAliases(objectName, common.DerefStringSlice(r.NameAliases)), Authorizations: authzMap, Properties: map[string]interface{}{}, } @@ -319,6 +321,7 @@ func InternalObjectToInternalRecord(obj models.InternalObject) internalapi.Inter CreatedTime: common.Ptr(obj.CreatedTime.Format(time.RFC3339)), Description: obj.Description, Name: obj.Name, + NameAliases: common.Ptr(common.NormalizeNameAliases(common.StringVal(obj.Name), obj.NameAliases)), Version: obj.Version, AccessMethods: obj.AccessMethods, } @@ -349,6 +352,7 @@ func InternalObjectToInternalRecordResponse(obj models.InternalObject) internala CreatedTime: rec.CreatedTime, Description: rec.Description, Name: rec.Name, + NameAliases: rec.NameAliases, Version: rec.Version, UpdatedTime: rec.UpdatedTime, Hashes: rec.Hashes, diff --git a/internal/core/object_canonicalization.go b/internal/core/object_canonicalization.go new file mode 100644 index 00000000..b8eafa01 --- /dev/null +++ b/internal/core/object_canonicalization.go @@ -0,0 +1,340 @@ +package core + +import ( + "sort" + "strings" + "time" + + "github.com/calypr/syfon/apigen/server/drs" + syfoncommon "github.com/calypr/syfon/common" + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/models" +) + +func canonicalizeProjectScopedObjects(objects []models.InternalObject, organization, project string) []models.InternalObject { + if len(objects) <= 1 { + return cloneObjects(objects) + } + + forcedResource := "" + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization != "" && project != "" { + if resource, err := syfoncommon.ResourcePath(organization, project); err == nil { + forcedResource = resource + } + } + + grouped := make(map[string][]models.InternalObject) + passthrough := make([]models.InternalObject, 0) + for _, obj := range objects { + key, ok := canonicalProjectChecksumKey(&obj, forcedResource) + if !ok { + passthrough = append(passthrough, cloneObject(obj)) + continue + } + grouped[key] = append(grouped[key], cloneObject(obj)) + } + + keys := make([]string, 0, len(grouped)) + for key := range grouped { + keys = append(keys, key) + } + sort.Strings(keys) + + out := make([]models.InternalObject, 0, len(keys)+len(passthrough)) + for _, key := range keys { + out = append(out, collapseCanonicalGroup(grouped[key])) + } + out = append(out, passthrough...) + sort.Slice(out, func(i, j int) bool { + if out[i].Id == out[j].Id { + return canonicalObjectSortTime(out[i]).After(canonicalObjectSortTime(out[j])) + } + return out[i].Id < out[j].Id + }) + return out +} + +func canonicalProjectChecksumKey(obj *models.InternalObject, forcedResource string) (string, bool) { + if obj == nil { + return "", false + } + sha, ok := common.CanonicalSHA256(obj.Checksums) + if !ok || strings.TrimSpace(sha) == "" { + return "", false + } + resource := strings.TrimSpace(forcedResource) + if resource == "" { + resources := projectScopeResources(obj) + if len(resources) != 1 { + return "", false + } + resource = resources[0] + } + return resource + "|" + sha, true +} + +func projectScopeResources(obj *models.InternalObject) []string { + resources := ObjectAccessResources(obj) + out := make([]string, 0, len(resources)) + for _, resource := range resources { + org, project, ok := syfoncommon.ResourceScope(resource) + if !ok || strings.TrimSpace(org) == "" || strings.TrimSpace(project) == "" { + continue + } + out = append(out, resource) + } + return syfoncommon.NormalizeAccessResources(out) +} + +func collapseCanonicalGroup(group []models.InternalObject) models.InternalObject { + if len(group) == 0 { + return models.InternalObject{} + } + + canonical := cloneObject(group[0]) + latest := canonical + for i := 1; i < len(group); i++ { + obj := cloneObject(group[i]) + if canonicalObjectNewer(obj, canonical) { + canonical = obj + } + if canonicalObjectNewer(obj, latest) { + latest = obj + } + } + + merged := cloneObject(canonical) + merged.Name = latest.Name + merged.Size = pickLatestNonZeroSize(group, canonical.Size) + merged.Description = pickLatestStringPtr(group, func(obj models.InternalObject) *string { return obj.Description }, canonical.Description) + merged.MimeType = pickLatestStringPtr(group, func(obj models.InternalObject) *string { return obj.MimeType }, canonical.MimeType) + merged.Version = pickLatestStringPtr(group, func(obj models.InternalObject) *string { return obj.Version }, canonical.Version) + updated := canonicalObjectSortTime(latest) + merged.UpdatedTime = &updated + merged.Checksums = mergeChecksums(group) + merged.AccessMethods = mergeAccessMethods(group) + controlled := mergeControlledAccess(group) + if len(controlled) > 0 { + merged.ControlledAccess = &controlled + merged.Authorizations = syfoncommon.ControlledAccessToAuthzMap(controlled) + } + merged.NameAliases = mergeNameAliases(merged.Name, group) + merged.Aliases = mergeStringPointerValues(func(obj models.InternalObject) []string { return common.DerefStringSlice(obj.Aliases) }, group) + merged.SelfUri = "drs://" + merged.Id + return merged +} + +func canonicalObjectNewer(a, b models.InternalObject) bool { + at := canonicalObjectSortTime(a) + bt := canonicalObjectSortTime(b) + if !at.Equal(bt) { + return at.After(bt) + } + return a.Id > b.Id +} + +func canonicalObjectSortTime(obj models.InternalObject) time.Time { + if obj.UpdatedTime != nil && !obj.UpdatedTime.IsZero() { + return obj.UpdatedTime.UTC() + } + return obj.CreatedTime.UTC() +} + +func cloneObjects(objects []models.InternalObject) []models.InternalObject { + out := make([]models.InternalObject, 0, len(objects)) + for _, obj := range objects { + out = append(out, cloneObject(obj)) + } + return out +} + +func cloneObject(obj models.InternalObject) models.InternalObject { + cloned := obj + cloned.Checksums = append([]drs.Checksum(nil), obj.Checksums...) + cloned.NameAliases = append([]string(nil), obj.NameAliases...) + if obj.AccessMethods != nil { + methods := append([]drs.AccessMethod(nil), (*obj.AccessMethods)...) + cloned.AccessMethods = &methods + } + if obj.ControlledAccess != nil { + controlled := append([]string(nil), (*obj.ControlledAccess)...) + cloned.ControlledAccess = &controlled + } + if obj.Aliases != nil { + aliases := append([]string(nil), (*obj.Aliases)...) + cloned.Aliases = &aliases + } + if obj.Authorizations != nil { + authz := make(map[string][]string, len(obj.Authorizations)) + for org, projects := range obj.Authorizations { + authz[org] = append([]string(nil), projects...) + } + cloned.Authorizations = authz + } + if obj.Properties != nil { + props := make(map[string]interface{}, len(obj.Properties)) + for key, value := range obj.Properties { + props[key] = value + } + cloned.Properties = props + } + return cloned +} + +func mergeChecksums(group []models.InternalObject) []drs.Checksum { + seen := make(map[string]struct{}) + merged := make([]drs.Checksum, 0) + for _, obj := range group { + for _, checksum := range obj.Checksums { + key := checksum.Type + "|" + checksum.Checksum + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + merged = append(merged, checksum) + } + } + sort.Slice(merged, func(i, j int) bool { + if merged[i].Type == merged[j].Type { + return merged[i].Checksum < merged[j].Checksum + } + return merged[i].Type < merged[j].Type + }) + return merged +} + +func mergeAccessMethods(group []models.InternalObject) *[]drs.AccessMethod { + seen := make(map[string]struct{}) + methods := make([]drs.AccessMethod, 0) + for _, obj := range group { + if obj.AccessMethods == nil { + continue + } + for _, method := range *obj.AccessMethods { + url := "" + if method.AccessUrl != nil { + url = method.AccessUrl.Url + } + key := string(method.Type) + "|" + url + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + methods = append(methods, method) + } + } + if len(methods) == 0 { + return nil + } + sort.Slice(methods, func(i, j int) bool { + iURL := "" + jURL := "" + if methods[i].AccessUrl != nil { + iURL = methods[i].AccessUrl.Url + } + if methods[j].AccessUrl != nil { + jURL = methods[j].AccessUrl.Url + } + if methods[i].Type == methods[j].Type { + return iURL < jURL + } + return methods[i].Type < methods[j].Type + }) + return &methods +} + +func mergeControlledAccess(group []models.InternalObject) []string { + resources := make([]string, 0) + for _, obj := range group { + resources = append(resources, ObjectAccessResources(&obj)...) + } + return syfoncommon.NormalizeAccessResources(resources) +} + +func mergeNameAliases(primary *string, group []models.InternalObject) []string { + candidates := make([]string, 0) + for _, obj := range group { + if obj.Name != nil { + candidates = append(candidates, *obj.Name) + } + candidates = append(candidates, obj.NameAliases...) + } + return common.NormalizeNameAliases(common.StringVal(primary), candidates) +} + +func pickLatestNonZeroSize(group []models.InternalObject, fallback int64) int64 { + best := fallback + var bestTime time.Time + for _, obj := range group { + if obj.Size <= 0 { + continue + } + when := canonicalObjectSortTime(obj) + if best <= 0 || when.After(bestTime) { + best = obj.Size + bestTime = when + } + } + return best +} + +func pickLatestStringPtr(group []models.InternalObject, getter func(models.InternalObject) *string, fallback *string) *string { + best := fallback + var bestTime time.Time + for _, obj := range group { + value := getter(obj) + if value == nil || strings.TrimSpace(*value) == "" { + continue + } + when := canonicalObjectSortTime(obj) + if best == nil || when.After(bestTime) { + trimmed := strings.TrimSpace(*value) + best = &trimmed + bestTime = when + } + } + return best +} + +func mergeStringPointerValues(getter func(models.InternalObject) []string, group []models.InternalObject) *[]string { + seen := make(map[string]struct{}) + values := make([]string, 0) + for _, obj := range group { + for _, value := range getter(obj) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + values = append(values, trimmed) + } + } + if len(values) == 0 { + return nil + } + sort.Strings(values) + return &values +} + +func uniqueStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + sort.Strings(out) + return out +} diff --git a/internal/core/object_mutations.go b/internal/core/object_mutations.go index 3c26a0a0..21cc1d2c 100644 --- a/internal/core/object_mutations.go +++ b/internal/core/object_mutations.go @@ -198,7 +198,192 @@ func (m *ObjectManager) RegisterObjects(ctx context.Context, objs []models.Inter if err := m.bulkObjectMethodError(ctx, objs, objectMethodCreate); err != nil { return err } - return m.db.RegisterObjects(ctx, objs) + canonical, aliases, err := m.canonicalizeRegistrationObjects(ctx, objs) + if err != nil { + return err + } + if err := m.db.RegisterObjects(ctx, canonical); err != nil { + return err + } + for aliasID, canonicalID := range aliases { + if err := m.db.CreateObjectAlias(ctx, aliasID, canonicalID); err != nil { + return err + } + } + return nil +} + +func (m *ObjectManager) CollapseProjectChecksumDuplicates(ctx context.Context, organization, project string) (int, error) { + ids, err := m.db.ListObjectIDsByScope(ctx, organization, project) + if err != nil { + return 0, err + } + objects, err := m.db.GetBulkObjects(ctx, ids) + if err != nil { + return 0, err + } + if err := m.bulkObjectMethodError(ctx, objects, objectMethodUpdate); err != nil { + return 0, err + } + + grouped := make(map[string][]models.InternalObject) + for _, obj := range objects { + key, ok := canonicalProjectChecksumKey(&obj, "") + if !ok { + continue + } + grouped[key] = append(grouped[key], cloneObject(obj)) + } + + merged := make([]models.InternalObject, 0, len(grouped)) + aliasMap := make(map[string]string) + toDelete := make([]string, 0) + keys := make([]string, 0, len(grouped)) + for key, group := range grouped { + if len(group) < 2 { + continue + } + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + group := grouped[key] + canonical := collapseCanonicalGroup(group) + merged = append(merged, canonical) + for _, obj := range group { + if obj.Id == canonical.Id { + continue + } + aliasMap[obj.Id] = canonical.Id + toDelete = append(toDelete, obj.Id) + } + } + + if len(merged) == 0 { + return 0, nil + } + if err := m.db.RegisterObjects(ctx, merged); err != nil { + return 0, err + } + for aliasID, canonicalID := range aliasMap { + if err := m.db.CreateObjectAlias(ctx, aliasID, canonicalID); err != nil { + return 0, err + } + } + if err := m.db.BulkDeleteObjects(ctx, uniqueStrings(toDelete)); err != nil { + return 0, err + } + return len(aliasMap), nil +} + +func (m *ObjectManager) canonicalizeRegistrationObjects(ctx context.Context, objs []models.InternalObject) ([]models.InternalObject, map[string]string, error) { + if len(objs) == 0 { + return nil, nil, nil + } + + checksums := make([]string, 0, len(objs)) + seenChecksums := make(map[string]struct{}, len(objs)) + for _, obj := range objs { + if sha, ok := common.CanonicalSHA256(obj.Checksums); ok { + if _, seen := seenChecksums[sha]; seen { + continue + } + seenChecksums[sha] = struct{}{} + checksums = append(checksums, sha) + } + } + + existingByChecksum, err := m.db.GetObjectsByChecksums(ctx, checksums) + if err != nil { + return nil, nil, err + } + + type registrationGroup struct { + existingCount int + objects []models.InternalObject + } + groups := make(map[string]*registrationGroup) + passthrough := make([]models.InternalObject, 0) + + for _, obj := range objs { + sha, ok := common.CanonicalSHA256(obj.Checksums) + if !ok { + passthrough = append(passthrough, cloneObject(obj)) + continue + } + resources := projectScopeResources(&obj) + if len(resources) != 1 { + passthrough = append(passthrough, cloneObject(obj)) + continue + } + key := resources[0] + "|" + sha + group, ok := groups[key] + if !ok { + group = ®istrationGroup{} + for _, existing := range existingByChecksum[sha] { + existingKey, ok := canonicalProjectChecksumKey(&existing, "") + if ok && existingKey == key { + group.objects = append(group.objects, cloneObject(existing)) + } + } + group.existingCount = len(group.objects) + groups[key] = group + } + group.objects = append(group.objects, cloneObject(obj)) + } + + keys := make([]string, 0, len(groups)) + for key := range groups { + keys = append(keys, key) + } + sort.Strings(keys) + + merged := make([]models.InternalObject, 0, len(keys)+len(passthrough)) + aliasMap := make(map[string]string) + + for _, key := range keys { + state := groups[key] + group := state.objects + if len(group) == 0 { + continue + } + + hasExisting := state.existingCount > 0 + canonical := cloneObject(group[0]) + latest := canonical + if hasExisting { + canonical = cloneObject(group[0]) + } + for i, obj := range group { + if hasExisting && i < state.existingCount && canonicalObjectNewer(obj, canonical) { + canonical = cloneObject(obj) + } + if canonicalObjectNewer(obj, latest) { + latest = cloneObject(obj) + } + } + if !hasExisting { + canonical = cloneObject(latest) + } + + collapsed := collapseCanonicalGroup(group) + collapsed.Id = canonical.Id + collapsed.SelfUri = "drs://" + canonical.Id + collapsed.CreatedTime = canonical.CreatedTime + collapsed.Name = latest.Name + collapsed.NameAliases = common.NormalizeNameAliases(common.StringVal(collapsed.Name), append(collapsed.NameAliases, common.StringVal(canonical.Name))) + merged = append(merged, collapsed) + + for _, obj := range group { + if obj.Id == collapsed.Id || strings.TrimSpace(obj.Id) == "" { + continue + } + aliasMap[obj.Id] = collapsed.Id + } + } + + merged = append(merged, passthrough...) + return merged, aliasMap, nil } func (m *ObjectManager) ReplaceObjects(ctx context.Context, objs []models.InternalObject) error { diff --git a/internal/core/object_mutations_test.go b/internal/core/object_mutations_test.go index 0d12ecee..78676e0e 100644 --- a/internal/core/object_mutations_test.go +++ b/internal/core/object_mutations_test.go @@ -2,10 +2,13 @@ package core import ( "context" + "slices" "testing" + "time" "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/db" + "github.com/calypr/syfon/internal/models" ) func ptr[T any](v T) *T { return &v } @@ -97,3 +100,88 @@ func TestBulkDeleteObjects_DeletesAuthorizedObjects(t *testing.T) { t.Fatalf("expected object to be deleted") } } + +func TestRegisterObjects_CanonicalizesProjectChecksumDuplicates(t *testing.T) { + database := db.NewInMemoryDB() + om := NewObjectManager(database, nil) + now := time.Now().UTC() + later := now.Add(time.Minute) + accessURL1 := "s3://bucket/original" + accessURL2 := "s3://bucket/renamed" + + first := models.InternalObject{ + Authorizations: map[string][]string{"org": {"proj"}}, + DrsObject: drs.DrsObject{ + Id: "did-1", + Name: ptr("original.tsv"), + Size: 42, + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: accessURL1}, + }}, + }, + } + second := models.InternalObject{ + Authorizations: map[string][]string{"org": {"proj"}}, + DrsObject: drs.DrsObject{ + Id: "did-2", + Name: ptr("renamed.tsv"), + Size: 42, + CreatedTime: later, + UpdatedTime: &later, + Checksums: first.Checksums, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: accessURL2}, + }}, + }, + } + + if err := om.RegisterObjects(context.Background(), []models.InternalObject{first}); err != nil { + t.Fatalf("RegisterObjects(first) error: %v", err) + } + if err := om.RegisterObjects(context.Background(), []models.InternalObject{second}); err != nil { + t.Fatalf("RegisterObjects(second) error: %v", err) + } + + records, err := om.GetObjectsByChecksum(context.Background(), first.Checksums[0].Checksum, "") + if err != nil { + t.Fatalf("GetObjectsByChecksum error: %v", err) + } + if len(records) != 1 { + t.Fatalf("expected 1 canonical record, got %d", len(records)) + } + if records[0].Id != "did-1" { + t.Fatalf("expected canonical did-1, got %q", records[0].Id) + } + if got := records[0].Name; got == nil || *got != "renamed.tsv" { + t.Fatalf("expected latest name renamed.tsv, got %+v", got) + } + if !slices.Equal(records[0].NameAliases, []string{"original.tsv"}) { + t.Fatalf("unexpected name aliases: %#v", records[0].NameAliases) + } + + aliasObj, err := om.GetObject(context.Background(), "did-2", "") + if err != nil { + t.Fatalf("GetObject(alias) error: %v", err) + } + if aliasObj.Id != "did-2" { + t.Fatalf("expected alias lookup to preserve requested did, got %q", aliasObj.Id) + } + scopeIDs, err := om.ListObjectIDsByScope(context.Background(), "org", "proj", "") + if err != nil { + t.Fatalf("ListObjectIDsByScope error: %v", err) + } + if !slices.Equal(scopeIDs, []string{"did-1"}) { + t.Fatalf("unexpected scoped ids: %#v", scopeIDs) + } +} diff --git a/internal/core/object_reads.go b/internal/core/object_reads.go index b310b01a..98272afe 100644 --- a/internal/core/object_reads.go +++ b/internal/core/object_reads.go @@ -2,8 +2,10 @@ package core import ( "context" + "log" "sort" "strings" + "time" syfoncommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/authz" @@ -40,45 +42,23 @@ func (m *ObjectManager) GetObject(ctx context.Context, ident string, requiredMet } func (m *ObjectManager) lookupObjectByChecksum(ctx context.Context, ident string, requiredMethod string) (*models.InternalObject, bool, error) { - if ids, optimized, err := m.authorizedChecksumIDs(ctx, ident, requiredMethod); err != nil { - return nil, false, err - } else if optimized { - if len(ids) == 0 { - if strings.TrimSpace(requiredMethod) != "" { - if allMatches, ok, err := m.authorizedChecksumIDs(ctx, ident, ""); err != nil { - return nil, false, err - } else if ok && len(allMatches) > 0 { - return nil, true, common.ErrUnauthorized - } - } - return nil, false, nil - } - objects, err := m.db.GetBulkObjects(ctx, ids[:1]) - if err != nil { - return nil, false, err - } - if len(objects) > 0 { - return &objects[0], true, nil - } - return nil, false, nil - } - - byChecksum, err := m.db.GetObjectsByChecksum(ctx, ident) + byChecksum, err := m.GetObjectsByChecksum(ctx, ident, requiredMethod) if err != nil { return nil, false, err } if len(byChecksum) == 0 { - return nil, false, nil - } - for i := range byChecksum { - if m.hasObjectMethod(ctx, &byChecksum[i], requiredMethod) { - return &byChecksum[i], true, nil + if strings.TrimSpace(requiredMethod) != "" { + allMatches, err := m.GetObjectsByChecksum(ctx, ident, "") + if err != nil { + return nil, false, err + } + if len(allMatches) > 0 { + return nil, true, common.ErrUnauthorized + } } + return nil, false, nil } - if requiredMethod != "" { - return nil, true, common.ErrUnauthorized - } - return nil, false, nil + return &byChecksum[0], true, nil } func (m *ObjectManager) lookupObjectByID(ctx context.Context, ident string) (*models.InternalObject, bool, error) { @@ -132,26 +112,17 @@ func (m *ObjectManager) GetObjectsByChecksums(ctx context.Context, hashes []stri } filtered := make(map[string][]models.InternalObject, len(objectsByChecksum)) for checksum, objects := range objectsByChecksum { - filtered[checksum] = m.filterObjectsByMethod(ctx, objects, requiredMethod) + filtered[checksum] = canonicalizeProjectScopedObjects(m.filterObjectsByMethod(ctx, objects, requiredMethod), "", "") } return filtered, nil } func (m *ObjectManager) GetObjectsByChecksum(ctx context.Context, checksum string, requiredMethod string) ([]models.InternalObject, error) { - if ids, optimized, err := m.authorizedChecksumIDs(ctx, checksum, requiredMethod); err != nil { - return nil, err - } else if optimized { - if len(ids) == 0 { - return []models.InternalObject{}, nil - } - return m.db.GetBulkObjects(ctx, ids) - } - objects, err := m.db.GetObjectsByChecksum(ctx, checksum) if err != nil { return nil, err } - return m.filterObjectsByMethod(ctx, objects, requiredMethod), nil + return canonicalizeProjectScopedObjects(m.filterObjectsByMethod(ctx, objects, requiredMethod), "", ""), nil } func (m *ObjectManager) GetBulkObjects(ctx context.Context, ids []string, requiredMethod string) ([]models.InternalObject, error) { @@ -162,16 +133,23 @@ func (m *ObjectManager) GetBulkObjects(ctx context.Context, ids []string, requir return m.filterObjectsByMethod(ctx, objects, requiredMethod), nil } +func (m *ObjectManager) PrepareScopedObjects(ctx context.Context, objects []models.InternalObject, organization, project, requiredMethod string) ([]models.InternalObject, error) { + started := time.Now() + expanded, err := m.expandProjectChecksumSiblingObjects(ctx, objects, organization, project) + if err != nil { + return nil, err + } + filtered := m.filterObjectsByMethod(ctx, expanded, requiredMethod) + canonicalStart := time.Now() + canonical := canonicalizeProjectScopedObjects(filtered, organization, project) + log.Printf("INFO: syfon_prepare_scoped_objects organization=%s project=%s input_count=%d expanded_count=%d filtered_count=%d output_count=%d canonicalize_scoped_objects_ms=%d duration_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), len(objects), len(expanded), len(filtered), len(canonical), time.Since(canonicalStart).Milliseconds(), time.Since(started).Milliseconds()) + return canonical, nil +} + func (m *ObjectManager) ListObjectIDsPageByChecksum(ctx context.Context, checksum, checksumType, organization, project, requiredMethod, startAfter string, limit, offset int) ([]string, error) { if limit <= 0 { return []string{}, nil } - if pager, ok := m.db.(db.ObjectChecksumPageLister); ok && strings.EqualFold(strings.TrimSpace(requiredMethod), objectMethodRead) { - resources, includeUnscoped, restrictToResources, ok := m.readableChecksumFilter(ctx, organization, project) - if ok { - return pager.ListObjectIDsPageByChecksum(ctx, checksum, checksumType, organization, project, startAfter, limit, offset, resources, includeUnscoped, restrictToResources) - } - } objects, err := m.GetObjectsByChecksum(ctx, checksum, requiredMethod) if err != nil { @@ -205,17 +183,6 @@ func (m *ObjectManager) ListObjectIDsPageByScope(ctx context.Context, organizati if limit <= 0 { return []string{}, nil } - if pager, ok := m.db.(db.ObjectIDPageLister); ok { - if strings.TrimSpace(organization) == "" && strings.EqualFold(strings.TrimSpace(requiredMethod), objectMethodRead) { - if ids, ok, err := m.listReadableObjectIDsPage(ctx, startAfter, limit, offset); ok || err != nil { - return ids, err - } - return pager.ListObjectIDsPageByScope(ctx, organization, project, startAfter, limit, offset) - } - if strings.EqualFold(strings.TrimSpace(requiredMethod), objectMethodRead) && m.canPageScopeRead(ctx, organization, project) { - return pager.ListObjectIDsPageByScope(ctx, organization, project, startAfter, limit, offset) - } - } ids, err := m.ListObjectIDsByScope(ctx, organization, project, requiredMethod) if err != nil { @@ -288,15 +255,20 @@ func (m *ObjectManager) ListObjectIDsByScope(ctx context.Context, organization, return ids, err } } + listStart := time.Now() ids, err := m.db.ListObjectIDsByScope(ctx, organization, project) if err != nil { return nil, err } + log.Printf("INFO: syfon_list_object_ids_by_scope organization=%s project=%s ids=%d list_scope_ids_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), len(ids), time.Since(listStart).Milliseconds()) objects, err := m.db.GetBulkObjects(ctx, ids) if err != nil { return nil, err } - filtered := m.filterObjectsByMethod(ctx, objects, requiredMethod) + filtered, err := m.PrepareScopedObjects(ctx, objects, organization, project, requiredMethod) + if err != nil { + return nil, err + } out := make([]string, 0, len(filtered)) for _, obj := range filtered { out = append(out, obj.Id) @@ -325,7 +297,89 @@ func (m *ObjectManager) ListObjectsByScope(ctx context.Context, organization, pr if err != nil { return nil, err } - return m.filterObjectsByMethod(ctx, objects, requiredMethod), nil + return m.PrepareScopedObjects(ctx, objects, organization, project, requiredMethod) +} + +func (m *ObjectManager) expandProjectChecksumSiblingObjects(ctx context.Context, objects []models.InternalObject, organization, project string) ([]models.InternalObject, error) { + if len(objects) == 0 { + return []models.InternalObject{}, nil + } + + wantedKeys := make(map[string]struct{}, len(objects)) + checksums := make([]string, 0, len(objects)) + seenChecksums := make(map[string]struct{}, len(objects)) + for _, obj := range objects { + key, ok := canonicalProjectChecksumKey(&obj, "") + if !ok { + continue + } + wantedKeys[key] = struct{}{} + sha, _ := common.CanonicalSHA256(obj.Checksums) + if _, seen := seenChecksums[sha]; seen { + continue + } + seenChecksums[sha] = struct{}{} + checksums = append(checksums, sha) + } + if len(wantedKeys) == 0 || len(checksums) == 0 || strings.TrimSpace(organization) == "" || strings.TrimSpace(project) == "" { + return objects, nil + } + + listStart := time.Now() + idsByChecksum, err := m.db.ListScopedObjectIDsByChecksums(ctx, organization, project, checksums) + if err != nil { + return nil, err + } + listDuration := time.Since(listStart) + + expanded := make([]models.InternalObject, 0, len(objects)) + seenIDs := make(map[string]struct{}, len(objects)) + missingIDs := make([]string, 0) + missingSeen := make(map[string]struct{}) + for _, obj := range objects { + if _, seen := seenIDs[obj.Id]; seen { + continue + } + seenIDs[obj.Id] = struct{}{} + expanded = append(expanded, obj) + } + for _, sha := range checksums { + for _, id := range idsByChecksum[sha] { + if _, seen := seenIDs[id]; seen { + continue + } + if _, queued := missingSeen[id]; queued { + continue + } + missingSeen[id] = struct{}{} + missingIDs = append(missingIDs, id) + } + } + + hydrateStart := time.Now() + if len(missingIDs) > 0 { + siblings, err := m.db.GetBulkObjects(ctx, missingIDs) + if err != nil { + return nil, err + } + for _, obj := range siblings { + key, ok := canonicalProjectChecksumKey(&obj, "") + if !ok { + continue + } + if _, wanted := wantedKeys[key]; !wanted { + continue + } + if _, seen := seenIDs[obj.Id]; seen { + continue + } + seenIDs[obj.Id] = struct{}{} + expanded = append(expanded, obj) + } + } + hydrateDuration := time.Since(hydrateStart) + log.Printf("INFO: syfon_expand_scoped_checksum_siblings organization=%s project=%s checksums=%d input_count=%d missing_ids=%d output_count=%d list_scoped_sibling_ids_ms=%d hydrate_missing_siblings_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), len(checksums), len(objects), len(missingIDs), len(expanded), listDuration.Milliseconds(), hydrateDuration.Milliseconds()) + return expanded, nil } func objectHasAccessURL(obj *models.InternalObject, objectURL string) bool { diff --git a/internal/core/object_reads_test.go b/internal/core/object_reads_test.go index d66aad41..50ea673a 100644 --- a/internal/core/object_reads_test.go +++ b/internal/core/object_reads_test.go @@ -2,10 +2,13 @@ package core import ( "context" + "slices" "testing" + "time" "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/db" + "github.com/calypr/syfon/internal/db/sqlite" "github.com/calypr/syfon/internal/models" ) @@ -126,6 +129,105 @@ func TestObjectMatchesScope(t *testing.T) { } } +type trackingDB struct { + db.DatabaseInterface + bulkCalls [][]string +} + +func (t *trackingDB) GetBulkObjects(ctx context.Context, ids []string) ([]models.InternalObject, error) { + copyIDs := append([]string(nil), ids...) + t.bulkCalls = append(t.bulkCalls, copyIDs) + return t.DatabaseInterface.GetBulkObjects(ctx, ids) +} + +func TestPrepareScopedObjects_HydratesOnlyMissingSiblingIDs(t *testing.T) { + base, err := sqlite.NewSqliteDB(":memory:") + if err != nil { + t.Fatalf("NewSqliteDB failed: %v", err) + } + tracked := &trackingDB{DatabaseInterface: base} + om := NewObjectManager(tracked, nil) + checksum := "6666666666666666666666666666666666666666666666666666666666666666" + controlled := []string{"/organization/org/project/proj"} + + for _, obj := range []models.InternalObject{ + { + Authorizations: map[string][]string{"org": {"proj"}}, + DrsObject: drs.DrsObject{ + Id: "dup-a", + CreatedTime: drsISOTime("2026-01-01T00:00:00Z"), + UpdatedTime: ptrTime("2026-01-01T00:00:00Z"), + Checksums: []drs.Checksum{{Type: "sha256", Checksum: checksum}}, + ControlledAccess: &controlled, + AccessMethods: &[]drs.AccessMethod{{ + Type: drs.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/dup-a"}, + }}, + }, + }, + { + Authorizations: map[string][]string{"org": {"proj"}}, + DrsObject: drs.DrsObject{ + Id: "dup-b", + CreatedTime: drsISOTime("2026-01-02T00:00:00Z"), + UpdatedTime: ptrTime("2026-01-02T00:00:00Z"), + Checksums: []drs.Checksum{{Type: "sha256", Checksum: checksum}}, + ControlledAccess: &controlled, + AccessMethods: &[]drs.AccessMethod{{ + Type: drs.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/dup-b"}, + }}, + }, + }, + } { + if err := tracked.CreateObject(context.Background(), &obj); err != nil { + t.Fatalf("CreateObject failed: %v", err) + } + } + + initial, err := tracked.GetBulkObjects(context.Background(), []string{"dup-a"}) + if err != nil { + t.Fatalf("GetBulkObjects failed: %v", err) + } + tracked.bulkCalls = nil + + prepared, err := om.PrepareScopedObjects(context.Background(), initial, "org", "proj", "") + if err != nil { + t.Fatalf("PrepareScopedObjects failed: %v", err) + } + if len(prepared) != 1 { + t.Fatalf("expected 1 canonical record, got %d", len(prepared)) + } + if prepared[0].AccessMethods == nil || len(*prepared[0].AccessMethods) != 2 { + t.Fatalf("expected merged access methods, got %+v", prepared[0].AccessMethods) + } + if len(tracked.bulkCalls) != 1 { + t.Fatalf("expected 1 sibling hydration call, got %d", len(tracked.bulkCalls)) + } + if !slices.Equal(tracked.bulkCalls[0], []string{"dup-b"}) { + t.Fatalf("expected only missing sibling id to be hydrated, got %+v", tracked.bulkCalls[0]) + } +} + +func drsISOTime(raw string) time.Time { + tm, err := time.Parse(time.RFC3339, raw) + if err != nil { + panic(err) + } + return tm +} + +func ptrTime(raw string) *time.Time { + tm := drsISOTime(raw) + return &tm +} + func TestReadableChecksumFilter(t *testing.T) { database := db.NewInMemoryDB() om := NewObjectManager(database, nil) diff --git a/internal/core/object_storage_inventory.go b/internal/core/object_storage_inventory.go index a7354a37..a9692210 100644 --- a/internal/core/object_storage_inventory.go +++ b/internal/core/object_storage_inventory.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log" "path" "sort" "strings" @@ -31,6 +32,70 @@ type StorageBucketObject struct { LastModTime time.Time } +type StoragePrefixListOptions struct { + IncludeHead bool + ExactPrefix bool + MaxKeys int32 +} + +type ProjectStorageInspectMode string + +const ( + ProjectStorageInspectItems ProjectStorageInspectMode = "items" + ProjectStorageInspectExists ProjectStorageInspectMode = "exists" + ProjectStorageInspectSummary ProjectStorageInspectMode = "summary" +) + +type ProjectStorageSummary struct { + Provider string + Bucket string + Prefix string + ObjectURL string + Exists bool + ObjectCount int + TotalBytes int64 + ComputedAt time.Time + Mode ProjectStorageInspectMode +} + +type ProjectStorageInspectResult struct { + Summary ProjectStorageSummary + Items []StorageBucketObject +} + +type ProjectStorageInspectOptions struct { + Mode ProjectStorageInspectMode + IncludeHead bool + PathPrefix string +} + +type StorageListValidationRequest struct { + ID string + ObjectURL string + ExpectedSizeBytes *int64 + ExpectedName string +} + +type StorageListValidationResult struct { + ID string + ObjectURL string + Provider string + Bucket string + Key string + Path string + Exists bool + Status StorageProbeStatus + Error string + ErrorKind string + SizeBytes *int64 + ETag string + LastModTime time.Time + ValidationStatus StorageValidationStatus + SizeMatch *bool + NameMatch *bool + ValidationMismatches []string +} + type resolvedStorageScopeTarget struct { provider string bucket string @@ -45,6 +110,16 @@ type ProjectStorageDeleteResult struct { } func (m *ObjectManager) SetS3PrefixLister(fn func(context.Context, models.S3Credential, string, string, bool) ([]StorageBucketObject, error)) { + if fn == nil { + m.listS3Prefix = defaultS3PrefixLister + return + } + m.listS3Prefix = func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options StoragePrefixListOptions) ([]StorageBucketObject, error) { + return fn(ctx, cred, bucket, prefix, options.IncludeHead) + } +} + +func (m *ObjectManager) SetS3PrefixListerWithOptions(fn func(context.Context, models.S3Credential, string, string, StoragePrefixListOptions) ([]StorageBucketObject, error)) { if fn == nil { m.listS3Prefix = defaultS3PrefixLister return @@ -53,15 +128,65 @@ func (m *ObjectManager) SetS3PrefixLister(fn func(context.Context, models.S3Cred } func (m *ObjectManager) ListProjectStorageObjects(ctx context.Context, organization, project string, includeHead bool) ([]StorageBucketObject, error) { + result, err := m.InspectProjectStorage(ctx, organization, project, ProjectStorageInspectOptions{Mode: ProjectStorageInspectItems, IncludeHead: includeHead}) + if err != nil { + return nil, err + } + return result.Items, nil +} + +func (m *ObjectManager) InspectProjectStorage(ctx context.Context, organization, project string, inspectOptions ProjectStorageInspectOptions) (*ProjectStorageInspectResult, error) { + started := time.Now() ctx = WithStorageInspectCache(ctx) target, err := m.resolveProjectStorageScopeTarget(ctx, organization, project) if err != nil { + log.Printf("INFO: syfon_project_storage_inspect_done organization=%s project=%s mode=%s path_prefix=%q duration_ms=%d error=%q", organization, project, inspectOptions.Mode, inspectOptions.PathPrefix, time.Since(started).Milliseconds(), err.Error()) return nil, err } - items, err := m.listS3Prefix(ctx, target.cred, target.bucket, target.prefix, includeHead) + target = target.withPathPrefix(inspectOptions.PathPrefix) + normalizedMode := normalizeProjectStorageInspectMode(inspectOptions.Mode) + options := StoragePrefixListOptions{IncludeHead: inspectOptions.IncludeHead} + if normalizedMode == ProjectStorageInspectExists { + options.MaxKeys = 1 + } + items, err := m.listS3Prefix(ctx, target.cred, target.bucket, target.prefix, options) if err != nil { + log.Printf("INFO: syfon_project_storage_inspect_done organization=%s project=%s mode=%s path_prefix=%q bucket=%s prefix=%q max_keys=%d include_head=%t duration_ms=%d error=%q", organization, project, normalizedMode, inspectOptions.PathPrefix, target.bucket, target.prefix, options.MaxKeys, options.IncludeHead, time.Since(started).Milliseconds(), err.Error()) return nil, err } + out := normalizeStorageBucketObjects(items, target) + summary := summarizeProjectStorageObjects(out, target, normalizedMode) + if normalizedMode != ProjectStorageInspectItems { + out = []StorageBucketObject{} + } + log.Printf("INFO: syfon_project_storage_inspect_done organization=%s project=%s mode=%s path_prefix=%q bucket=%s prefix=%q max_keys=%d include_head=%t exists=%t object_count=%d returned_items=%d total_bytes=%d duration_ms=%d", organization, project, normalizedMode, inspectOptions.PathPrefix, target.bucket, target.prefix, options.MaxKeys, options.IncludeHead, summary.Exists, summary.ObjectCount, len(out), summary.TotalBytes, time.Since(started).Milliseconds()) + return &ProjectStorageInspectResult{Summary: summary, Items: out}, nil +} + +func (target *resolvedStorageScopeTarget) withPathPrefix(pathPrefix string) *resolvedStorageScopeTarget { + trimmed := strings.Trim(strings.TrimSpace(pathPrefix), "/") + if target == nil || trimmed == "" { + return target + } + copyTarget := *target + if copyTarget.prefix == "" { + copyTarget.prefix = trimmed + } else { + copyTarget.prefix = strings.Trim(strings.TrimSpace(copyTarget.prefix), "/") + "/" + trimmed + } + return ©Target +} + +func normalizeProjectStorageInspectMode(mode ProjectStorageInspectMode) ProjectStorageInspectMode { + switch mode { + case ProjectStorageInspectExists, ProjectStorageInspectSummary: + return mode + default: + return ProjectStorageInspectItems + } +} + +func normalizeStorageBucketObjects(items []StorageBucketObject, target *resolvedStorageScopeTarget) []StorageBucketObject { out := make([]StorageBucketObject, 0, len(items)) for _, item := range items { entry := item @@ -77,7 +202,24 @@ func (m *ObjectManager) ListProjectStorageObjects(ctx context.Context, organizat sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) - return out, nil + return out +} + +func summarizeProjectStorageObjects(items []StorageBucketObject, target *resolvedStorageScopeTarget, mode ProjectStorageInspectMode) ProjectStorageSummary { + summary := ProjectStorageSummary{ + Provider: target.provider, + Bucket: target.bucket, + Prefix: strings.Trim(strings.TrimSpace(target.prefix), "/"), + ObjectURL: common.BucketToURL(target.bucket, strings.Trim(strings.TrimSpace(target.prefix), "/")), + Exists: len(items) > 0, + ObjectCount: len(items), + ComputedAt: time.Now().UTC(), + Mode: mode, + } + for _, item := range items { + summary.TotalBytes += item.SizeBytes + } + return summary } func (m *ObjectManager) resolveProjectStorageScopeTarget(ctx context.Context, organization, project string) (*resolvedStorageScopeTarget, error) { @@ -228,7 +370,198 @@ func uniqueStorageObjectURLs(values []string) []string { return out } -func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket string, prefix string, includeHead bool) ([]StorageBucketObject, error) { +func (m *ObjectManager) ListValidateStorageObjects(ctx context.Context, items []StorageListValidationRequest) []StorageListValidationResult { + started := time.Now() + ctx = WithStorageInspectCache(ctx) + if len(items) == 0 { + log.Printf("INFO: syfon_bulk_list_validate_done items=0 duration_ms=0") + return []StorageListValidationResult{} + } + results := make([]StorageListValidationResult, len(items)) + workers := len(items) + if workers > maxStorageInspectWorkers { + workers = maxStorageInspectWorkers + } + type workItem struct { + index int + req StorageListValidationRequest + } + workCh := make(chan workItem) + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for work := range workCh { + results[work.index] = m.listValidateStorageObject(ctx, work.req) + } + }() + } + for index, item := range items { + workCh <- workItem{index: index, req: item} + } + close(workCh) + wg.Wait() + statusCounts := map[StorageProbeStatus]int{} + validationCounts := map[StorageValidationStatus]int{} + for _, result := range results { + statusCounts[result.Status]++ + validationCounts[result.ValidationStatus]++ + } + log.Printf( + "INFO: syfon_bulk_list_validate_done items=%d present=%d not_found=%d invalid=%d error=%d matched=%d mismatched=%d unverifiable=%d duration_ms=%d", + len(items), + statusCounts[StorageProbeStatusPresent], + statusCounts[StorageProbeStatusNotFound], + statusCounts[StorageProbeStatusInvalid], + statusCounts[StorageProbeStatusError], + validationCounts[StorageValidationMatched], + validationCounts[StorageValidationMismatched], + validationCounts[StorageValidationUnverifiable], + time.Since(started).Milliseconds(), + ) + return results +} + +func (m *ObjectManager) listValidateStorageObject(ctx context.Context, req StorageListValidationRequest) StorageListValidationResult { + result := StorageListValidationResult{ + ID: strings.TrimSpace(req.ID), + ObjectURL: strings.TrimSpace(req.ObjectURL), + Status: StorageProbeStatusError, + ValidationStatus: StorageValidationNotRequested, + } + target, ok, err := m.storageTargetFromURL(ctx, req.ObjectURL) + if err != nil { + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = strings.TrimSpace(err.Error()) + result.ValidationStatus = storageListValidationStatusForError(req) + return result + } + if !ok || target.provider != common.S3Provider { + log.Printf("INFO: syfon_bulk_list_validate_resolve id=%s object_url=%q status=invalid provider=%s bucket=%s key=%q", result.ID, result.ObjectURL, target.provider, target.bucket, target.key) + result.Status = StorageProbeStatusInvalid + result.ErrorKind = string(StorageInspectInvalidInput) + result.Error = "object_url must be a valid s3://bucket/key URL" + result.ValidationStatus = storageListValidationStatusForError(req) + return result + } + result.Provider = common.S3Provider + result.Bucket = target.bucket + result.Key = target.key + result.Path = path.Base(target.key) + log.Printf("INFO: syfon_bulk_list_validate_resolve id=%s object_url=%q bucket=%s key=%q expected_size_set=%t expected_name=%q", result.ID, result.ObjectURL, target.bucket, target.key, req.ExpectedSizeBytes != nil, strings.TrimSpace(req.ExpectedName)) + + cred, err := m.credentialForBucket(ctx, target.bucket) + if err != nil { + log.Printf("INFO: syfon_bulk_list_validate_credential id=%s bucket=%s key=%q error=%q", result.ID, target.bucket, target.key, err.Error()) + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = strings.TrimSpace(err.Error()) + result.ValidationStatus = storageListValidationStatusForError(req) + return result + } + log.Printf("INFO: syfon_bulk_list_validate_credential id=%s request_bucket=%s request_key=%q credential_id=%s credential_bucket=%s provider=%s", result.ID, target.bucket, target.key, credentialIDForCredential(*cred), strings.TrimSpace(cred.Bucket), strings.TrimSpace(cred.Provider)) + visible, err := m.ListVisibleBuckets(ctx) + if err != nil { + log.Printf("INFO: syfon_bulk_list_validate_visible id=%s bucket=%s key=%q error=%q", result.ID, target.bucket, target.key, err.Error()) + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = strings.TrimSpace(err.Error()) + result.ValidationStatus = storageListValidationStatusForError(req) + return result + } + if !bucketVisibleToCaller(visible, target.bucket, credentialIDForCredential(*cred)) { + err := &StorageInspectError{Kind: StorageInspectPermissionDenied, Message: fmt.Sprintf("bucket %q is not visible to the caller", target.bucket)} + log.Printf("INFO: syfon_bulk_list_validate_visible id=%s bucket=%s key=%q visible_count=%d error=%q", result.ID, target.bucket, target.key, len(visible), err.Error()) + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = err.Error() + result.ValidationStatus = storageListValidationStatusForError(req) + return result + } + + listStart := time.Now() + log.Printf("INFO: syfon_bulk_list_validate_list_start id=%s bucket=%s prefix=%q exact_prefix=true max_keys=1", result.ID, target.bucket, target.key) + matches, err := m.listS3Prefix(ctx, *cred, target.bucket, target.key, StoragePrefixListOptions{ExactPrefix: true, MaxKeys: 1}) + if err != nil { + log.Printf("INFO: syfon_bulk_list_validate_list_done id=%s bucket=%s prefix=%q duration_ms=%d error=%q", result.ID, target.bucket, target.key, time.Since(listStart).Milliseconds(), err.Error()) + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = strings.TrimSpace(err.Error()) + result.ValidationStatus = storageListValidationStatusForError(req) + return result + } + firstKey := "" + if len(matches) > 0 { + firstKey = strings.Trim(strings.TrimSpace(matches[0].Key), "/") + } + log.Printf("INFO: syfon_bulk_list_validate_list_done id=%s bucket=%s prefix=%q duration_ms=%d matches=%d first_key=%q", result.ID, target.bucket, target.key, time.Since(listStart).Milliseconds(), len(matches), firstKey) + for _, item := range matches { + key := strings.Trim(strings.TrimSpace(item.Key), "/") + if key != target.key { + log.Printf("INFO: syfon_bulk_list_validate_exact_miss id=%s requested_key=%q listed_key=%q", result.ID, target.key, key) + continue + } + normalized := normalizeStorageBucketObjects([]StorageBucketObject{item}, &resolvedStorageScopeTarget{provider: common.S3Provider, bucket: target.bucket, prefix: target.key}) + if len(normalized) == 0 { + break + } + return storageListValidationPresentResult(req, result, normalized[0]) + } + result.Status = StorageProbeStatusNotFound + result.ErrorKind = string(StorageInspectObjectNotFound) + result.Error = fmt.Sprintf("object %q was not found", req.ObjectURL) + result.ValidationStatus = storageListValidationStatusForError(req) + return result +} + +func storageListValidationPresentResult(req StorageListValidationRequest, result StorageListValidationResult, item StorageBucketObject) StorageListValidationResult { + result.ObjectURL = item.ObjectURL + result.Key = item.Key + result.Path = item.Path + result.Exists = true + result.Status = StorageProbeStatusPresent + result.Error = "" + result.ErrorKind = "" + result.SizeBytes = ptrInt64(item.SizeBytes) + result.ETag = strings.TrimSpace(item.ETag) + result.LastModTime = item.LastModTime + result.ValidationStatus, result.SizeMatch, result.NameMatch, result.ValidationMismatches = validateStorageListResult(req, item) + return result +} + +func storageListValidationStatusForError(req StorageListValidationRequest) StorageValidationStatus { + if req.ExpectedSizeBytes == nil && strings.TrimSpace(req.ExpectedName) == "" { + return StorageValidationNotRequested + } + return StorageValidationUnverifiable +} + +func validateStorageListResult(req StorageListValidationRequest, item StorageBucketObject) (StorageValidationStatus, *bool, *bool, []string) { + if req.ExpectedSizeBytes == nil && strings.TrimSpace(req.ExpectedName) == "" { + return StorageValidationNotRequested, nil, nil, nil + } + mismatches := make([]string, 0, 2) + var sizeMatch *bool + if req.ExpectedSizeBytes != nil { + matched := item.SizeBytes == *req.ExpectedSizeBytes + sizeMatch = &matched + if !matched { + mismatches = append(mismatches, "size_mismatch") + } + } + var nameMatch *bool + if expectedName := strings.TrimSpace(req.ExpectedName); expectedName != "" { + matched := path.Base(item.Key) == expectedName + nameMatch = &matched + if !matched { + mismatches = append(mismatches, "name_mismatch") + } + } + if len(mismatches) > 0 { + return StorageValidationMismatched, sizeMatch, nameMatch, mismatches + } + return StorageValidationMatched, sizeMatch, nameMatch, nil +} + +func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options StoragePrefixListOptions) ([]StorageBucketObject, error) { + started := time.Now() client, err := s3ClientFromContext(ctx, cred) if err != nil { return nil, err @@ -237,20 +570,36 @@ func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket Bucket: aws.String(bucket), } if trimmedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); trimmedPrefix != "" { - input.Prefix = aws.String(trimmedPrefix + "/") + if options.ExactPrefix { + input.Prefix = aws.String(trimmedPrefix) + } else { + input.Prefix = aws.String(trimmedPrefix + "/") + } } + if options.MaxKeys > 0 { + input.MaxKeys = aws.Int32(options.MaxKeys) + } + requestPrefix := aws.ToString(input.Prefix) + log.Printf("INFO: syfon_s3_prefix_list_start bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead) paginator := awss3.NewListObjectsV2Paginator(client, input) out := make([]StorageBucketObject, 0) + pageCount := 0 + firstKeys := make([]string, 0, 5) for paginator.HasMorePages() { page, err := paginator.NextPage(ctx) if err != nil { + log.Printf("INFO: syfon_s3_prefix_list_done bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d duration_ms=%d error=%q", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead, pageCount, len(out), time.Since(started).Milliseconds(), err.Error()) return nil, classifyS3ListError(bucket, prefix, err) } + pageCount++ for _, item := range page.Contents { key := strings.Trim(strings.TrimSpace(aws.ToString(item.Key)), "/") if key == "" { continue } + if len(firstKeys) < cap(firstKeys) { + firstKeys = append(firstKeys, key) + } var size int64 if item.Size != nil { size = *item.Size @@ -271,7 +620,8 @@ func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket }) } } - if !includeHead || len(out) == 0 { + log.Printf("INFO: syfon_s3_prefix_list_done bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d first_keys=%q duration_ms=%d", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead, pageCount, len(out), strings.Join(firstKeys, ","), time.Since(started).Milliseconds()) + if !options.IncludeHead || len(out) == 0 { return out, nil } diff --git a/internal/core/service.go b/internal/core/service.go index 7f5e4d3a..901a0a2d 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -37,7 +37,7 @@ type ObjectManager struct { uM urlmanager.UrlManager bucketScopeCache *bucketScopeCache inspectS3Object func(context.Context, models.S3Credential, string, string) (*StorageObjectMetadata, error) - listS3Prefix func(context.Context, models.S3Credential, string, string, bool) ([]StorageBucketObject, error) + listS3Prefix func(context.Context, models.S3Credential, string, string, StoragePrefixListOptions) ([]StorageBucketObject, error) } type VisibleBucket struct { diff --git a/internal/db/interface.go b/internal/db/interface.go index b7cf389e..36f9e4a1 100644 --- a/internal/db/interface.go +++ b/internal/db/interface.go @@ -21,6 +21,7 @@ type ObjectStore interface { CreateObject(ctx context.Context, obj *models.InternalObject) error GetObjectsByChecksum(ctx context.Context, checksum string) ([]models.InternalObject, error) GetObjectsByChecksums(ctx context.Context, checksums []string) (map[string][]models.InternalObject, error) + ListScopedObjectIDsByChecksums(ctx context.Context, organization, project string, checksums []string) (map[string][]string, error) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) CreateObjectAlias(ctx context.Context, aliasID, canonicalObjectID string) error ResolveObjectAlias(ctx context.Context, aliasID string) (string, error) diff --git a/internal/db/postgres/object_schema.sql b/internal/db/postgres/object_schema.sql index 339da7c9..617e4ff7 100644 --- a/internal/db/postgres/object_schema.sql +++ b/internal/db/postgres/object_schema.sql @@ -37,6 +37,13 @@ CREATE TABLE IF NOT EXISTS drs_object_alias ( FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE ); +CREATE TABLE IF NOT EXISTS drs_object_name_alias ( + object_id TEXT NOT NULL, + name_alias TEXT NOT NULL, + PRIMARY KEY (object_id, name_alias), + FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE +); + DROP TABLE IF EXISTS drs_object_browse_index; CREATE INDEX IF NOT EXISTS drs_object_access_method_object_id_idx @@ -65,3 +72,6 @@ CREATE INDEX IF NOT EXISTS drs_object_controlled_access_object_id_resource_idx CREATE INDEX IF NOT EXISTS drs_object_alias_object_id_idx ON drs_object_alias(object_id); + +CREATE INDEX IF NOT EXISTS drs_object_name_alias_object_id_idx + ON drs_object_name_alias(object_id); diff --git a/internal/db/postgres/objects.go b/internal/db/postgres/objects.go index 1173c498..ee0c301d 100644 --- a/internal/db/postgres/objects.go +++ b/internal/db/postgres/objects.go @@ -136,6 +136,10 @@ retryLookup: r.Name = strings.TrimSpace(name.String) r.Version = version.String r.Description = description.String + nameAliases, err := db.nameAliasesForObject(ctx, lookupID) + if err != nil { + return nil, err + } objectID := r.ID if resolvedAlias && requestID != "" { objectID = requestID @@ -152,7 +156,8 @@ retryLookup: Name: common.Ptr(r.Name), SelfUri: "drs://" + objectID, }, - Properties: map[string]interface{}{}, + NameAliases: nameAliases, + Properties: map[string]interface{}{}, } // 2. Fetch storage access methods. urlRows, err := db.db.QueryContext(ctx, "SELECT url, type FROM drs_object_access_method WHERE object_id = $1", lookupID) @@ -237,6 +242,12 @@ func (db *PostgresDB) CreateObject(ctx context.Context, obj *models.InternalObje if err := insertControlledAccessTx(ctx, tx, obj.Id, objectAccessResources(obj)); err != nil { return err } + for _, alias := range normalizeObjectNameAliases(obj) { + _, err = tx.ExecContext(ctx, `INSERT INTO drs_object_name_alias (object_id, name_alias) VALUES ($1, $2)`, obj.Id, alias) + if err != nil { + return fmt.Errorf("failed to insert name alias: %w", err) + } + } // Insert storage access methods. if obj.AccessMethods != nil { @@ -297,6 +308,8 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte checksumObjectIDs := make([]string, 0) checksumTypes := make([]string, 0) checksumValues := make([]string, 0) + nameAliasObjectIDs := make([]string, 0) + nameAliasValues := make([]string, 0) for _, obj := range objects { ids = append(ids, obj.Id) @@ -312,6 +325,10 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte controlledObjectIDs = append(controlledObjectIDs, obj.Id) controlledResources = append(controlledResources, resource) } + for _, alias := range normalizeObjectNameAliases(&obj) { + nameAliasObjectIDs = append(nameAliasObjectIDs, obj.Id) + nameAliasValues = append(nameAliasValues, alias) + } if obj.AccessMethods != nil { for _, am := range *obj.AccessMethods { if am.AccessUrl == nil || am.AccessUrl.Url == "" { @@ -366,6 +383,9 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_checksum WHERE object_id = ANY($1)`, pq.Array(ids)); err != nil { return fmt.Errorf("failed bulk clear checksums: %w", err) } + if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_name_alias WHERE object_id = ANY($1)`, pq.Array(ids)); err != nil { + return fmt.Errorf("failed bulk clear name aliases: %w", err) + } if len(accessObjectIDs) > 0 { if _, err := tx.ExecContext(ctx, ` @@ -385,6 +405,15 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte return fmt.Errorf("failed bulk insert controlled access: %w", err) } } + if len(nameAliasObjectIDs) > 0 { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO drs_object_name_alias (object_id, name_alias) + SELECT * FROM UNNEST($1::text[], $2::text[])`, + pq.Array(nameAliasObjectIDs), pq.Array(nameAliasValues), + ); err != nil { + return fmt.Errorf("failed bulk insert name aliases: %w", err) + } + } if len(checksumObjectIDs) > 0 { if _, err := tx.ExecContext(ctx, ` @@ -476,6 +505,61 @@ func (db *PostgresDB) GetObjectsByChecksums(ctx context.Context, checksums []str return result, nil } +func (db *PostgresDB) ListScopedObjectIDsByChecksums(ctx context.Context, organization, project string, checksums []string) (map[string][]string, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" || project == "" || len(checksums) == 0 { + return map[string][]string{}, nil + } + resource, err := sycommon.ResourcePath(organization, project) + if err != nil { + return nil, err + } + normalized := uniqueNonEmptyStrings(checksums) + if len(normalized) == 0 { + return map[string][]string{}, nil + } + rows, err := db.db.QueryContext(ctx, ` + SELECT DISTINCT c.checksum, c.object_id + FROM drs_object_checksum c + INNER JOIN drs_object_controlled_access ca ON ca.object_id = c.object_id + WHERE ca.resource = $1 AND c.type = $2 AND c.checksum = ANY($3) + ORDER BY c.checksum, c.object_id`, resource, "sha256", pq.Array(normalized)) + if err != nil { + return nil, err + } + defer rows.Close() + out := make(map[string][]string, len(normalized)) + for _, checksum := range normalized { + out[checksum] = []string{} + } + seen := make(map[string]map[string]struct{}, len(normalized)) + for rows.Next() { + var checksum string + var objectID string + if err := rows.Scan(&checksum, &objectID); err != nil { + return nil, err + } + checksum = strings.TrimSpace(checksum) + objectID = strings.TrimSpace(objectID) + if checksum == "" || objectID == "" { + continue + } + if seen[checksum] == nil { + seen[checksum] = make(map[string]struct{}) + } + if _, ok := seen[checksum][objectID]; ok { + continue + } + seen[checksum][objectID] = struct{}{} + out[checksum] = append(out[checksum], objectID) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func (db *PostgresDB) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) { organization = strings.TrimSpace(organization) project = strings.TrimSpace(project) @@ -1161,6 +1245,9 @@ func (db *PostgresDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []st if err := db.attachControlledAccess(ctx, objectsByID); err != nil { return nil, err } + if err := db.attachNameAliases(ctx, objectsByID); err != nil { + return nil, err + } return objectsByID, nil } @@ -1253,6 +1340,23 @@ func sortedObjectIDs(objectsByID map[string]*models.InternalObject) []string { return ids } +func uniqueNonEmptyStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + normalized := strings.TrimSpace(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + func (db *PostgresDB) UpdateObjectAccessMethods(ctx context.Context, objectID string, accessMethods []drs.AccessMethod) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -1339,6 +1443,13 @@ func objectAccessResources(obj *models.InternalObject) []string { return sycommon.AuthzMapToList(obj.Authorizations) } +func normalizeObjectNameAliases(obj *models.InternalObject) []string { + if obj == nil { + return nil + } + return common.NormalizeNameAliases(common.StringVal(obj.Name), obj.NameAliases) +} + func insertControlledAccessTx(ctx context.Context, tx *sql.Tx, objectID string, resources []string) error { for _, resource := range sycommon.NormalizeAccessResources(resources) { if _, err := tx.ExecContext(ctx, `INSERT INTO drs_object_controlled_access (object_id, resource) VALUES ($1, $2)`, objectID, resource); err != nil { @@ -1368,6 +1479,27 @@ func (db *PostgresDB) controlledAccessForObject(ctx context.Context, objectID st return sycommon.NormalizeAccessResources(resources), nil } +func (db *PostgresDB) nameAliasesForObject(ctx context.Context, objectID string) ([]string, error) { + rows, err := db.db.QueryContext(ctx, `SELECT name_alias FROM drs_object_name_alias WHERE object_id = $1 ORDER BY name_alias`, objectID) + if err != nil { + return nil, err + } + defer rows.Close() + + aliases := make([]string, 0) + for rows.Next() { + var alias string + if err := rows.Scan(&alias); err != nil { + return nil, err + } + aliases = append(aliases, alias) + } + if err := rows.Err(); err != nil { + return nil, err + } + return common.NormalizeNameAliases("", aliases), nil +} + func (db *PostgresDB) attachControlledAccess(ctx context.Context, objectsByID map[string]*models.InternalObject) error { if len(objectsByID) == 0 { return nil @@ -1413,3 +1545,39 @@ func (db *PostgresDB) attachControlledAccess(ctx context.Context, objectsByID ma } return nil } + +func (db *PostgresDB) attachNameAliases(ctx context.Context, objectsByID map[string]*models.InternalObject) error { + if len(objectsByID) == 0 { + return nil + } + rows, err := db.db.QueryContext(ctx, ` + SELECT object_id, name_alias + FROM drs_object_name_alias + WHERE object_id = ANY($1) + ORDER BY object_id, name_alias`, pq.Array(sortedObjectIDs(objectsByID))) + if err != nil { + return fmt.Errorf("failed to fetch bulk object name aliases: %w", err) + } + defer rows.Close() + + byObject := make(map[string][]string, len(objectsByID)) + for rows.Next() { + var objectID, alias string + if err := rows.Scan(&objectID, &alias); err != nil { + return err + } + byObject[objectID] = append(byObject[objectID], alias) + } + if err := rows.Err(); err != nil { + return err + } + + for objectID, aliases := range byObject { + obj := objectsByID[objectID] + if obj == nil { + continue + } + obj.NameAliases = common.NormalizeNameAliases(common.StringVal(obj.Name), aliases) + } + return nil +} diff --git a/internal/db/postgres/objects_sqlmock_test.go b/internal/db/postgres/objects_sqlmock_test.go index 505e156a..75036002 100644 --- a/internal/db/postgres/objects_sqlmock_test.go +++ b/internal/db/postgres/objects_sqlmock_test.go @@ -277,6 +277,40 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { } } +func TestListScopedObjectIDsByChecksums(t *testing.T) { + pg, mock, rawDB := newMockPostgresDB(t) + defer rawDB.Close() + + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT DISTINCT c.checksum, c.object_id + FROM drs_object_checksum c + INNER JOIN drs_object_controlled_access ca ON ca.object_id = c.object_id + WHERE ca.resource = $1 AND c.type = $2 AND c.checksum = ANY($3) + ORDER BY c.checksum, c.object_id`)). + WithArgs("/organization/org/project/p1", "sha256", pq.Array([]string{"sha-a", "sha-b", "missing"})). + WillReturnRows(sqlmock.NewRows([]string{"checksum", "object_id"}). + AddRow("sha-a", "obj-1"). + AddRow("sha-a", "obj-2"). + AddRow("sha-b", "obj-3")) + + res, err := pg.ListScopedObjectIDsByChecksums(context.Background(), "org", "p1", []string{"sha-a", "sha-a", "sha-b", "missing", ""}) + if err != nil { + t.Fatalf("ListScopedObjectIDsByChecksums returned error: %v", err) + } + if got := res["sha-a"]; len(got) != 2 || got[0] != "obj-1" || got[1] != "obj-2" { + t.Fatalf("unexpected ids for sha-a: %+v", got) + } + if got := res["sha-b"]; len(got) != 1 || got[0] != "obj-3" { + t.Fatalf("unexpected ids for sha-b: %+v", got) + } + if got := res["missing"]; len(got) != 0 { + t.Fatalf("expected empty ids for missing, got %+v", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } +} + func TestListObjectIDsByScopeOrgIncludesProjectScopes(t *testing.T) { pg, mock, rawDB := newMockPostgresDB(t) defer rawDB.Close() diff --git a/internal/db/sqlite/objects.go b/internal/db/sqlite/objects.go index c9774a68..09865cd2 100644 --- a/internal/db/sqlite/objects.go +++ b/internal/db/sqlite/objects.go @@ -137,6 +137,10 @@ retryLookup: r.Name = strings.TrimSpace(name.String) r.Version = version.String r.Description = description.String + nameAliases, err := db.nameAliasesForObject(ctx, lookupID) + if err != nil { + return nil, err + } objectID := r.ID if resolvedAlias && requestID != "" { objectID = requestID @@ -153,7 +157,8 @@ retryLookup: Name: common.Ptr(r.Name), SelfUri: "drs://" + objectID, }, - Properties: map[string]interface{}{}, + NameAliases: nameAliases, + Properties: map[string]interface{}{}, } // 2. Fetch storage access methods. @@ -239,6 +244,12 @@ func (db *SqliteDB) CreateObject(ctx context.Context, obj *models.InternalObject if err := insertControlledAccessTx(ctx, tx, obj.Id, objectAccessResources(obj)); err != nil { return err } + for _, alias := range normalizeObjectNameAliases(obj) { + _, err = tx.ExecContext(ctx, `INSERT INTO drs_object_name_alias (object_id, name_alias) VALUES (?, ?)`, obj.Id, alias) + if err != nil { + return fmt.Errorf("failed to insert name alias: %w", err) + } + } // Insert storage access methods. if obj.AccessMethods != nil { @@ -292,6 +303,7 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern accessArgs := make([]interface{}, 0) controlledArgs := make([]interface{}, 0) checksumArgs := make([]interface{}, 0) + nameAliasArgs := make([]interface{}, 0) for _, obj := range objects { ids = append(ids, obj.Id) @@ -301,6 +313,9 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern for _, resource := range objectAccessResources(&obj) { controlledArgs = append(controlledArgs, obj.Id, resource) } + for _, alias := range normalizeObjectNameAliases(&obj) { + nameAliasArgs = append(nameAliasArgs, obj.Id, alias) + } if obj.AccessMethods != nil { for _, am := range *obj.AccessMethods { if am.AccessUrl == nil || am.AccessUrl.Url == "" { @@ -347,6 +362,9 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern if err := execSQLiteDeleteByIDs(tx, "drs_object_checksum", ids); err != nil { return fmt.Errorf("failed bulk clear checksums: %w", err) } + if err := execSQLiteDeleteByIDs(tx, "drs_object_name_alias", ids); err != nil { + return fmt.Errorf("failed bulk clear name aliases: %w", err) + } if len(accessArgs) > 0 { if err := execSQLiteBulkInsert( @@ -372,6 +390,18 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern return fmt.Errorf("failed bulk insert controlled access: %w", err) } } + if len(nameAliasArgs) > 0 { + if err := execSQLiteBulkInsert( + tx, + "INSERT INTO drs_object_name_alias (object_id, name_alias) VALUES ", + "(?, ?)", + 2, + nameAliasArgs, + "", + ); err != nil { + return fmt.Errorf("failed bulk insert name aliases: %w", err) + } + } if len(checksumArgs) > 0 { if err := execSQLiteBulkInsert( tx, @@ -445,6 +475,67 @@ func (db *SqliteDB) GetObjectsByChecksums(ctx context.Context, checksums []strin return result, nil } +func (db *SqliteDB) ListScopedObjectIDsByChecksums(ctx context.Context, organization, project string, checksums []string) (map[string][]string, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" || project == "" || len(checksums) == 0 { + return map[string][]string{}, nil + } + resource, err := sycommon.ResourcePath(organization, project) + if err != nil { + return nil, err + } + normalized := uniqueNonEmptyStrings(checksums) + if len(normalized) == 0 { + return map[string][]string{}, nil + } + args := make([]any, 0, len(normalized)+2) + args = append(args, resource, "sha256") + placeholders := makePlaceholders(len(normalized)) + for _, checksum := range normalized { + args = append(args, checksum) + } + rows, err := db.db.QueryContext(ctx, fmt.Sprintf(` + SELECT DISTINCT c.checksum, c.object_id + FROM drs_object_checksum c + INNER JOIN drs_object_controlled_access ca ON ca.object_id = c.object_id + WHERE ca.resource = ? AND c.type = ? AND c.checksum IN (%s) + ORDER BY c.checksum, c.object_id`, placeholders), args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := make(map[string][]string, len(normalized)) + for _, checksum := range normalized { + out[checksum] = []string{} + } + seen := make(map[string]map[string]struct{}, len(normalized)) + for rows.Next() { + var checksum string + var objectID string + if err := rows.Scan(&checksum, &objectID); err != nil { + return nil, err + } + checksum = strings.TrimSpace(checksum) + objectID = strings.TrimSpace(objectID) + if checksum == "" || objectID == "" { + continue + } + if seen[checksum] == nil { + seen[checksum] = make(map[string]struct{}) + } + if _, ok := seen[checksum][objectID]; ok { + continue + } + seen[checksum][objectID] = struct{}{} + out[checksum] = append(out[checksum], objectID) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func (db *SqliteDB) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) { organization = strings.TrimSpace(organization) project = strings.TrimSpace(project) @@ -1175,6 +1266,9 @@ func (db *SqliteDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []stri if err := db.attachControlledAccess(ctx, objectsByID); err != nil { return nil, err } + if err := db.attachNameAliases(ctx, objectsByID); err != nil { + return nil, err + } return objectsByID, nil } @@ -1280,6 +1374,13 @@ func objectAccessResources(obj *models.InternalObject) []string { return sycommon.AuthzMapToList(obj.Authorizations) } +func normalizeObjectNameAliases(obj *models.InternalObject) []string { + if obj == nil { + return nil + } + return common.NormalizeNameAliases(common.StringVal(obj.Name), obj.NameAliases) +} + func sortedObjectIDs(objectsByID map[string]*models.InternalObject) []string { ids := make([]string, 0, len(objectsByID)) for id := range objectsByID { @@ -1289,6 +1390,23 @@ func sortedObjectIDs(objectsByID map[string]*models.InternalObject) []string { return ids } +func uniqueNonEmptyStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + normalized := strings.TrimSpace(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + func insertControlledAccessTx(ctx context.Context, tx *sql.Tx, objectID string, resources []string) error { for _, resource := range sycommon.NormalizeAccessResources(resources) { if _, err := tx.ExecContext(ctx, `INSERT INTO drs_object_controlled_access (object_id, resource) VALUES (?, ?)`, objectID, resource); err != nil { @@ -1318,6 +1436,27 @@ func (db *SqliteDB) controlledAccessForObject(ctx context.Context, objectID stri return sycommon.NormalizeAccessResources(resources), nil } +func (db *SqliteDB) nameAliasesForObject(ctx context.Context, objectID string) ([]string, error) { + rows, err := db.db.QueryContext(ctx, `SELECT name_alias FROM drs_object_name_alias WHERE object_id = ? ORDER BY name_alias`, objectID) + if err != nil { + return nil, err + } + defer rows.Close() + + aliases := make([]string, 0) + for rows.Next() { + var alias string + if err := rows.Scan(&alias); err != nil { + return nil, err + } + aliases = append(aliases, alias) + } + if err := rows.Err(); err != nil { + return nil, err + } + return common.NormalizeNameAliases("", aliases), nil +} + func (db *SqliteDB) attachControlledAccess(ctx context.Context, objectsByID map[string]*models.InternalObject) error { if len(objectsByID) == 0 { return nil @@ -1365,6 +1504,48 @@ func (db *SqliteDB) attachControlledAccess(ctx context.Context, objectsByID map[ return nil } +func (db *SqliteDB) attachNameAliases(ctx context.Context, objectsByID map[string]*models.InternalObject) error { + if len(objectsByID) == 0 { + return nil + } + ids := sortedObjectIDs(objectsByID) + query := fmt.Sprintf(` + SELECT object_id, name_alias + FROM drs_object_name_alias + WHERE object_id IN (%s) + ORDER BY object_id, name_alias`, makePlaceholders(len(ids))) + args := make([]any, 0, len(ids)) + for _, id := range ids { + args = append(args, id) + } + rows, err := db.db.QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("failed to fetch bulk object name aliases: %w", err) + } + defer rows.Close() + + byObject := make(map[string][]string, len(objectsByID)) + for rows.Next() { + var objectID, alias string + if err := rows.Scan(&objectID, &alias); err != nil { + return err + } + byObject[objectID] = append(byObject[objectID], alias) + } + if err := rows.Err(); err != nil { + return err + } + + for objectID, aliases := range byObject { + obj := objectsByID[objectID] + if obj == nil { + continue + } + obj.NameAliases = common.NormalizeNameAliases(common.StringVal(obj.Name), aliases) + } + return nil +} + func (db *SqliteDB) UpdateObjectAccessMethods(ctx context.Context, objectID string, accessMethods []drs.AccessMethod) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { diff --git a/internal/db/sqlite/sqlite.go b/internal/db/sqlite/sqlite.go index abe5e9f6..8c0c135e 100644 --- a/internal/db/sqlite/sqlite.go +++ b/internal/db/sqlite/sqlite.go @@ -94,6 +94,13 @@ func (db *SqliteDB) initSchema() error { FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE )`, `CREATE INDEX IF NOT EXISTS idx_drs_object_alias_object_id ON drs_object_alias(object_id)`, + `CREATE TABLE IF NOT EXISTS drs_object_name_alias ( + object_id TEXT NOT NULL, + name_alias TEXT NOT NULL, + PRIMARY KEY (object_id, name_alias), + FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE + )`, + `CREATE INDEX IF NOT EXISTS idx_drs_object_name_alias_object_id ON drs_object_name_alias(object_id)`, `CREATE TABLE IF NOT EXISTS s3_credential ( credential_id TEXT PRIMARY KEY, bucket TEXT NOT NULL, diff --git a/internal/db/sqlite/sqlite_test.go b/internal/db/sqlite/sqlite_test.go index ff6f7641..6c40fb87 100644 --- a/internal/db/sqlite/sqlite_test.go +++ b/internal/db/sqlite/sqlite_test.go @@ -924,6 +924,83 @@ func TestSqliteDB_GetObjectsByChecksumsAndListByPrefix(t *testing.T) { } } +func TestSqliteDB_ListScopedObjectIDsByChecksums(t *testing.T) { + ctx := context.Background() + db, _ := NewSqliteDB(":memory:") + now := time.Now() + objects := []models.InternalObject{ + { + DrsObject: drs.DrsObject{ + Id: "proj-a-1", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-a"}}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "proj-a-2", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-a"}}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "other-project", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-a"}}, + }, + Authorizations: map[string][]string{"org": {"p2"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "other-org", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-a"}}, + }, + Authorizations: map[string][]string{"other": {"p1"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "proj-b", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-b"}}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + } + if err := db.RegisterObjects(ctx, objects); err != nil { + t.Fatalf("RegisterObjects failed: %v", err) + } + + res, err := db.ListScopedObjectIDsByChecksums(ctx, "org", "p1", []string{"sha-a", "sha-a", "sha-b", "missing", ""}) + if err != nil { + t.Fatalf("ListScopedObjectIDsByChecksums failed: %v", err) + } + if got := res["sha-a"]; len(got) != 2 || got[0] != "proj-a-1" || got[1] != "proj-a-2" { + t.Fatalf("unexpected scoped ids for sha-a: %+v", got) + } + if got := res["sha-b"]; len(got) != 1 || got[0] != "proj-b" { + t.Fatalf("unexpected scoped ids for sha-b: %+v", got) + } + if got := res["missing"]; len(got) != 0 { + t.Fatalf("expected empty ids for missing checksum, got %+v", got) + } + emptyRes, err := db.ListScopedObjectIDsByChecksums(ctx, "org", "p1", nil) + if err != nil { + t.Fatalf("ListScopedObjectIDsByChecksums empty failed: %v", err) + } + if len(emptyRes) != 0 { + t.Fatalf("expected empty map for empty checksum input, got %+v", emptyRes) + } +} + func TestSqliteDB_ListObjectIDsByScopeRootIncludesUnscoped(t *testing.T) { ctx := context.Background() db, _ := NewSqliteDB(":memory:") diff --git a/internal/models/object.go b/internal/models/object.go index cfc07fe0..ea76288c 100644 --- a/internal/models/object.go +++ b/internal/models/object.go @@ -3,16 +3,19 @@ package models import ( "encoding/json" "fmt" + "sort" + "strings" "time" "github.com/calypr/syfon/apigen/server/drs" - "github.com/calypr/syfon/common" + syfoncommon "github.com/calypr/syfon/common" ) // InternalObject is the primary DRS domain model. It wraps the GA4GH DrsObject // and adds Syfon-specific authorization metadata. type InternalObject struct { drs.DrsObject + NameAliases []string `json:"name_aliases,omitempty"` Authorizations map[string][]string `json:"-"` Properties map[string]interface{} `json:"-"` } @@ -41,8 +44,9 @@ func (o *InternalObject) UnmarshalJSON(data []byte) error { type wireObject struct { drs.DrsObject - Did string `json:"did,omitempty"` - Hashes map[string]string `json:"hashes,omitempty"` + Did string `json:"did,omitempty"` + Hashes map[string]string `json:"hashes,omitempty"` + NameAliases []string `json:"name_aliases,omitempty"` } var wire wireObject @@ -71,13 +75,14 @@ func (o *InternalObject) UnmarshalJSON(data []byte) error { o.UpdatedTime = wire.UpdatedTime o.Size = wire.Size o.Name = wire.Name + o.NameAliases = normalizeNameAliases(stringValue(wire.Name), wire.NameAliases) o.Description = wire.Description o.MimeType = wire.MimeType o.SelfUri = wire.SelfUri o.Version = wire.Version if wire.ControlledAccess != nil { - o.Authorizations = common.ControlledAccessToAuthzMap(*wire.ControlledAccess) + o.Authorizations = syfoncommon.ControlledAccessToAuthzMap(*wire.ControlledAccess) } return nil @@ -115,6 +120,9 @@ func (o InternalObject) MarshalJSON() ([]byte, error) { if o.Name != nil { out["name"] = *o.Name } + if len(o.NameAliases) > 0 { + out["name_aliases"] = normalizeNameAliases(stringValue(o.Name), o.NameAliases) + } if o.Description != nil { out["description"] = *o.Description } @@ -148,6 +156,46 @@ func isRetiredInternalAuthField(key string) bool { } } +func stringValue(v *string) string { + if v == nil { + return "" + } + return *v +} + +func normalizeNameAliases(primary string, aliases []string) []string { + primary = cleanToBasename(primary) + seen := make(map[string]struct{}, len(aliases)+1) + out := make([]string, 0, len(aliases)) + for _, alias := range aliases { + name := cleanToBasename(alias) + if name == "" || name == primary { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + out = append(out, name) + } + sort.Strings(out) + return out +} + +func cleanToBasename(name string) string { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return "" + } + trimmed = strings.ReplaceAll(trimmed, "\\", "/") + parts := strings.Split(trimmed, "/") + base := parts[len(parts)-1] + if base == "" { + return trimmed + } + return base +} + func (o InternalObject) External() drs.DrsObject { return o.DrsObject } diff --git a/internal/testutils/mocks.go b/internal/testutils/mocks.go index a1bbb261..5595c91b 100644 --- a/internal/testutils/mocks.go +++ b/internal/testutils/mocks.go @@ -119,6 +119,54 @@ func (m *MockDatabase) GetObjectsByChecksums(ctx context.Context, checksums []st return out, nil } +func (m *MockDatabase) ListScopedObjectIDsByChecksums(ctx context.Context, organization, project string, checksums []string) (map[string][]string, error) { + out := make(map[string][]string, len(checksums)) + for _, checksum := range checksums { + checksum = strings.TrimSpace(checksum) + if checksum == "" { + continue + } + out[checksum] = []string{} + matches, err := m.GetObjectsByChecksum(ctx, checksum) + if err != nil { + return nil, err + } + seen := map[string]struct{}{} + for _, match := range matches { + if !objectMatchesMockScope(m.ObjectAuthz[match.Id], organization, project) { + continue + } + if _, ok := seen[match.Id]; ok { + continue + } + seen[match.Id] = struct{}{} + out[checksum] = append(out[checksum], match.Id) + } + } + return out, nil +} + +func objectMatchesMockScope(authz map[string][]string, organization, project string) bool { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" { + return true + } + projects, ok := authz[organization] + if !ok { + return false + } + if project == "" || len(projects) == 0 { + return true + } + for _, candidate := range projects { + if candidate == project { + return true + } + } + return false +} + func (m *MockDatabase) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) { ids := make([]string, 0) for id := range m.Objects { @@ -823,8 +871,6 @@ func transferBreakdownKey(ev models.TransferAttributionEvent, groupBy string) st } } - - func cloneAuthzMap(in map[string][]string) map[string][]string { if len(in) == 0 { return nil From 0666d172d8d3be051c2e2c78f17855645212dc40 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 2 Jul 2026 14:18:14 -0700 Subject: [PATCH 11/24] fix tests --- apigen/openapi/internal.openapi.yaml | 4 ++++ cmd/copyproject/main.go | 3 +++ internal/core/object_reads_test.go | 4 ++-- internal/db/postgres/objects_sqlmock_test.go | 23 ++++++++++++++++++++ internal/db/postgres/postgres_schema_test.go | 3 +++ 5 files changed, 35 insertions(+), 2 deletions(-) diff --git a/apigen/openapi/internal.openapi.yaml b/apigen/openapi/internal.openapi.yaml index f0df1ade..3e194d56 100644 --- a/apigen/openapi/internal.openapi.yaml +++ b/apigen/openapi/internal.openapi.yaml @@ -657,6 +657,10 @@ components: minLength: 1 name: type: string + name_aliases: + type: array + items: + type: string hashes: $ref: '#/components/schemas/HashInfo' size: diff --git a/cmd/copyproject/main.go b/cmd/copyproject/main.go index d3a584ca..878ad1e6 100644 --- a/cmd/copyproject/main.go +++ b/cmd/copyproject/main.go @@ -479,6 +479,9 @@ func copyRecord(ctx context.Context, cmd *cobra.Command, sourceClient, targetCli if err := targetClient.Index().Upsert(ctx, did, targetObjectURL, fileName, size, checksum, authzMap); err != nil { return fmt.Errorf("failed to sync index record for DID %s: %w", did, err) } + if _, err := targetClient.DRS().UpdateObjectAccessMethods(ctx, did, []drsapi.AccessMethod{targetAccessMethod}); err != nil { + return fmt.Errorf("failed to replace access methods for DID %s: %w", did, err) + } return nil } diff --git a/internal/core/object_reads_test.go b/internal/core/object_reads_test.go index 50ea673a..40c53f37 100644 --- a/internal/core/object_reads_test.go +++ b/internal/core/object_reads_test.go @@ -42,8 +42,8 @@ func TestListObjectIDsPageByChecksum_StartAfterAndLimit(t *testing.T) { checksum := "1111111111111111111111111111111111111111111111111111111111111111" registerScopedCandidate(t, om, "chk-a", checksum, "org1", "proj1") - registerScopedCandidate(t, om, "chk-b", checksum, "org1", "proj1") - registerScopedCandidate(t, om, "chk-c", checksum, "org1", "proj1") + registerScopedCandidate(t, om, "chk-b", checksum, "org1", "proj2") + registerScopedCandidate(t, om, "chk-c", checksum, "org2", "proj1") ids, err := om.ListObjectIDsPageByChecksum(context.Background(), checksum, "sha256", "", "", "read", "chk-a", 2, 0) if err != nil { diff --git a/internal/db/postgres/objects_sqlmock_test.go b/internal/db/postgres/objects_sqlmock_test.go index 75036002..a262703f 100644 --- a/internal/db/postgres/objects_sqlmock_test.go +++ b/internal/db/postgres/objects_sqlmock_test.go @@ -114,6 +114,10 @@ func TestGetObject_DeduplicatesAndPropagatesAuthz(t *testing.T) { WillReturnRows(sqlmock.NewRows([]string{ "id", "size", "created_time", "updated_time", "name", "version", "description", }).AddRow("obj-1", int64(123), created, updated, "file.txt", "v1", "desc")) + mock.ExpectQuery(regexp.QuoteMeta("SELECT name_alias FROM drs_object_name_alias WHERE object_id = $1 ORDER BY name_alias")). + WithArgs("obj-1"). + WillReturnRows(sqlmock.NewRows([]string{"name_alias"}). + AddRow("file-old.txt")) mock.ExpectQuery(regexp.QuoteMeta("SELECT url, type FROM drs_object_access_method WHERE object_id = $1")). WithArgs("obj-1"). @@ -145,6 +149,9 @@ func TestGetObject_DeduplicatesAndPropagatesAuthz(t *testing.T) { if obj.AccessMethods == nil || len(*obj.AccessMethods) != 2 { t.Fatalf("expected 2 deduplicated access methods, got %+v", obj.AccessMethods) } + if len(obj.NameAliases) != 1 || obj.NameAliases[0] != "file-old.txt" { + t.Fatalf("expected propagated name aliases, got %+v", obj.NameAliases) + } if len(obj.Checksums) != 2 { t.Fatalf("expected 2 deduplicated checksums, got %d", len(obj.Checksums)) } @@ -168,6 +175,9 @@ func TestGetObject_IgnoresAuthContext(t *testing.T) { WillReturnRows(sqlmock.NewRows([]string{ "id", "size", "created_time", "updated_time", "name", "version", "description", }).AddRow("obj-2", int64(1), now, now, "n", "v", "d")) + mock.ExpectQuery(regexp.QuoteMeta("SELECT name_alias FROM drs_object_name_alias WHERE object_id = $1 ORDER BY name_alias")). + WithArgs("obj-2"). + WillReturnRows(sqlmock.NewRows([]string{"name_alias"})) mock.ExpectQuery(regexp.QuoteMeta("SELECT url, type FROM drs_object_access_method WHERE object_id = $1")). WithArgs("obj-2"). WillReturnRows(sqlmock.NewRows([]string{"url", "type"}). @@ -259,6 +269,16 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { AddRow("obj-1", "/organization/org/project/p1"). AddRow("obj-2", "/organization/org/project/p1")) + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT object_id, name_alias + FROM drs_object_name_alias + WHERE object_id = ANY($1) + ORDER BY object_id, name_alias`)). + WithArgs(pq.Array([]string{"obj-1", "obj-2"})). + WillReturnRows(sqlmock.NewRows([]string{"object_id", "name_alias"}). + AddRow("obj-1", "file-1-old"). + AddRow("obj-2", "file-2-old")) + objects, err := pg.GetBulkObjects(context.Background(), []string{"obj-2", "obj-1", "obj-2"}) if err != nil { t.Fatalf("GetBulkObjects returned error: %v", err) @@ -269,6 +289,9 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { if objects[0].AccessMethods == nil || len(*objects[0].AccessMethods) != 1 { t.Fatalf("expected one access method on obj-2, got %+v", objects[0].AccessMethods) } + if len(objects[0].NameAliases) != 1 || objects[0].NameAliases[0] != "file-2-old" { + t.Fatalf("expected propagated aliases on obj-2, got %+v", objects[0].NameAliases) + } if len(objects[1].Checksums) != 1 || objects[1].Checksums[0].Checksum != "aaa" { t.Fatalf("expected deduplicated checksum on obj-1, got %+v", objects[1].Checksums) } diff --git a/internal/db/postgres/postgres_schema_test.go b/internal/db/postgres/postgres_schema_test.go index 4e1cf8df..738f0b49 100644 --- a/internal/db/postgres/postgres_schema_test.go +++ b/internal/db/postgres/postgres_schema_test.go @@ -19,6 +19,7 @@ func TestSchemaEnsurers(t *testing.T) { mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_controlled_access").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_checksum").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_alias").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_name_alias").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("DROP TABLE IF EXISTS drs_object_browse_index").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_access_method_object_id_idx ON drs_object_access_method(object_id)")). WillReturnResult(sqlmock.NewResult(0, 0)) @@ -38,6 +39,8 @@ func TestSchemaEnsurers(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_alias_object_id_idx ON drs_object_alias(object_id)")). WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_name_alias_object_id_idx ON drs_object_name_alias(object_id)")). + WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectQuery("information_schema\\.columns"). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) From 5bb4699712f4a11e2d4837887b2ab4c81743342e Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 2 Jul 2026 14:48:19 -0700 Subject: [PATCH 12/24] fix hanging coverage test --- client/transfer/engine/engine_test.go | 69 +++++++++++++++++++++++---- scripts/run_coverage.sh | 13 ++++- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/client/transfer/engine/engine_test.go b/client/transfer/engine/engine_test.go index cc957766..f0993e58 100644 --- a/client/transfer/engine/engine_test.go +++ b/client/transfer/engine/engine_test.go @@ -494,12 +494,36 @@ func TestGenericUploaderMultipartStreamsProgressAndCompletesAfterFinalize(t *tes uploader := &GenericUploader{Backend: backend} var finalized atomic.Bool + var closeGate sync.Once + releaseComplete := func() { + closeGate.Do(func() { + close(completeGate) + }) + } + + var eventsMu sync.Mutex var events []common.ProgressEvent + progressReady := make(chan struct{}, 1) + progressErr := make(chan error, 1) ctx := common.WithOid(context.Background(), "oid-slow-multipart") ctx = common.WithProgress(ctx, func(ev common.ProgressEvent) error { + eventsMu.Lock() events = append(events, ev) + eventCount := len(events) + eventsMu.Unlock() if ev.BytesSoFar == 21*common.MB && !finalized.Load() { - t.Fatalf("terminal multipart progress emitted before completion: %+v", ev) + err := fmt.Errorf("terminal multipart progress emitted before completion: %+v", ev) + select { + case progressErr <- err: + default: + } + return err + } + if eventCount >= 2 { + select { + case progressReady <- struct{}{}: + default: + } } return nil }) @@ -514,21 +538,46 @@ func TestGenericUploaderMultipartStreamsProgressAndCompletesAfterFinalize(t *tes }, true) }() - time.Sleep(20 * time.Millisecond) - if len(events) < 2 { - close(completeGate) - if err := <-done; err != nil { - t.Fatalf("Upload returned error: %v", err) + select { + case <-progressReady: + case err := <-progressErr: + releaseComplete() + if uploadErr := <-done; uploadErr != nil && uploadErr.Error() != err.Error() { + t.Fatalf("Upload returned error %v after progress error %v", uploadErr, err) + } + t.Fatal(err) + case <-time.After(2 * time.Second): + releaseComplete() + select { + case err := <-done: + if err != nil { + t.Fatalf("Upload returned error: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("multipart upload did not emit streaming progress before completion gate release") } - t.Fatalf("expected streaming multipart progress before completion, got %+v", events) + eventsMu.Lock() + gotEvents := append([]common.ProgressEvent(nil), events...) + eventsMu.Unlock() + t.Fatalf("expected streaming multipart progress before completion, got %+v", gotEvents) } finalized.Store(true) - close(completeGate) - if err := <-done; err != nil { - t.Fatalf("Upload returned error: %v", err) + releaseComplete() + select { + case err := <-done: + if err != nil { + t.Fatalf("Upload returned error: %v", err) + } + case err := <-progressErr: + t.Fatal(err) + case <-time.After(2 * time.Second): + t.Fatal("multipart upload did not complete after completion gate release") } + + eventsMu.Lock() last := events[len(events)-1] + eventsMu.Unlock() if last.BytesSoFar != 21*common.MB { t.Fatalf("expected final multipart BytesSoFar=%d, got %+v", 21*common.MB, last) } diff --git a/scripts/run_coverage.sh b/scripts/run_coverage.sh index fc761df0..ee684448 100755 --- a/scripts/run_coverage.sh +++ b/scripts/run_coverage.sh @@ -42,7 +42,18 @@ if [[ -z "${PKGS}" ]]; then fi cd "${WORK_DIR}" -CGO_ENABLED=1 go test -count=1 -covermode=atomic -coverprofile "${OUT_FILE}" ${PKGS} +echo "coverage scope: ${SCOPE}" +echo "coverage workdir: ${WORK_DIR}" +echo "coverage output: ${OUT_FILE}" +echo "packages:" +printf ' %s\n' ${PKGS} + +GO_TEST_FLAGS=(-count=1 -covermode=atomic -coverprofile "${OUT_FILE}") +if [[ "${SCOPE}" == "client" ]]; then + GO_TEST_FLAGS=(-v "${GO_TEST_FLAGS[@]}") +fi + +CGO_ENABLED=1 go test "${GO_TEST_FLAGS[@]}" ${PKGS} go tool cover -func="${OUT_FILE}" | tee "${OUT_DIR}/coverage.txt" go tool cover -html="${OUT_FILE}" -o "${HTML_FILE}" From 1aab36ad56b9a828b16c2e195d1495fd5c2410bf Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 2 Jul 2026 16:19:08 -0700 Subject: [PATCH 13/24] normalize inspect route --- internal/api/internaldrs/inspect.go | 50 +++ .../api/internaldrs/inspect_routes_test.go | 157 ++++++++ internal/core/object_storage_delete.go | 2 +- internal/core/object_storage_inventory.go | 362 +++++++++++++++--- 4 files changed, 513 insertions(+), 58 deletions(-) diff --git a/internal/api/internaldrs/inspect.go b/internal/api/internaldrs/inspect.go index 11cd4294..60c69ba0 100644 --- a/internal/api/internaldrs/inspect.go +++ b/internal/api/internaldrs/inspect.go @@ -41,6 +41,7 @@ type internalInspectProjectBucketRequest struct { type internalInspectProjectRecordsRequest struct { Organization string `json:"organization,omitempty"` Project string `json:"project,omitempty"` + PathPrefix string `json:"path_prefix,omitempty"` } type internalInspectProjectScopesRequest struct { @@ -343,12 +344,23 @@ func handleInternalInspectProjectRecordsFiber(om *core.ObjectManager) fiber.Hand if err != nil { return apiutil.HandleError(c, err) } + pathPrefix := strings.Trim(strings.TrimSpace(req.PathPrefix), "/") + pathPrefixes := []string{} + if pathPrefix != "" { + pathPrefixes = append(pathPrefixes, pathPrefix) + if resolvedPrefix, err := om.ResolveProjectStoragePathPrefix(c.Context(), organization, project, pathPrefix); err == nil && strings.TrimSpace(resolvedPrefix) != "" && !strings.EqualFold(strings.TrimSpace(resolvedPrefix), pathPrefix) { + pathPrefixes = append(pathPrefixes, strings.TrimSpace(resolvedPrefix)) + } + } out := internalInspectProjectRecordsResponse{Items: make([]internalInspectProjectRecordItem, 0, len(objects))} for _, obj := range objects { record, ok := projectRecordAuditItemFromObject(obj, organization, project) if !ok { continue } + if len(pathPrefixes) > 0 && !projectRecordMatchesAnyPathPrefix(record, pathPrefixes...) { + continue + } out.Items = append(out.Items, record) } return c.JSON(out) @@ -564,6 +576,44 @@ func projectRecordAuditItemFromObject(obj models.InternalObject, organization, p return item, true } +func projectRecordMatchesAnyPathPrefix(record internalInspectProjectRecordItem, pathPrefixes ...string) bool { + for _, prefix := range pathPrefixes { + if projectRecordMatchesPathPrefix(record, prefix) { + return true + } + } + return false +} + +func projectRecordMatchesPathPrefix(record internalInspectProjectRecordItem, pathPrefix string) bool { + normalizedPrefix := strings.Trim(strings.TrimSpace(pathPrefix), "/") + if normalizedPrefix == "" { + return true + } + prefixWithSlash := normalizedPrefix + "/" + for _, raw := range record.AccessURLs { + parsedBucket, parsedKey, ok := common.ParseS3URL(strings.TrimSpace(raw)) + if !ok || strings.TrimSpace(parsedBucket) == "" { + continue + } + key := strings.Trim(strings.TrimSpace(parsedKey), "/") + if key == normalizedPrefix || strings.HasPrefix(key, prefixWithSlash) { + return true + } + } + for _, method := range record.AccessMethods { + parsedBucket, parsedKey, ok := common.ParseS3URL(strings.TrimSpace(method.URL)) + if !ok || strings.TrimSpace(parsedBucket) == "" { + continue + } + key := strings.Trim(strings.TrimSpace(parsedKey), "/") + if key == normalizedPrefix || strings.HasPrefix(key, prefixWithSlash) { + return true + } + } + return false +} + func primarySHA256Checksum(checksums []drs.Checksum) string { for _, checksum := range checksums { if strings.EqualFold(strings.TrimSpace(checksum.Type), "sha256") { diff --git a/internal/api/internaldrs/inspect_routes_test.go b/internal/api/internaldrs/inspect_routes_test.go index d50150d3..9c8ffaca 100644 --- a/internal/api/internaldrs/inspect_routes_test.go +++ b/internal/api/internaldrs/inspect_routes_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -121,6 +122,12 @@ func TestHandleInternalInspectProjectRecords(t *testing.T) { created := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) updated := time.Date(2026, 7, 1, 12, 30, 0, 0, time.UTC) db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, Objects: map[string]*drs.DrsObject{ "obj-1": { Id: "obj-1", @@ -175,6 +182,21 @@ func TestHandleInternalInspectProjectRecords(t *testing.T) { } } +func TestProjectRecordMatchesAnyPathPrefixAvoidsFalsePrefixMatches(t *testing.T) { + record := internalInspectProjectRecordItem{ + AccessURLs: []string{"s3://bucket-a/project-root/CONFIG/a.json"}, + AccessMethods: []internalProjectAccessMethod{ + {URL: "s3://bucket-a/project-root/CONFIG/a.json"}, + }, + } + if !projectRecordMatchesAnyPathPrefix(record, "CONFIG", "project-root/CONFIG") { + t.Fatalf("expected CONFIG path prefix to match CONFIG/a.json") + } + if projectRecordMatchesAnyPathPrefix(record, "CONFIGURATION", "project-root/CONFIGURATION") { + t.Fatalf("expected CONFIGURATION path prefix not to match CONFIG/a.json") + } +} + func TestHandleInternalInspectProjectScopes(t *testing.T) { body, _ := json.Marshal(internalInspectProjectScopesRequest{Organization: "syfon", Project: "e2e"}) db := &testutils.MockDatabase{ @@ -378,3 +400,138 @@ func TestHandleInternalInspectObjectBulkListValidatesExactKeyWithoutHead(t *test t.Fatalf("expected bulk-list not to HEAD objects, got %d inspector calls", inspectCalls) } } + +func TestHandleInternalInspectObjectBulkListDeduplicatesExactTargets(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + listCalls := 0 + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + listCalls++ + if bucket != "bucket-a" || prefix != "project-root/file.bin" || !options.ExactPrefix || options.MaxKeys != 1 { + t.Fatalf("unexpected bulk-list request bucket=%q prefix=%q options=%+v", bucket, prefix, options) + } + return []core.StorageBucketObject{{Provider: "s3", Bucket: bucket, Key: "project-root/file.bin", Path: "file.bin", SizeBytes: 17}}, nil + }) + expectedSize := int64(17) + body, _ := json.Marshal(internalInspectObjectsBulkRequest{Items: []internalInspectObjectRequest{ + {ID: "one", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &expectedSize, ExpectedName: "file.bin"}, + {ID: "two", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &expectedSize, ExpectedName: "file.bin"}, + }}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/bulk-list", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectObjectBulkResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if listCalls != 1 { + t.Fatalf("expected duplicate exact keys to share one LIST call, got %d", listCalls) + } + if len(resp.Items) != 2 || resp.Items[0].ID != "one" || resp.Items[1].ID != "two" { + t.Fatalf("expected one response per original request, got %+v", resp.Items) + } +} + +func TestHandleInternalInspectObjectBulkListSharesRemoteEvidenceAcrossValidationExpectations(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + listCalls := 0 + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + listCalls++ + return []core.StorageBucketObject{{Provider: "s3", Bucket: bucket, Key: "project-root/file.bin", Path: "file.bin", SizeBytes: 17}}, nil + }) + size17 := int64(17) + size99 := int64(99) + body, _ := json.Marshal(internalInspectObjectsBulkRequest{Items: []internalInspectObjectRequest{ + {ID: "matched", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &size17, ExpectedName: "file.bin"}, + {ID: "mismatched", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &size99, ExpectedName: "file.bin"}, + }}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/bulk-list", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectObjectBulkResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if listCalls != 1 { + t.Fatalf("expected one shared LIST call, got %d", listCalls) + } + if len(resp.Items) != 2 || resp.Items[0].ValidationStatus != "matched" || resp.Items[1].ValidationStatus != "mismatched" { + t.Fatalf("expected per-request validation after shared evidence, got %+v", resp.Items) + } +} + +func TestHandleInternalInspectObjectBulkListCoalescesDensePrefixes(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + listCalls := 0 + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + listCalls++ + if bucket != "bucket-a" || prefix != "project-root/dense/" || !options.ExactPrefix || options.MaxKeys != 0 || options.IncludeHead { + t.Fatalf("unexpected coalesced LIST request bucket=%q prefix=%q options=%+v", bucket, prefix, options) + } + out := make([]core.StorageBucketObject, 0, 25) + for i := 0; i < 25; i++ { + key := fmt.Sprintf("project-root/dense/file-%02d.bin", i) + out = append(out, core.StorageBucketObject{Provider: "s3", Bucket: bucket, Key: key, Path: fmt.Sprintf("file-%02d.bin", i), SizeBytes: int64(i + 1)}) + } + return out, nil + }) + items := make([]internalInspectObjectRequest, 0, 25) + for i := 0; i < 25; i++ { + size := int64(i + 1) + items = append(items, internalInspectObjectRequest{ + ID: fmt.Sprintf("item-%02d", i), + ObjectURL: fmt.Sprintf("s3://bucket-a/project-root/dense/file-%02d.bin", i), + ExpectedSizeBytes: &size, + ExpectedName: fmt.Sprintf("file-%02d.bin", i), + }) + } + body, _ := json.Marshal(internalInspectObjectsBulkRequest{Items: items}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/bulk-list", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectObjectBulkResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if listCalls != 1 { + t.Fatalf("expected dense prefix to coalesce into one LIST call, got %d", listCalls) + } + if len(resp.Items) != 25 { + t.Fatalf("expected 25 items, got %d", len(resp.Items)) + } +} diff --git a/internal/core/object_storage_delete.go b/internal/core/object_storage_delete.go index 45d106e2..48bd8db8 100644 --- a/internal/core/object_storage_delete.go +++ b/internal/core/object_storage_delete.go @@ -106,7 +106,7 @@ func (m *ObjectManager) storageTargetFromURL(ctx context.Context, raw string) (s if bucket == "" || key == "" { return storageTarget{}, false, nil } - cred, err := m.db.GetS3Credential(ctx, bucket) + cred, err := m.credentialForBucket(ctx, bucket) if err != nil { return storageTarget{}, false, fmt.Errorf("lookup credential for bucket %s: %w", bucket, err) } diff --git a/internal/core/object_storage_inventory.go b/internal/core/object_storage_inventory.go index a9692210..396918d3 100644 --- a/internal/core/object_storage_inventory.go +++ b/internal/core/object_storage_inventory.go @@ -46,6 +46,14 @@ const ( ProjectStorageInspectSummary ProjectStorageInspectMode = "summary" ) +const ( + storageListCoalesceThreshold = 25 + storageListFallbackObjectLimit = 5000 + storageListSlowExactThreshold = time.Second + storageListSlowPrefixThreshold = 3 * time.Second + storagePrefixListLoggingDisabled = "disabled" +) + type ProjectStorageSummary struct { Provider string Bucket string @@ -69,6 +77,25 @@ type ProjectStorageInspectOptions struct { PathPrefix string } +type storageListTargetWork struct { + bucket string + key string + cred models.S3Credential + baseResult StorageListValidationResult + requestIndexes []int +} + +type storageListRunStats struct { + inputItemCount int + distinctTargetCount int + duplicateCount int + bucketCount int + coalescedPrefixCount int + exactListCallCount int + prefixListPageCount int + fallbackCount int +} + type StorageListValidationRequest struct { ID string ObjectURL string @@ -163,6 +190,18 @@ func (m *ObjectManager) InspectProjectStorage(ctx context.Context, organization, return &ProjectStorageInspectResult{Summary: summary, Items: out}, nil } +func (m *ObjectManager) ResolveProjectStoragePathPrefix(ctx context.Context, organization, project, pathPrefix string) (string, error) { + target, err := m.resolveProjectStorageScopeTarget(ctx, organization, project) + if err != nil { + return "", err + } + target = target.withPathPrefix(pathPrefix) + if target == nil { + return "", nil + } + return strings.Trim(strings.TrimSpace(target.prefix), "/"), nil +} + func (target *resolvedStorageScopeTarget) withPathPrefix(pathPrefix string) *resolvedStorageScopeTarget { trimmed := strings.Trim(strings.TrimSpace(pathPrefix), "/") if target == nil || trimmed == "" { @@ -177,6 +216,15 @@ func (target *resolvedStorageScopeTarget) withPathPrefix(pathPrefix string) *res return ©Target } +func withStoragePrefixListLogging(ctx context.Context, mode string) context.Context { + return context.WithValue(ctx, contextKey("storagePrefixListLogging"), strings.TrimSpace(mode)) +} + +func storagePrefixListLoggingEnabled(ctx context.Context) bool { + mode, _ := ctx.Value(contextKey("storagePrefixListLogging")).(string) + return strings.TrimSpace(mode) != storagePrefixListLoggingDisabled +} + func normalizeProjectStorageInspectMode(mode ProjectStorageInspectMode) ProjectStorageInspectMode { switch mode { case ProjectStorageInspectExists, ProjectStorageInspectSummary: @@ -373,35 +421,86 @@ func uniqueStorageObjectURLs(values []string) []string { func (m *ObjectManager) ListValidateStorageObjects(ctx context.Context, items []StorageListValidationRequest) []StorageListValidationResult { started := time.Now() ctx = WithStorageInspectCache(ctx) + ctx = withStoragePrefixListLogging(ctx, storagePrefixListLoggingDisabled) if len(items) == 0 { log.Printf("INFO: syfon_bulk_list_validate_done items=0 duration_ms=0") return []StorageListValidationResult{} } + visible, visibleErr := m.ListVisibleBuckets(ctx) results := make([]StorageListValidationResult, len(items)) - workers := len(items) - if workers > maxStorageInspectWorkers { - workers = maxStorageInspectWorkers + workByTarget := make(map[string]*storageListTargetWork) + buckets := map[string]struct{}{} + stats := storageListRunStats{inputItemCount: len(items)} + for index, item := range items { + result, targetWork, ok := m.resolveListValidationTarget(ctx, item, index, visible, visibleErr) + if !ok { + results[index] = result + continue + } + targetKey := storageListTargetKey(targetWork.bucket, targetWork.key) + if existing, found := workByTarget[targetKey]; found { + existing.requestIndexes = append(existing.requestIndexes, index) + stats.duplicateCount++ + continue + } + workByTarget[targetKey] = targetWork + buckets[targetWork.bucket] = struct{}{} } - type workItem struct { - index int - req StorageListValidationRequest + stats.distinctTargetCount = len(workByTarget) + stats.bucketCount = len(buckets) + + prefixGroups := groupStorageListTargetsByDirectory(workByTarget) + coalescedPrefixGroups := make([][]*storageListTargetWork, 0, len(prefixGroups)) + for _, group := range prefixGroups { + if len(group) < storageListCoalesceThreshold { + continue + } + stats.coalescedPrefixCount++ + coalescedPrefixGroups = append(coalescedPrefixGroups, group) } - workCh := make(chan workItem) - var wg sync.WaitGroup - for worker := 0; worker < workers; worker++ { - wg.Add(1) - go func() { - defer wg.Done() - for work := range workCh { - results[work.index] = m.listValidateStorageObject(ctx, work.req) - } - }() + log.Printf( + "INFO: syfon_bulk_list_validate_start items=%d distinct_targets=%d duplicates=%d buckets=%d coalesced_prefixes=%d", + stats.inputItemCount, + stats.distinctTargetCount, + stats.duplicateCount, + stats.bucketCount, + stats.coalescedPrefixCount, + ) + + outcomes := make(map[string]StorageListValidationResult, len(workByTarget)) + matchedItems := make(map[string]StorageBucketObject, len(workByTarget)) + unresolved := cloneStorageListWorkMap(workByTarget) + + for _, group := range coalescedPrefixGroups { + m.runCoalescedStorageListGroup(ctx, group, outcomes, matchedItems, unresolved, &stats) } - for index, item := range items { - workCh <- workItem{index: index, req: item} + m.runExactStorageListTargets(ctx, unresolved, outcomes, matchedItems, &stats) + + for targetKey, work := range workByTarget { + outcome, ok := outcomes[targetKey] + if !ok { + outcome = work.baseResult + outcome.Status = StorageProbeStatusNotFound + outcome.ErrorKind = string(StorageInspectObjectNotFound) + outcome.Error = fmt.Sprintf("object %q was not found", work.baseResult.ObjectURL) + } + for _, index := range work.requestIndexes { + req := items[index] + if item, found := matchedItems[targetKey]; found { + result := storageListValidationPresentResult(req, work.baseResult, item) + result.ID = strings.TrimSpace(req.ID) + result.ObjectURL = strings.TrimSpace(req.ObjectURL) + results[index] = result + continue + } + result := outcome + result.ID = strings.TrimSpace(req.ID) + result.ObjectURL = strings.TrimSpace(req.ObjectURL) + result.ValidationStatus = storageListValidationStatusForError(req) + results[index] = result + } } - close(workCh) - wg.Wait() + statusCounts := map[StorageProbeStatus]int{} validationCounts := map[StorageValidationStatus]int{} for _, result := range results { @@ -409,8 +508,16 @@ func (m *ObjectManager) ListValidateStorageObjects(ctx context.Context, items [] validationCounts[result.ValidationStatus]++ } log.Printf( - "INFO: syfon_bulk_list_validate_done items=%d present=%d not_found=%d invalid=%d error=%d matched=%d mismatched=%d unverifiable=%d duration_ms=%d", - len(items), + "INFO: syfon_bulk_list_validate_done items=%d distinct_targets=%d duplicates=%d buckets=%d coalesced_prefixes=%d exact_list_calls=%d prefix_list_pages=%d fallbacks=%d visible_error=%t present=%d not_found=%d invalid=%d error=%d matched=%d mismatched=%d unverifiable=%d duration_ms=%d", + stats.inputItemCount, + stats.distinctTargetCount, + stats.duplicateCount, + stats.bucketCount, + stats.coalescedPrefixCount, + stats.exactListCallCount, + stats.prefixListPageCount, + stats.fallbackCount, + visibleErr != nil, statusCounts[StorageProbeStatusPresent], statusCounts[StorageProbeStatusNotFound], statusCounts[StorageProbeStatusInvalid], @@ -423,7 +530,7 @@ func (m *ObjectManager) ListValidateStorageObjects(ctx context.Context, items [] return results } -func (m *ObjectManager) listValidateStorageObject(ctx context.Context, req StorageListValidationRequest) StorageListValidationResult { +func (m *ObjectManager) resolveListValidationTarget(ctx context.Context, req StorageListValidationRequest, index int, visible map[string]VisibleBucket, visibleErr error) (StorageListValidationResult, *storageListTargetWork, bool) { result := StorageListValidationResult{ ID: strings.TrimSpace(req.ID), ObjectURL: strings.TrimSpace(req.ObjectURL), @@ -435,21 +542,19 @@ func (m *ObjectManager) listValidateStorageObject(ctx context.Context, req Stora result.Status, result.ErrorKind = classifyStorageProbeError(err) result.Error = strings.TrimSpace(err.Error()) result.ValidationStatus = storageListValidationStatusForError(req) - return result + return result, nil, false } if !ok || target.provider != common.S3Provider { - log.Printf("INFO: syfon_bulk_list_validate_resolve id=%s object_url=%q status=invalid provider=%s bucket=%s key=%q", result.ID, result.ObjectURL, target.provider, target.bucket, target.key) result.Status = StorageProbeStatusInvalid result.ErrorKind = string(StorageInspectInvalidInput) result.Error = "object_url must be a valid s3://bucket/key URL" result.ValidationStatus = storageListValidationStatusForError(req) - return result + return result, nil, false } result.Provider = common.S3Provider result.Bucket = target.bucket result.Key = target.key result.Path = path.Base(target.key) - log.Printf("INFO: syfon_bulk_list_validate_resolve id=%s object_url=%q bucket=%s key=%q expected_size_set=%t expected_name=%q", result.ID, result.ObjectURL, target.bucket, target.key, req.ExpectedSizeBytes != nil, strings.TrimSpace(req.ExpectedName)) cred, err := m.credentialForBucket(ctx, target.bucket) if err != nil { @@ -457,16 +562,14 @@ func (m *ObjectManager) listValidateStorageObject(ctx context.Context, req Stora result.Status, result.ErrorKind = classifyStorageProbeError(err) result.Error = strings.TrimSpace(err.Error()) result.ValidationStatus = storageListValidationStatusForError(req) - return result + return result, nil, false } - log.Printf("INFO: syfon_bulk_list_validate_credential id=%s request_bucket=%s request_key=%q credential_id=%s credential_bucket=%s provider=%s", result.ID, target.bucket, target.key, credentialIDForCredential(*cred), strings.TrimSpace(cred.Bucket), strings.TrimSpace(cred.Provider)) - visible, err := m.ListVisibleBuckets(ctx) - if err != nil { - log.Printf("INFO: syfon_bulk_list_validate_visible id=%s bucket=%s key=%q error=%q", result.ID, target.bucket, target.key, err.Error()) - result.Status, result.ErrorKind = classifyStorageProbeError(err) - result.Error = strings.TrimSpace(err.Error()) + if visibleErr != nil { + log.Printf("INFO: syfon_bulk_list_validate_visible id=%s bucket=%s key=%q error=%q", result.ID, target.bucket, target.key, visibleErr.Error()) + result.Status, result.ErrorKind = classifyStorageProbeError(visibleErr) + result.Error = strings.TrimSpace(visibleErr.Error()) result.ValidationStatus = storageListValidationStatusForError(req) - return result + return result, nil, false } if !bucketVisibleToCaller(visible, target.bucket, credentialIDForCredential(*cred)) { err := &StorageInspectError{Kind: StorageInspectPermissionDenied, Message: fmt.Sprintf("bucket %q is not visible to the caller", target.bucket)} @@ -474,41 +577,179 @@ func (m *ObjectManager) listValidateStorageObject(ctx context.Context, req Stora result.Status, result.ErrorKind = classifyStorageProbeError(err) result.Error = err.Error() result.ValidationStatus = storageListValidationStatusForError(req) - return result + return result, nil, false + } + return result, &storageListTargetWork{ + bucket: target.bucket, + key: target.key, + cred: *cred, + baseResult: result, + requestIndexes: []int{index}, + }, true +} + +func groupStorageListTargetsByDirectory(workByTarget map[string]*storageListTargetWork) map[string][]*storageListTargetWork { + groups := make(map[string][]*storageListTargetWork) + for _, work := range workByTarget { + dirPrefix := storageListDirectoryPrefix(work.key) + groupKey := strings.TrimSpace(work.bucket) + "\x00" + dirPrefix + groups[groupKey] = append(groups[groupKey], work) + } + return groups +} + +func cloneStorageListWorkMap(input map[string]*storageListTargetWork) map[string]*storageListTargetWork { + out := make(map[string]*storageListTargetWork, len(input)) + for key, work := range input { + out[key] = work + } + return out +} + +func storageListTargetKey(bucket string, key string) string { + return strings.TrimSpace(bucket) + "\x00" + strings.Trim(strings.TrimSpace(key), "/") +} + +func storageListDirectoryPrefix(key string) string { + trimmed := strings.Trim(strings.TrimSpace(key), "/") + if trimmed == "" { + return "" } + dir := strings.Trim(strings.TrimSpace(path.Dir(trimmed)), "/") + if dir == "." || dir == "" { + return "" + } + return dir + "/" +} - listStart := time.Now() - log.Printf("INFO: syfon_bulk_list_validate_list_start id=%s bucket=%s prefix=%q exact_prefix=true max_keys=1", result.ID, target.bucket, target.key) - matches, err := m.listS3Prefix(ctx, *cred, target.bucket, target.key, StoragePrefixListOptions{ExactPrefix: true, MaxKeys: 1}) +func (m *ObjectManager) runCoalescedStorageListGroup(ctx context.Context, group []*storageListTargetWork, outcomes map[string]StorageListValidationResult, matchedItems map[string]StorageBucketObject, unresolved map[string]*storageListTargetWork, stats *storageListRunStats) { + if len(group) == 0 { + return + } + requestedKeys := make(map[string]*storageListTargetWork, len(group)) + for _, work := range group { + requestedKeys[storageListTargetKey(work.bucket, work.key)] = work + } + dirPrefix := storageListDirectoryPrefix(group[0].key) + started := time.Now() + listed, err := m.listS3Prefix(ctx, group[0].cred, group[0].bucket, dirPrefix, StoragePrefixListOptions{ExactPrefix: true}) if err != nil { - log.Printf("INFO: syfon_bulk_list_validate_list_done id=%s bucket=%s prefix=%q duration_ms=%d error=%q", result.ID, target.bucket, target.key, time.Since(listStart).Milliseconds(), err.Error()) + log.Printf("INFO: syfon_bulk_list_validate_prefix bucket=%s prefix=%q targets=%d duration_ms=%d error=%q", group[0].bucket, dirPrefix, len(group), time.Since(started).Milliseconds(), err.Error()) + stats.fallbackCount += len(group) + return + } + pageEstimate := 1 + if len(listed) > 1000 { + pageEstimate = (len(listed) + 999) / 1000 + } + stats.prefixListPageCount += pageEstimate + found := 0 + for _, item := range listed { + key := storageListTargetKey(group[0].bucket, item.Key) + work, ok := requestedKeys[key] + if !ok { + continue + } + normalized := normalizeStorageBucketObjects([]StorageBucketObject{item}, &resolvedStorageScopeTarget{provider: common.S3Provider, bucket: work.bucket, prefix: work.key}) + if len(normalized) == 0 { + continue + } + matchedItems[key] = normalized[0] + result := work.baseResult + result.Exists = true + result.Status = StorageProbeStatusPresent + result.Error = "" + result.ErrorKind = "" + outcomes[key] = result + delete(unresolved, key) + found++ + } + if len(listed) > storageListFallbackObjectLimit && found < len(group) { + stats.fallbackCount += len(group) - found + } + if duration := time.Since(started); duration > storageListSlowPrefixThreshold { + log.Printf("INFO: syfon_bulk_list_validate_prefix bucket=%s prefix=%q targets=%d listed_objects=%d matched_targets=%d duration_ms=%d", group[0].bucket, dirPrefix, len(group), len(listed), found, duration.Milliseconds()) + } +} + +func (m *ObjectManager) runExactStorageListTargets(ctx context.Context, unresolved map[string]*storageListTargetWork, outcomes map[string]StorageListValidationResult, matchedItems map[string]StorageBucketObject, stats *storageListRunStats) { + workers := len(unresolved) + if workers > maxStorageInspectWorkers { + workers = maxStorageInspectWorkers + } + if workers == 0 { + return + } + keys := make([]string, 0, len(unresolved)) + for key := range unresolved { + keys = append(keys, key) + } + sort.Strings(keys) + workCh := make(chan string) + var wg sync.WaitGroup + var mu sync.Mutex + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for targetKey := range workCh { + work := unresolved[targetKey] + outcome, item, matched := m.runExactStorageListTarget(ctx, work, stats) + mu.Lock() + outcomes[targetKey] = outcome + if matched { + matchedItems[targetKey] = item + } + mu.Unlock() + } + }() + } + for _, key := range keys { + workCh <- key + } + close(workCh) + wg.Wait() +} + +func (m *ObjectManager) runExactStorageListTarget(ctx context.Context, work *storageListTargetWork, stats *storageListRunStats) (StorageListValidationResult, StorageBucketObject, bool) { + started := time.Now() + matches, err := m.listS3Prefix(ctx, work.cred, work.bucket, work.key, StoragePrefixListOptions{ExactPrefix: true, MaxKeys: 1}) + if err != nil { + log.Printf("INFO: syfon_bulk_list_validate_exact bucket=%s key=%q duration_ms=%d error=%q", work.bucket, work.key, time.Since(started).Milliseconds(), err.Error()) + result := work.baseResult result.Status, result.ErrorKind = classifyStorageProbeError(err) result.Error = strings.TrimSpace(err.Error()) - result.ValidationStatus = storageListValidationStatusForError(req) - return result - } - firstKey := "" - if len(matches) > 0 { - firstKey = strings.Trim(strings.TrimSpace(matches[0].Key), "/") + stats.exactListCallCount++ + return result, StorageBucketObject{}, false } - log.Printf("INFO: syfon_bulk_list_validate_list_done id=%s bucket=%s prefix=%q duration_ms=%d matches=%d first_key=%q", result.ID, target.bucket, target.key, time.Since(listStart).Milliseconds(), len(matches), firstKey) + stats.exactListCallCount++ for _, item := range matches { key := strings.Trim(strings.TrimSpace(item.Key), "/") - if key != target.key { - log.Printf("INFO: syfon_bulk_list_validate_exact_miss id=%s requested_key=%q listed_key=%q", result.ID, target.key, key) + if key != work.key { continue } - normalized := normalizeStorageBucketObjects([]StorageBucketObject{item}, &resolvedStorageScopeTarget{provider: common.S3Provider, bucket: target.bucket, prefix: target.key}) + normalized := normalizeStorageBucketObjects([]StorageBucketObject{item}, &resolvedStorageScopeTarget{provider: common.S3Provider, bucket: work.bucket, prefix: work.key}) if len(normalized) == 0 { break } - return storageListValidationPresentResult(req, result, normalized[0]) + if duration := time.Since(started); duration > storageListSlowExactThreshold { + log.Printf("INFO: syfon_bulk_list_validate_exact bucket=%s key=%q duration_ms=%d status=present", work.bucket, work.key, duration.Milliseconds()) + } + result := work.baseResult + result.Exists = true + result.Status = StorageProbeStatusPresent + result.Error = "" + result.ErrorKind = "" + return result, normalized[0], true + } + if duration := time.Since(started); duration > storageListSlowExactThreshold { + log.Printf("INFO: syfon_bulk_list_validate_exact bucket=%s key=%q duration_ms=%d status=not_found", work.bucket, work.key, duration.Milliseconds()) } + result := work.baseResult result.Status = StorageProbeStatusNotFound result.ErrorKind = string(StorageInspectObjectNotFound) - result.Error = fmt.Sprintf("object %q was not found", req.ObjectURL) - result.ValidationStatus = storageListValidationStatusForError(req) - return result + result.Error = fmt.Sprintf("object %q was not found", work.baseResult.ObjectURL) + return result, StorageBucketObject{}, false } func storageListValidationPresentResult(req StorageListValidationRequest, result StorageListValidationResult, item StorageBucketObject) StorageListValidationResult { @@ -580,7 +821,10 @@ func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket input.MaxKeys = aws.Int32(options.MaxKeys) } requestPrefix := aws.ToString(input.Prefix) - log.Printf("INFO: syfon_s3_prefix_list_start bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead) + logEnabled := storagePrefixListLoggingEnabled(ctx) + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_start bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead) + } paginator := awss3.NewListObjectsV2Paginator(client, input) out := make([]StorageBucketObject, 0) pageCount := 0 @@ -588,7 +832,9 @@ func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket for paginator.HasMorePages() { page, err := paginator.NextPage(ctx) if err != nil { - log.Printf("INFO: syfon_s3_prefix_list_done bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d duration_ms=%d error=%q", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead, pageCount, len(out), time.Since(started).Milliseconds(), err.Error()) + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_done bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d duration_ms=%d error=%q", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead, pageCount, len(out), time.Since(started).Milliseconds(), err.Error()) + } return nil, classifyS3ListError(bucket, prefix, err) } pageCount++ @@ -620,7 +866,9 @@ func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket }) } } - log.Printf("INFO: syfon_s3_prefix_list_done bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d first_keys=%q duration_ms=%d", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead, pageCount, len(out), strings.Join(firstKeys, ","), time.Since(started).Milliseconds()) + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_done bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d first_keys=%q duration_ms=%d", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead, pageCount, len(out), strings.Join(firstKeys, ","), time.Since(started).Milliseconds()) + } if !options.IncludeHead || len(out) == 0 { return out, nil } From c134cd3915ebf9fce79a6561d5e91a098442fc49 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 3 Jul 2026 13:43:45 -0700 Subject: [PATCH 14/24] upgrade bulk s3 object deletion plumbing --- internal/core/object_mutations.go | 6 +- internal/core/object_storage_delete.go | 177 ++++++++++++++++++-- internal/core/object_storage_delete_test.go | 114 +++++++++++++ 3 files changed, 281 insertions(+), 16 deletions(-) diff --git a/internal/core/object_mutations.go b/internal/core/object_mutations.go index 21cc1d2c..d0b28418 100644 --- a/internal/core/object_mutations.go +++ b/internal/core/object_mutations.go @@ -108,10 +108,8 @@ func (m *ObjectManager) BulkDeleteObjectsWithOptions(ctx context.Context, ids [] if err != nil { return err } - for i := range objects { - if err := m.deleteObjectStorage(ctx, &objects[i]); err != nil { - return err - } + if err := m.deleteObjectsStorage(ctx, objects); err != nil { + return err } } return m.db.BulkDeleteObjects(ctx, toDelete) diff --git a/internal/core/object_storage_delete.go b/internal/core/object_storage_delete.go index 48bd8db8..8c58ac8d 100644 --- a/internal/core/object_storage_delete.go +++ b/internal/core/object_storage_delete.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "path/filepath" + "sort" "strings" "cloud.google.com/go/storage" @@ -17,6 +18,7 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" @@ -31,16 +33,78 @@ type storageTarget struct { path string } +type s3ObjectDeleter interface { + DeleteObject(context.Context, *awss3.DeleteObjectInput, ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) + DeleteObjects(context.Context, *awss3.DeleteObjectsInput, ...func(*awss3.Options)) (*awss3.DeleteObjectsOutput, error) +} + +var newS3ObjectDeleter = func(ctx context.Context, cred *models.S3Credential) (s3ObjectDeleter, error) { + cfg, err := awsConfigForStorageCredential(ctx, cred) + if err != nil { + return nil, err + } + return awss3.NewFromConfig(cfg, func(o *awss3.Options) { + if strings.TrimSpace(cred.Endpoint) != "" { + o.UsePathStyle = true + } + }), nil +} + func (m *ObjectManager) deleteObjectStorage(ctx context.Context, obj *models.InternalObject) error { targets, err := m.storageTargetsForObject(ctx, obj) if err != nil { return err } + return m.deleteStorageTargets(ctx, targets) +} + +func (m *ObjectManager) deleteObjectsStorage(ctx context.Context, objects []models.InternalObject) error { + targets := make([]storageTarget, 0, len(objects)) + seen := make(map[string]struct{}) + for i := range objects { + objectTargets, err := m.storageTargetsForObject(ctx, &objects[i]) + if err != nil { + return err + } + for _, target := range objectTargets { + key := storageTargetKey(target) + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + targets = append(targets, target) + } + } + return m.deleteStorageTargets(ctx, targets) +} + +func (m *ObjectManager) deleteStorageTargets(ctx context.Context, targets []storageTarget) error { + s3ByBucket := make(map[string][]string) for _, target := range targets { + if target.provider == common.S3Provider { + bucket := strings.TrimSpace(target.bucket) + key := strings.Trim(strings.TrimSpace(target.key), "/") + if bucket == "" || key == "" { + continue + } + s3ByBucket[bucket] = append(s3ByBucket[bucket], key) + continue + } if err := m.deleteStorageTarget(ctx, target); err != nil { return err } } + buckets := make([]string, 0, len(s3ByBucket)) + for bucket := range s3ByBucket { + buckets = append(buckets, bucket) + } + sort.Strings(buckets) + for _, bucket := range buckets { + keys := dedupeSortedStrings(s3ByBucket[bucket]) + if err := m.deleteS3Objects(ctx, bucket, keys); err != nil { + return err + } + } return nil } @@ -85,7 +149,7 @@ func (m *ObjectManager) storageTargetsForObject(ctx context.Context, obj *models continue } - key := target.provider + "\x00" + target.bucket + "\x00" + target.key + "\x00" + target.path + key := storageTargetKey(target) if _, exists := seen[key]; exists { continue } @@ -163,21 +227,70 @@ func (m *ObjectManager) deleteStorageTarget(ctx context.Context, target storageT } func (m *ObjectManager) deleteS3Object(ctx context.Context, bucket, key string) error { - cred, err := m.db.GetS3Credential(ctx, bucket) + return m.deleteS3Objects(ctx, bucket, []string{key}) +} + +func (m *ObjectManager) deleteS3Objects(ctx context.Context, bucket string, keys []string) error { + keys = dedupeSortedStrings(keys) + if len(keys) == 0 { + return nil + } + cred, err := m.credentialForBucket(ctx, bucket) if err != nil { return fmt.Errorf("lookup s3 credential for bucket %s: %w", bucket, err) } if cred == nil { return fmt.Errorf("credentials not found for bucket %s", bucket) } + client, err := newS3ObjectDeleter(ctx, cred) + if err != nil { + return err + } + + if len(keys) == 1 { + _, err = client.DeleteObject(ctx, &awss3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(keys[0]), + }) + return err + } + + const maxS3DeleteObjects = 1000 + quiet := true + for start := 0; start < len(keys); start += maxS3DeleteObjects { + end := start + maxS3DeleteObjects + if end > len(keys) { + end = len(keys) + } + objects := make([]s3types.ObjectIdentifier, 0, end-start) + for _, key := range keys[start:end] { + objects = append(objects, s3types.ObjectIdentifier{Key: aws.String(key)}) + } + out, err := client.DeleteObjects(ctx, &awss3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &s3types.Delete{ + Objects: objects, + Quiet: aws.Bool(quiet), + }, + }) + if err != nil { + return err + } + if out != nil && len(out.Errors) > 0 { + return fmt.Errorf("s3 bulk delete failed for bucket %s: %s", bucket, formatS3DeleteErrors(out.Errors)) + } + } + return nil +} +func awsConfigForStorageCredential(ctx context.Context, cred *models.S3Credential) (aws.Config, error) { loadOpts := []func(*awsconfig.LoadOptions) error{ awsconfig.WithRegion(cred.Region), awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cred.AccessKey, cred.SecretKey, "")), } cfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...) if err != nil { - return fmt.Errorf("load aws config: %w", err) + return aws.Config{}, fmt.Errorf("load aws config: %w", err) } if endpoint := strings.TrimSpace(cred.Endpoint); endpoint != "" { if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { @@ -189,17 +302,57 @@ func (m *ObjectManager) deleteS3Object(ctx context.Context, bucket, key string) } cfg.BaseEndpoint = aws.String(endpoint) } + return cfg, nil +} + +func storageTargetKey(target storageTarget) string { + return target.provider + "\x00" + target.bucket + "\x00" + target.key + "\x00" + target.path +} + +func dedupeSortedStrings(values []string) []string { + if len(values) == 0 { + return nil + } + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + sort.Strings(out) + return out +} - client := awss3.NewFromConfig(cfg, func(o *awss3.Options) { - if strings.TrimSpace(cred.Endpoint) != "" { - o.UsePathStyle = true +func formatS3DeleteErrors(errors []s3types.Error) string { + const maxErrors = 5 + parts := make([]string, 0, len(errors)) + for i, item := range errors { + if i >= maxErrors { + parts = append(parts, fmt.Sprintf("and %d more", len(errors)-maxErrors)) + break } - }) - _, err = client.DeleteObject(ctx, &awss3.DeleteObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) - return err + key := strings.TrimSpace(aws.ToString(item.Key)) + code := strings.TrimSpace(aws.ToString(item.Code)) + message := strings.TrimSpace(aws.ToString(item.Message)) + switch { + case key != "" && code != "" && message != "": + parts = append(parts, fmt.Sprintf("%s: %s: %s", key, code, message)) + case key != "" && code != "": + parts = append(parts, fmt.Sprintf("%s: %s", key, code)) + case code != "": + parts = append(parts, code) + default: + parts = append(parts, "unknown delete error") + } + } + return strings.Join(parts, "; ") } func (m *ObjectManager) deleteGCSObject(ctx context.Context, bucket, key string) error { diff --git a/internal/core/object_storage_delete_test.go b/internal/core/object_storage_delete_test.go index cf33d854..1e4dea80 100644 --- a/internal/core/object_storage_delete_test.go +++ b/internal/core/object_storage_delete_test.go @@ -2,11 +2,15 @@ package core import ( "context" + "fmt" "os" "path/filepath" + "reflect" "strings" "testing" + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" @@ -81,6 +85,87 @@ func TestDeleteStorageTargetFileProvider(t *testing.T) { } } +func TestBulkDeleteObjectsWithOptionsUsesS3BatchDelete(t *testing.T) { + deleter := &fakeS3ObjectDeleter{} + restore := replaceS3ObjectDeleterForTest(deleter) + defer restore() + + accessMethods := func(rawURL string) *[]drs.AccessMethod { + return &[]drs.AccessMethod{{ + Type: drs.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: rawURL}, + }} + } + db := &testutils.MockDatabase{ + Objects: map[string]*drs.DrsObject{ + "obj-1": {Id: "obj-1", AccessMethods: accessMethods("s3://bucket/path/a.txt")}, + "obj-2": {Id: "obj-2", AccessMethods: accessMethods("s3://bucket/path/b.txt")}, + "obj-3": {Id: "obj-3", AccessMethods: accessMethods("s3://bucket/path/a.txt")}, + }, + Credentials: map[string]models.S3Credential{ + "bucket": { + Bucket: "bucket", + Provider: common.S3Provider, + Region: "us-east-1", + AccessKey: "test-key", + SecretKey: "test-secret", + }, + }, + } + om := NewObjectManager(db, &capturingURLManager{}) + + if err := om.BulkDeleteObjectsWithOptions(context.Background(), []string{"obj-1", "obj-2", "obj-3"}, DeleteOptions{DeleteStorageData: true}); err != nil { + t.Fatalf("BulkDeleteObjectsWithOptions failed: %v", err) + } + if deleter.deleteObjectCalls != 0 { + t.Fatalf("expected no single DeleteObject calls, got %d", deleter.deleteObjectCalls) + } + if len(deleter.deleteObjectsKeys) != 1 { + t.Fatalf("expected one DeleteObjects call, got %d", len(deleter.deleteObjectsKeys)) + } + if got, want := deleter.deleteObjectsKeys[0], []string{"path/a.txt", "path/b.txt"}; !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected batch keys: got %+v want %+v", got, want) + } + for _, id := range []string{"obj-1", "obj-2", "obj-3"} { + if _, exists := db.Objects[id]; exists { + t.Fatalf("expected %s to be removed from db", id) + } + } +} + +func TestDeleteS3ObjectsChunksLargeBatches(t *testing.T) { + deleter := &fakeS3ObjectDeleter{} + restore := replaceS3ObjectDeleterForTest(deleter) + defer restore() + + keys := make([]string, 0, 1002) + for i := 0; i < 1002; i++ { + keys = append(keys, fmt.Sprintf("key-%04d", i)) + } + keys = append(keys, keys[0]) + om := NewObjectManager(&testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "bucket": {Bucket: "bucket", Provider: common.S3Provider, Region: "us-east-1"}, + }, + }, &capturingURLManager{}) + + if err := om.deleteS3Objects(context.Background(), "bucket", keys); err != nil { + t.Fatalf("deleteS3Objects failed: %v", err) + } + if len(deleter.deleteObjectsKeys) != 2 { + t.Fatalf("expected two DeleteObjects chunks, got %d", len(deleter.deleteObjectsKeys)) + } + if got := len(deleter.deleteObjectsKeys[0]); got != 1000 { + t.Fatalf("expected first chunk to contain 1000 keys, got %d", got) + } + if got := len(deleter.deleteObjectsKeys[1]); got != 2 { + t.Fatalf("expected second chunk to contain 2 keys, got %d", got) + } +} + func TestStorageTargetsForScopedObjectUseCanonicalChecksumPath(t *testing.T) { checksum := strings.Repeat("a", 64) om := NewObjectManager(&testutils.MockDatabase{ @@ -132,6 +217,35 @@ func TestStorageTargetsForScopedObjectUseCanonicalChecksumPath(t *testing.T) { } } +type fakeS3ObjectDeleter struct { + deleteObjectCalls int + deleteObjectsKeys [][]string +} + +func (f *fakeS3ObjectDeleter) DeleteObject(context.Context, *awss3.DeleteObjectInput, ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) { + f.deleteObjectCalls++ + return &awss3.DeleteObjectOutput{}, nil +} + +func (f *fakeS3ObjectDeleter) DeleteObjects(_ context.Context, input *awss3.DeleteObjectsInput, _ ...func(*awss3.Options)) (*awss3.DeleteObjectsOutput, error) { + keys := make([]string, 0, len(input.Delete.Objects)) + for _, obj := range input.Delete.Objects { + keys = append(keys, aws.ToString(obj.Key)) + } + f.deleteObjectsKeys = append(f.deleteObjectsKeys, keys) + return &awss3.DeleteObjectsOutput{}, nil +} + +func replaceS3ObjectDeleterForTest(deleter s3ObjectDeleter) func() { + previous := newS3ObjectDeleter + newS3ObjectDeleter = func(context.Context, *models.S3Credential) (s3ObjectDeleter, error) { + return deleter, nil + } + return func() { + newS3ObjectDeleter = previous + } +} + func TestScopedStorageHelperUtilities(t *testing.T) { bucket, key, ok := parseS3Location("s3://bucket-name/path/to/object") if !ok || bucket != "bucket-name" || key != "path/to/object" { From 3c25db97e481c096860122e6b606db160ffe0baa Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 6 Jul 2026 11:07:36 -0700 Subject: [PATCH 15/24] fix an issue where project query with limit would not stop at first limit rows fetched --- internal/api/internaldrs/index.go | 9 +++ internal/api/internaldrs/repair_cleanup.go | 4 ++ internal/core/object_reads.go | 12 ++++ internal/core/object_reads_test.go | 67 ++++++++++++++++++++++ 4 files changed, 92 insertions(+) diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index b4e2c9de..97cd6d95 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -2,6 +2,7 @@ package internaldrs import ( "fmt" + "log" "strconv" "strings" "time" @@ -99,6 +100,8 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { return c.Status(fiber.StatusBadRequest).SendString(err.Error()) } + requestStart := time.Now() + listStart := time.Now() var ids []string if objectURL != "" { ids, err = om.ListObjectIDsPageByURL(c.Context(), objectURL, filterOrg, filterProject, "read", start, limit, offset) @@ -108,19 +111,25 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { if err != nil { return apiutil.HandleError(c, err) } + listDuration := time.Since(listStart) + bulkStart := time.Now() objs, err := om.GetBulkObjects(c.Context(), ids, "read") if err != nil { return apiutil.HandleError(c, err) } + bulkDuration := time.Since(bulkStart) + prepareStart := time.Now() objs, err = om.PrepareScopedObjects(c.Context(), objs, filterOrg, filterProject, "read") if err != nil { return apiutil.HandleError(c, err) } + prepareDuration := time.Since(prepareStart) records := make([]internalapi.InternalRecord, 0, len(objs)) for _, obj := range objs { records = append(records, core.InternalObjectToInternalRecord(obj)) } + log.Printf("INFO: syfon_internal_index_list organization=%s project=%s url_filter=%t start_after=%t limit=%d offset=%d ids=%d records=%d list_ids_ms=%d bulk_objects_ms=%d prepare_scoped_ms=%d duration_ms=%d", filterOrg, filterProject, objectURL != "", strings.TrimSpace(start) != "", limit, offset, len(ids), len(records), listDuration.Milliseconds(), bulkDuration.Milliseconds(), prepareDuration.Milliseconds(), time.Since(requestStart).Milliseconds()) return c.JSON(internalapi.ListRecordsResponse{Records: &records}) } } diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go index 7e1506a0..07ccdbbf 100644 --- a/internal/api/internaldrs/repair_cleanup.go +++ b/internal/api/internaldrs/repair_cleanup.go @@ -113,6 +113,10 @@ func (a scopeRepairIndexAdapter) List(ctx context.Context, opts repair.ListRecor if err != nil { return clientinternalapi.ListRecordsResponse{}, err } + objects, err = a.om.PrepareScopedObjects(ctx, objects, strings.TrimSpace(opts.Organization), strings.TrimSpace(opts.Project), "read") + if err != nil { + return clientinternalapi.ListRecordsResponse{}, err + } records := make([]clientinternalapi.InternalRecord, 0, len(objects)) for _, obj := range objects { records = append(records, clientRecordFromServer(core.InternalObjectToInternalRecord(obj))) diff --git a/internal/core/object_reads.go b/internal/core/object_reads.go index 98272afe..cb201bae 100644 --- a/internal/core/object_reads.go +++ b/internal/core/object_reads.go @@ -184,6 +184,13 @@ func (m *ObjectManager) ListObjectIDsPageByScope(ctx context.Context, organizati return []string{}, nil } + if lister, ok := m.db.(db.ObjectIDPageLister); ok && canUseUnrestrictedScopePage(ctx, requiredMethod) { + pageStart := time.Now() + ids, err := lister.ListObjectIDsPageByScope(ctx, organization, project, startAfter, limit, offset) + log.Printf("INFO: syfon_list_object_ids_page_by_scope organization=%s project=%s start_after=%t limit=%d offset=%d ids=%d db_page_ms=%d optimized=%t", strings.TrimSpace(organization), strings.TrimSpace(project), strings.TrimSpace(startAfter) != "", limit, offset, len(ids), time.Since(pageStart).Milliseconds(), true) + return ids, err + } + ids, err := m.ListObjectIDsByScope(ctx, organization, project, requiredMethod) if err != nil { return nil, err @@ -205,6 +212,11 @@ func (m *ObjectManager) ListObjectIDsPageByScope(ctx context.Context, organizati return ids[offset:end], nil } +func canUseUnrestrictedScopePage(ctx context.Context, requiredMethod string) bool { + resources, includeUnscoped, restrictToResources := objectMethodResourceFilter(ctx, requiredMethod) + return !restrictToResources && len(resources) == 0 && includeUnscoped +} + func (m *ObjectManager) ListObjectIDsPageByURL(ctx context.Context, objectURL, organization, project, requiredMethod, startAfter string, limit, offset int) ([]string, error) { if limit <= 0 { return []string{}, nil diff --git a/internal/core/object_reads_test.go b/internal/core/object_reads_test.go index 40c53f37..00454158 100644 --- a/internal/core/object_reads_test.go +++ b/internal/core/object_reads_test.go @@ -12,6 +12,26 @@ import ( "github.com/calypr/syfon/internal/models" ) +type pageSpyDB struct { + db.DatabaseInterface + pageCalls int + listCalls int +} + +func (s *pageSpyDB) ListObjectIDsPageByScope(ctx context.Context, organization, project, startAfter string, limit, offset int) ([]string, error) { + s.pageCalls++ + return s.DatabaseInterface.(db.ObjectIDPageLister).ListObjectIDsPageByScope(ctx, organization, project, startAfter, limit, offset) +} + +func (s *pageSpyDB) ListObjectIDsPageByResources(ctx context.Context, resources []string, includeUnscoped bool, startAfter string, limit, offset int) ([]string, error) { + return s.DatabaseInterface.(db.ObjectIDPageLister).ListObjectIDsPageByResources(ctx, resources, includeUnscoped, startAfter, limit, offset) +} + +func (s *pageSpyDB) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) { + s.listCalls++ + return s.DatabaseInterface.ListObjectIDsByScope(ctx, organization, project) +} + func registerScopedCandidate(t *testing.T, om *ObjectManager, id, checksum, org, project string) { t.Helper() controlled := []string{"/organization/" + org + "/project/" + project} @@ -73,6 +93,53 @@ func TestListObjectIDsPageByScope_StartAfterAndScopeFilter(t *testing.T) { } } +func TestListObjectIDsPageByScope_UsesDatabasePaginationForUnrestrictedScope(t *testing.T) { + database := &pageSpyDB{DatabaseInterface: db.NewInMemoryDB()} + om := NewObjectManager(database, nil) + + registerScopedCandidate(t, om, "scope-a", "2222222222222222222222222222222222222222222222222222222222222222", "org1", "proj1") + registerScopedCandidate(t, om, "scope-b", "3333333333333333333333333333333333333333333333333333333333333333", "org1", "proj1") + registerScopedCandidate(t, om, "scope-c", "4444444444444444444444444444444444444444444444444444444444444444", "org1", "proj1") + + ids, err := om.ListObjectIDsPageByScope(context.Background(), "org1", "proj1", "read", "scope-a", 1, 0) + if err != nil { + t.Fatalf("ListObjectIDsPageByScope error: %v", err) + } + if len(ids) != 1 || ids[0] != "scope-b" { + t.Fatalf("unexpected scoped page ids: %+v", ids) + } + if database.pageCalls != 1 { + t.Fatalf("expected one database page call, got %d", database.pageCalls) + } + if database.listCalls != 0 { + t.Fatalf("expected no full scope list calls, got %d", database.listCalls) + } +} + +func TestListObjectIDsPageByScope_FallsBackWhenAuthzRestrictsResources(t *testing.T) { + database := &pageSpyDB{DatabaseInterface: db.NewInMemoryDB()} + om := NewObjectManager(database, nil) + + registerScopedCandidate(t, om, "secure-obj", "5555555555555555555555555555555555555555555555555555555555555555", "secure", "p1") + restrictedCtx := buildLocalAuthzContext(map[string]map[string]bool{ + "/organization/other/project/p2": {"read": true}, + }) + + ids, err := om.ListObjectIDsPageByScope(restrictedCtx, "secure", "p1", "read", "", 10, 0) + if err != nil { + t.Fatalf("ListObjectIDsPageByScope error: %v", err) + } + if len(ids) != 0 { + t.Fatalf("expected authz fallback to filter ids, got %+v", ids) + } + if database.pageCalls != 0 { + t.Fatalf("expected no unrestricted page calls, got %d", database.pageCalls) + } + if database.listCalls == 0 { + t.Fatalf("expected fallback to full scope list") + } +} + func TestListObjectIDsByScope_AuthzFiltering(t *testing.T) { database := db.NewInMemoryDB() om := NewObjectManager(database, nil) From eddcac7a62490b85b6b8f4a9dd9ca2969a6aea4b Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 7 Jul 2026 13:24:24 -0700 Subject: [PATCH 16/24] add faster methods for listing files --- apigen/client/internalapi/internal.gen.go | 181 ++++++++++++++++ apigen/client/metricsapi/metrics.gen.go | 205 +++++++++++++++++- apigen/openapi/internal.openapi.yaml | 93 ++++++++ apigen/openapi/metrics.openapi.yaml | 49 +++++ apigen/server/internalapi/internal.gen.go | 152 +++++++++++++ apigen/server/metricsapi/metrics.gen.go | 150 ++++++++++++- internal/api/internaldrs/index.go | 45 ++-- internal/api/internaldrs/index_test.go | 98 +++++++++ internal/api/internaldrs/inspect.go | 82 +++++-- .../api/internaldrs/inspect_routes_test.go | 59 +++++ internal/api/internaldrs/repair_cleanup.go | 10 +- internal/api/internaldrs/transfer.go | 1 + internal/api/metrics/file_usage.go | 148 ++++++++++++- internal/api/metrics/file_usage_test.go | 85 +++++++- internal/common/routes.go | 43 ++-- internal/core/object_reads.go | 77 +++++++ internal/db/interface.go | 1 + internal/db/postgres/metrics.go | 25 ++- ...lize_object_names_to_basename_postgres.sql | 85 ++++++++ internal/db/sqlite/metrics.go | 25 ++- internal/db/sqlite/sqlite_test.go | 11 + internal/models/storage.go | 10 +- 22 files changed, 1540 insertions(+), 95 deletions(-) create mode 100644 internal/db/scripts/normalize_object_names_to_basename_postgres.sql diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index e50169cc..4143f1ca 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -100,6 +100,45 @@ type InternalMultipartUploadRequest struct { UploadId string `json:"uploadId"` } +// InternalProjectBucketInventoryItem defines model for InternalProjectBucketInventoryItem. +type InternalProjectBucketInventoryItem struct { + Bucket *string `json:"bucket,omitempty"` + Etag *string `json:"etag,omitempty"` + Key *string `json:"key,omitempty"` + LastModified *string `json:"last_modified,omitempty"` + MetaSha256 *string `json:"meta_sha256,omitempty"` + ObjectUrl *string `json:"object_url,omitempty"` + Path *string `json:"path,omitempty"` + Provider *string `json:"provider,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` +} + +// InternalProjectBucketInventoryRequest defines model for InternalProjectBucketInventoryRequest. +type InternalProjectBucketInventoryRequest struct { + Organization *string `json:"organization,omitempty"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project *string `json:"project,omitempty"` +} + +// InternalProjectBucketInventoryResponse defines model for InternalProjectBucketInventoryResponse. +type InternalProjectBucketInventoryResponse struct { + Items *[]InternalProjectBucketInventoryItem `json:"items,omitempty"` + Summary *InternalProjectBucketInventorySummary `json:"summary,omitempty"` +} + +// InternalProjectBucketInventorySummary defines model for InternalProjectBucketInventorySummary. +type InternalProjectBucketInventorySummary struct { + Bucket *string `json:"bucket,omitempty"` + ComputedAt *string `json:"computed_at,omitempty"` + Exists *bool `json:"exists,omitempty"` + Mode *string `json:"mode,omitempty"` + ObjectCount *int `json:"object_count,omitempty"` + ObjectUrl *string `json:"object_url,omitempty"` + Prefix *string `json:"prefix,omitempty"` + Provider *string `json:"provider,omitempty"` + TotalBytes *int64 `json:"total_bytes,omitempty"` +} + // InternalRecord defines model for InternalRecord. type InternalRecord struct { AccessMethods *[]externalRef0.AccessMethod `json:"access_methods,omitempty"` @@ -238,6 +277,9 @@ type InternalListParams struct { Page *int `form:"page,omitempty" json:"page,omitempty"` } +// InternalInspectProjectBucketInventoryJSONRequestBody defines body for InternalInspectProjectBucketInventory for application/json ContentType. +type InternalInspectProjectBucketInventoryJSONRequestBody = InternalProjectBucketInventoryRequest + // InternalMultipartCompleteJSONRequestBody defines body for InternalMultipartComplete for application/json ContentType. type InternalMultipartCompleteJSONRequestBody = InternalMultipartCompleteRequest @@ -418,6 +460,11 @@ type ClientInterface interface { // InternalDownloadPart request InternalDownloadPart(ctx context.Context, fileId string, params *InternalDownloadPartParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalInspectProjectBucketInventoryWithBody request with any body + InternalInspectProjectBucketInventoryWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + InternalInspectProjectBucketInventory(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalMultipartCompleteWithBody request with any body InternalMultipartCompleteWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -523,6 +570,30 @@ func (c *Client) InternalDownloadPart(ctx context.Context, fileId string, params return c.Client.Do(req) } +func (c *Client) InternalInspectProjectBucketInventoryWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectBucketInventoryRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalInspectProjectBucketInventory(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectBucketInventoryRequest(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) InternalMultipartCompleteWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewInternalMultipartCompleteRequestWithBody(c.Server, contentType, body) if err != nil { @@ -1031,6 +1102,46 @@ func NewInternalDownloadPartRequest(server string, fileId string, params *Intern return req, nil } +// NewInternalInspectProjectBucketInventoryRequest calls the generic InternalInspectProjectBucketInventory builder with application/json body +func NewInternalInspectProjectBucketInventoryRequest(server string, body InternalInspectProjectBucketInventoryJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewInternalInspectProjectBucketInventoryRequestWithBody(server, "application/json", bodyReader) +} + +// NewInternalInspectProjectBucketInventoryRequestWithBody generates requests for InternalInspectProjectBucketInventory with any type of body +func NewInternalInspectProjectBucketInventoryRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/data/inspect/project-bucket/inventory") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewInternalMultipartCompleteRequest calls the generic InternalMultipartComplete builder with application/json body func NewInternalMultipartCompleteRequest(server string, body InternalMultipartCompleteJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -2076,6 +2187,11 @@ type ClientWithResponsesInterface interface { // InternalDownloadPartWithResponse request InternalDownloadPartWithResponse(ctx context.Context, fileId string, params *InternalDownloadPartParams, reqEditors ...RequestEditorFn) (*InternalDownloadPartResponse, error) + // InternalInspectProjectBucketInventoryWithBodyWithResponse request with any body + InternalInspectProjectBucketInventoryWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalInspectProjectBucketInventoryResponse, error) + + InternalInspectProjectBucketInventoryWithResponse(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalInspectProjectBucketInventoryResponse, error) + // InternalMultipartCompleteWithBodyWithResponse request with any body InternalMultipartCompleteWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalMultipartCompleteResponse, error) @@ -2201,6 +2317,28 @@ func (r InternalDownloadPartResponse) StatusCode() int { return 0 } +type InternalInspectProjectBucketInventoryResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *InternalProjectBucketInventoryResponse +} + +// Status returns HTTPResponse.Status +func (r InternalInspectProjectBucketInventoryResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalInspectProjectBucketInventoryResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type InternalMultipartCompleteResponse struct { Body []byte HTTPResponse *http.Response @@ -2614,6 +2752,23 @@ func (c *ClientWithResponses) InternalDownloadPartWithResponse(ctx context.Conte return ParseInternalDownloadPartResponse(rsp) } +// InternalInspectProjectBucketInventoryWithBodyWithResponse request with arbitrary body returning *InternalInspectProjectBucketInventoryResponse +func (c *ClientWithResponses) InternalInspectProjectBucketInventoryWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalInspectProjectBucketInventoryResponse, error) { + rsp, err := c.InternalInspectProjectBucketInventoryWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectBucketInventoryResponse(rsp) +} + +func (c *ClientWithResponses) InternalInspectProjectBucketInventoryWithResponse(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalInspectProjectBucketInventoryResponse, error) { + rsp, err := c.InternalInspectProjectBucketInventory(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectBucketInventoryResponse(rsp) +} + // InternalMultipartCompleteWithBodyWithResponse request with arbitrary body returning *InternalMultipartCompleteResponse func (c *ClientWithResponses) InternalMultipartCompleteWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalMultipartCompleteResponse, error) { rsp, err := c.InternalMultipartCompleteWithBody(ctx, contentType, body, reqEditors...) @@ -2932,6 +3087,32 @@ func ParseInternalDownloadPartResponse(rsp *http.Response) (*InternalDownloadPar return response, nil } +// ParseInternalInspectProjectBucketInventoryResponse parses an HTTP response from a InternalInspectProjectBucketInventoryWithResponse call +func ParseInternalInspectProjectBucketInventoryResponse(rsp *http.Response) (*InternalInspectProjectBucketInventoryResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalInspectProjectBucketInventoryResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest InternalProjectBucketInventoryResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseInternalMultipartCompleteResponse parses an HTTP response from a InternalMultipartCompleteWithResponse call func ParseInternalMultipartCompleteResponse(rsp *http.Response) (*InternalMultipartCompleteResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/client/metricsapi/metrics.gen.go b/apigen/client/metricsapi/metrics.gen.go index 863b0183..5a117d21 100644 --- a/apigen/client/metricsapi/metrics.gen.go +++ b/apigen/client/metricsapi/metrics.gen.go @@ -59,12 +59,20 @@ type FileUsage struct { UploadCount *int64 `json:"upload_count,omitempty"` } +// FileUsageBulkRequest defines model for FileUsageBulkRequest. +type FileUsageBulkRequest struct { + InactiveDays *int `json:"inactive_days,omitempty"` + ObjectIds []string `json:"object_ids"` +} + // FileUsageSummary defines model for FileUsageSummary. type FileUsageSummary struct { - InactiveFileCount *int64 `json:"inactive_file_count,omitempty"` - TotalDownloads *int64 `json:"total_downloads,omitempty"` - TotalFiles *int64 `json:"total_files,omitempty"` - TotalUploads *int64 `json:"total_uploads,omitempty"` + InactiveFileCount *int64 `json:"inactive_file_count,omitempty"` + RecordCount *int64 `json:"record_count,omitempty"` + RecordLatestUpdatedTime *time.Time `json:"record_latest_updated_time"` + TotalDownloads *int64 `json:"total_downloads,omitempty"` + TotalFiles *int64 `json:"total_files,omitempty"` + TotalUploads *int64 `json:"total_uploads,omitempty"` } // MetricsListResponse defines model for MetricsListResponse. @@ -221,6 +229,15 @@ type ListMetricsFilesParams struct { Project *Project `form:"project,omitempty" json:"project,omitempty"` } +// BulkMetricsFilesParams defines parameters for BulkMetricsFiles. +type BulkMetricsFilesParams struct { + // Organization Organization/program scope filter. + Organization *Organization `form:"organization,omitempty" json:"organization,omitempty"` + + // Project Project scope filter. Requires organization when set. + Project *Project `form:"project,omitempty" json:"project,omitempty"` +} + // GetMetricsFileParams defines parameters for GetMetricsFile. type GetMetricsFileParams struct { // Organization Organization/program scope filter. @@ -298,6 +315,9 @@ type GetTransferSummaryParams struct { AllowStale *AllowStale `form:"allow_stale,omitempty" json:"allow_stale,omitempty"` } +// BulkMetricsFilesJSONRequestBody defines body for BulkMetricsFiles for application/json ContentType. +type BulkMetricsFilesJSONRequestBody = FileUsageBulkRequest + // RecordProviderTransferEventsJSONRequestBody defines body for RecordProviderTransferEvents for application/json ContentType. type RecordProviderTransferEventsJSONRequestBody = ProviderTransferEventsRequest @@ -377,6 +397,11 @@ type ClientInterface interface { // ListMetricsFiles request ListMetricsFiles(ctx context.Context, params *ListMetricsFilesParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // BulkMetricsFilesWithBody request with any body + BulkMetricsFilesWithBody(ctx context.Context, params *BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + BulkMetricsFiles(ctx context.Context, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetMetricsFile request GetMetricsFile(ctx context.Context, objectId string, params *GetMetricsFileParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -407,6 +432,30 @@ func (c *Client) ListMetricsFiles(ctx context.Context, params *ListMetricsFilesP return c.Client.Do(req) } +func (c *Client) BulkMetricsFilesWithBody(ctx context.Context, params *BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewBulkMetricsFilesRequestWithBody(c.Server, params, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) BulkMetricsFiles(ctx context.Context, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewBulkMetricsFilesRequest(c.Server, params, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) GetMetricsFile(ctx context.Context, objectId string, params *GetMetricsFileParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetMetricsFileRequest(c.Server, objectId, params) if err != nil { @@ -592,6 +641,84 @@ func NewListMetricsFilesRequest(server string, params *ListMetricsFilesParams) ( return req, nil } +// NewBulkMetricsFilesRequest calls the generic BulkMetricsFiles builder with application/json body +func NewBulkMetricsFilesRequest(server string, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewBulkMetricsFilesRequestWithBody(server, params, "application/json", bodyReader) +} + +// NewBulkMetricsFilesRequestWithBody generates requests for BulkMetricsFiles with any type of body +func NewBulkMetricsFilesRequestWithBody(server string, params *BulkMetricsFilesParams, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/index/v1/metrics/files/bulk") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if params.Organization != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "organization", runtime.ParamLocationQuery, *params.Organization); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.Project != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "project", runtime.ParamLocationQuery, *params.Project); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewGetMetricsFileRequest generates requests for GetMetricsFile func NewGetMetricsFileRequest(server string, objectId string, params *GetMetricsFileParams) (*http.Request, error) { var err error @@ -1303,6 +1430,11 @@ type ClientWithResponsesInterface interface { // ListMetricsFilesWithResponse request ListMetricsFilesWithResponse(ctx context.Context, params *ListMetricsFilesParams, reqEditors ...RequestEditorFn) (*ListMetricsFilesResponse, error) + // BulkMetricsFilesWithBodyWithResponse request with any body + BulkMetricsFilesWithBodyWithResponse(ctx context.Context, params *BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*BulkMetricsFilesResponse, error) + + BulkMetricsFilesWithResponse(ctx context.Context, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody, reqEditors ...RequestEditorFn) (*BulkMetricsFilesResponse, error) + // GetMetricsFileWithResponse request GetMetricsFileWithResponse(ctx context.Context, objectId string, params *GetMetricsFileParams, reqEditors ...RequestEditorFn) (*GetMetricsFileResponse, error) @@ -1343,6 +1475,28 @@ func (r ListMetricsFilesResponse) StatusCode() int { return 0 } +type BulkMetricsFilesResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *MetricsListResponse +} + +// Status returns HTTPResponse.Status +func (r BulkMetricsFilesResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r BulkMetricsFilesResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type GetMetricsFileResponse struct { Body []byte HTTPResponse *http.Response @@ -1462,6 +1616,23 @@ func (c *ClientWithResponses) ListMetricsFilesWithResponse(ctx context.Context, return ParseListMetricsFilesResponse(rsp) } +// BulkMetricsFilesWithBodyWithResponse request with arbitrary body returning *BulkMetricsFilesResponse +func (c *ClientWithResponses) BulkMetricsFilesWithBodyWithResponse(ctx context.Context, params *BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*BulkMetricsFilesResponse, error) { + rsp, err := c.BulkMetricsFilesWithBody(ctx, params, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseBulkMetricsFilesResponse(rsp) +} + +func (c *ClientWithResponses) BulkMetricsFilesWithResponse(ctx context.Context, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody, reqEditors ...RequestEditorFn) (*BulkMetricsFilesResponse, error) { + rsp, err := c.BulkMetricsFiles(ctx, params, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseBulkMetricsFilesResponse(rsp) +} + // GetMetricsFileWithResponse request returning *GetMetricsFileResponse func (c *ClientWithResponses) GetMetricsFileWithResponse(ctx context.Context, objectId string, params *GetMetricsFileParams, reqEditors ...RequestEditorFn) (*GetMetricsFileResponse, error) { rsp, err := c.GetMetricsFile(ctx, objectId, params, reqEditors...) @@ -1541,6 +1712,32 @@ func ParseListMetricsFilesResponse(rsp *http.Response) (*ListMetricsFilesRespons return response, nil } +// ParseBulkMetricsFilesResponse parses an HTTP response from a BulkMetricsFilesWithResponse call +func ParseBulkMetricsFilesResponse(rsp *http.Response) (*BulkMetricsFilesResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &BulkMetricsFilesResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest MetricsListResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseGetMetricsFileResponse parses an HTTP response from a GetMetricsFileWithResponse call func ParseGetMetricsFileResponse(rsp *http.Response) (*GetMetricsFileResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/openapi/internal.openapi.yaml b/apigen/openapi/internal.openapi.yaml index 3e194d56..a8160810 100644 --- a/apigen/openapi/internal.openapi.yaml +++ b/apigen/openapi/internal.openapi.yaml @@ -368,6 +368,37 @@ paths: description: Content too large '500': description: Internal server error + /data/inspect/project-bucket/inventory: + post: + tags: + - internal-data + operationId: internalInspectProjectBucketInventory + description: Recursively list objects under the configured project bucket scope for audit validation. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectBucketInventoryRequest' + responses: + '200': + description: Project bucket inventory + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectBucketInventoryResponse' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '404': + description: Bucket scope not found + '409': + description: Bucket inventory unavailable + '500': + description: Internal server error /data/upload: post: tags: @@ -758,6 +789,68 @@ components: properties: url: type: string + InternalProjectBucketInventoryRequest: + type: object + properties: + organization: + type: string + project: + type: string + path_prefix: + type: string + InternalProjectBucketInventoryResponse: + type: object + properties: + summary: + $ref: '#/components/schemas/InternalProjectBucketInventorySummary' + items: + type: array + items: + $ref: '#/components/schemas/InternalProjectBucketInventoryItem' + InternalProjectBucketInventorySummary: + type: object + properties: + provider: + type: string + bucket: + type: string + prefix: + type: string + object_url: + type: string + exists: + type: boolean + object_count: + type: integer + total_bytes: + type: integer + format: int64 + computed_at: + type: string + mode: + type: string + InternalProjectBucketInventoryItem: + type: object + properties: + object_url: + type: string + provider: + type: string + bucket: + type: string + key: + type: string + path: + type: string + size_bytes: + type: integer + format: int64 + meta_sha256: + type: string + etag: + type: string + last_modified: + type: string InternalUploadBlankRequest: type: object properties: diff --git a/apigen/openapi/metrics.openapi.yaml b/apigen/openapi/metrics.openapi.yaml index 5d1c502e..4661a72c 100644 --- a/apigen/openapi/metrics.openapi.yaml +++ b/apigen/openapi/metrics.openapi.yaml @@ -49,6 +49,35 @@ paths: description: Forbidden '500': description: Internal server error + /index/v1/metrics/files/bulk: + post: + tags: + - metrics + operationId: bulkMetricsFiles + parameters: + - $ref: '#/components/parameters/Organization' + - $ref: '#/components/parameters/Project' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FileUsageBulkRequest' + responses: + '200': + description: File usage metrics for requested object ids + content: + application/json: + schema: + $ref: '#/components/schemas/MetricsListResponse' + '400': + description: Invalid request body + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error /index/v1/metrics/files/{object_id}: get: tags: @@ -321,6 +350,26 @@ components: inactive_file_count: type: integer format: int64 + record_count: + type: integer + format: int64 + record_latest_updated_time: + type: string + format: date-time + nullable: true + FileUsageBulkRequest: + type: object + required: + - object_ids + properties: + object_ids: + type: array + minItems: 1 + items: + type: string + inactive_days: + type: integer + minimum: 0 MetricsListResponse: type: object properties: diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 8ffa2d3d..6b66c97b 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -98,6 +98,45 @@ type InternalMultipartUploadRequest struct { UploadId string `json:"uploadId"` } +// InternalProjectBucketInventoryItem defines model for InternalProjectBucketInventoryItem. +type InternalProjectBucketInventoryItem struct { + Bucket *string `json:"bucket,omitempty"` + Etag *string `json:"etag,omitempty"` + Key *string `json:"key,omitempty"` + LastModified *string `json:"last_modified,omitempty"` + MetaSha256 *string `json:"meta_sha256,omitempty"` + ObjectUrl *string `json:"object_url,omitempty"` + Path *string `json:"path,omitempty"` + Provider *string `json:"provider,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` +} + +// InternalProjectBucketInventoryRequest defines model for InternalProjectBucketInventoryRequest. +type InternalProjectBucketInventoryRequest struct { + Organization *string `json:"organization,omitempty"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project *string `json:"project,omitempty"` +} + +// InternalProjectBucketInventoryResponse defines model for InternalProjectBucketInventoryResponse. +type InternalProjectBucketInventoryResponse struct { + Items *[]InternalProjectBucketInventoryItem `json:"items,omitempty"` + Summary *InternalProjectBucketInventorySummary `json:"summary,omitempty"` +} + +// InternalProjectBucketInventorySummary defines model for InternalProjectBucketInventorySummary. +type InternalProjectBucketInventorySummary struct { + Bucket *string `json:"bucket,omitempty"` + ComputedAt *string `json:"computed_at,omitempty"` + Exists *bool `json:"exists,omitempty"` + Mode *string `json:"mode,omitempty"` + ObjectCount *int `json:"object_count,omitempty"` + ObjectUrl *string `json:"object_url,omitempty"` + Prefix *string `json:"prefix,omitempty"` + Provider *string `json:"provider,omitempty"` + TotalBytes *int64 `json:"total_bytes,omitempty"` +} + // InternalRecord defines model for InternalRecord. type InternalRecord struct { AccessMethods *[]externalRef0.AccessMethod `json:"access_methods,omitempty"` @@ -236,6 +275,9 @@ type InternalListParams struct { Page *int `form:"page,omitempty" json:"page,omitempty"` } +// InternalInspectProjectBucketInventoryJSONRequestBody defines body for InternalInspectProjectBucketInventory for application/json ContentType. +type InternalInspectProjectBucketInventoryJSONRequestBody = InternalProjectBucketInventoryRequest + // InternalMultipartCompleteJSONRequestBody defines body for InternalMultipartComplete for application/json ContentType. type InternalMultipartCompleteJSONRequestBody = InternalMultipartCompleteRequest @@ -346,6 +388,9 @@ type ServerInterface interface { // (GET /data/download/{file_id}/part) InternalDownloadPart(c fiber.Ctx, fileId string, params InternalDownloadPartParams) error + // (POST /data/inspect/project-bucket/inventory) + InternalInspectProjectBucketInventory(c fiber.Ctx) error + // (POST /data/multipart/complete) InternalMultipartComplete(c fiber.Ctx) error @@ -490,6 +535,12 @@ func (siw *ServerInterfaceWrapper) InternalDownloadPart(c fiber.Ctx) error { return siw.Handler.InternalDownloadPart(c, fileId, params) } +// InternalInspectProjectBucketInventory operation middleware +func (siw *ServerInterfaceWrapper) InternalInspectProjectBucketInventory(c fiber.Ctx) error { + + return siw.Handler.InternalInspectProjectBucketInventory(c) +} + // InternalMultipartComplete operation middleware func (siw *ServerInterfaceWrapper) InternalMultipartComplete(c fiber.Ctx) error { @@ -874,6 +925,8 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Get(options.BaseURL+"/data/download/:file_id/part", wrapper.InternalDownloadPart) + router.Post(options.BaseURL+"/data/inspect/project-bucket/inventory", wrapper.InternalInspectProjectBucketInventory) + router.Post(options.BaseURL+"/data/multipart/complete", wrapper.InternalMultipartComplete) router.Post(options.BaseURL+"/data/multipart/init", wrapper.InternalMultipartInit) @@ -1036,6 +1089,71 @@ func (response InternalDownloadPart500Response) VisitInternalDownloadPartRespons return nil } +type InternalInspectProjectBucketInventoryRequestObject struct { + Body *InternalInspectProjectBucketInventoryJSONRequestBody +} + +type InternalInspectProjectBucketInventoryResponseObject interface { + VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error +} + +type InternalInspectProjectBucketInventory200JSONResponse InternalProjectBucketInventoryResponse + +func (response InternalInspectProjectBucketInventory200JSONResponse) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalInspectProjectBucketInventory400Response struct { +} + +func (response InternalInspectProjectBucketInventory400Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalInspectProjectBucketInventory401Response struct { +} + +func (response InternalInspectProjectBucketInventory401Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalInspectProjectBucketInventory403Response struct { +} + +func (response InternalInspectProjectBucketInventory403Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalInspectProjectBucketInventory404Response struct { +} + +func (response InternalInspectProjectBucketInventory404Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(404) + return nil +} + +type InternalInspectProjectBucketInventory409Response struct { +} + +func (response InternalInspectProjectBucketInventory409Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(409) + return nil +} + +type InternalInspectProjectBucketInventory500Response struct { +} + +func (response InternalInspectProjectBucketInventory500Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + type InternalMultipartCompleteRequestObject struct { Body *InternalMultipartCompleteJSONRequestBody } @@ -1897,6 +2015,9 @@ type StrictServerInterface interface { // (GET /data/download/{file_id}/part) InternalDownloadPart(ctx context.Context, request InternalDownloadPartRequestObject) (InternalDownloadPartResponseObject, error) + // (POST /data/inspect/project-bucket/inventory) + InternalInspectProjectBucketInventory(ctx context.Context, request InternalInspectProjectBucketInventoryRequestObject) (InternalInspectProjectBucketInventoryResponseObject, error) + // (POST /data/multipart/complete) InternalMultipartComplete(ctx context.Context, request InternalMultipartCompleteRequestObject) (InternalMultipartCompleteResponseObject, error) @@ -2021,6 +2142,37 @@ func (sh *strictHandler) InternalDownloadPart(ctx fiber.Ctx, fileId string, para return nil } +// InternalInspectProjectBucketInventory operation middleware +func (sh *strictHandler) InternalInspectProjectBucketInventory(ctx fiber.Ctx) error { + var request InternalInspectProjectBucketInventoryRequestObject + + var body InternalInspectProjectBucketInventoryJSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalInspectProjectBucketInventory(ctx.Context(), request.(InternalInspectProjectBucketInventoryRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalInspectProjectBucketInventory") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalInspectProjectBucketInventoryResponseObject); ok { + if err := validResponse.VisitInternalInspectProjectBucketInventoryResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + // InternalMultipartComplete operation middleware func (sh *strictHandler) InternalMultipartComplete(ctx fiber.Ctx) error { var request InternalMultipartCompleteRequestObject diff --git a/apigen/server/metricsapi/metrics.gen.go b/apigen/server/metricsapi/metrics.gen.go index ab20b076..f8db9536 100644 --- a/apigen/server/metricsapi/metrics.gen.go +++ b/apigen/server/metricsapi/metrics.gen.go @@ -54,12 +54,20 @@ type FileUsage struct { UploadCount *int64 `json:"upload_count,omitempty"` } +// FileUsageBulkRequest defines model for FileUsageBulkRequest. +type FileUsageBulkRequest struct { + InactiveDays *int `json:"inactive_days,omitempty"` + ObjectIds []string `json:"object_ids"` +} + // FileUsageSummary defines model for FileUsageSummary. type FileUsageSummary struct { - InactiveFileCount *int64 `json:"inactive_file_count,omitempty"` - TotalDownloads *int64 `json:"total_downloads,omitempty"` - TotalFiles *int64 `json:"total_files,omitempty"` - TotalUploads *int64 `json:"total_uploads,omitempty"` + InactiveFileCount *int64 `json:"inactive_file_count,omitempty"` + RecordCount *int64 `json:"record_count,omitempty"` + RecordLatestUpdatedTime *time.Time `json:"record_latest_updated_time"` + TotalDownloads *int64 `json:"total_downloads,omitempty"` + TotalFiles *int64 `json:"total_files,omitempty"` + TotalUploads *int64 `json:"total_uploads,omitempty"` } // MetricsListResponse defines model for MetricsListResponse. @@ -216,6 +224,15 @@ type ListMetricsFilesParams struct { Project *Project `form:"project,omitempty" json:"project,omitempty"` } +// BulkMetricsFilesParams defines parameters for BulkMetricsFiles. +type BulkMetricsFilesParams struct { + // Organization Organization/program scope filter. + Organization *Organization `form:"organization,omitempty" json:"organization,omitempty"` + + // Project Project scope filter. Requires organization when set. + Project *Project `form:"project,omitempty" json:"project,omitempty"` +} + // GetMetricsFileParams defines parameters for GetMetricsFile. type GetMetricsFileParams struct { // Organization Organization/program scope filter. @@ -293,6 +310,9 @@ type GetTransferSummaryParams struct { AllowStale *AllowStale `form:"allow_stale,omitempty" json:"allow_stale,omitempty"` } +// BulkMetricsFilesJSONRequestBody defines body for BulkMetricsFiles for application/json ContentType. +type BulkMetricsFilesJSONRequestBody = FileUsageBulkRequest + // RecordProviderTransferEventsJSONRequestBody defines body for RecordProviderTransferEvents for application/json ContentType. type RecordProviderTransferEventsJSONRequestBody = ProviderTransferEventsRequest @@ -302,6 +322,9 @@ type ServerInterface interface { // (GET /index/v1/metrics/files) ListMetricsFiles(c fiber.Ctx, params ListMetricsFilesParams) error + // (POST /index/v1/metrics/files/bulk) + BulkMetricsFiles(c fiber.Ctx, params BulkMetricsFilesParams) error + // (GET /index/v1/metrics/files/{object_id}) GetMetricsFile(c fiber.Ctx, objectId string, params GetMetricsFileParams) error // Record provider-observed transfer events @@ -389,6 +412,37 @@ func (siw *ServerInterfaceWrapper) ListMetricsFiles(c fiber.Ctx) error { return siw.Handler.ListMetricsFiles(c, params) } +// BulkMetricsFiles operation middleware +func (siw *ServerInterfaceWrapper) BulkMetricsFiles(c fiber.Ctx) error { + var err error + var params BulkMetricsFilesParams + + // ------------- Optional query parameter "organization" ------------- + if paramValue := c.Query("organization"); paramValue != "" { + + var value Organization + err = runtime.BindStyledParameterWithOptions("form", "organization", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter organization: %w", err).Error()) + } + params.Organization = &value + + } + // ------------- Optional query parameter "project" ------------- + if paramValue := c.Query("project"); paramValue != "" { + + var value Project + err = runtime.BindStyledParameterWithOptions("form", "project", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter project: %w", err).Error()) + } + params.Project = &value + + } + + return siw.Handler.BulkMetricsFiles(c, params) +} + // GetMetricsFile operation middleware func (siw *ServerInterfaceWrapper) GetMetricsFile(c fiber.Ctx) error { var err error @@ -795,6 +849,8 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Get(options.BaseURL+"/index/v1/metrics/files", wrapper.ListMetricsFiles) + router.Post(options.BaseURL+"/index/v1/metrics/files/bulk", wrapper.BulkMetricsFiles) + router.Get(options.BaseURL+"/index/v1/metrics/files/:object_id", wrapper.GetMetricsFile) router.Post(options.BaseURL+"/index/v1/metrics/provider-transfer-events", wrapper.RecordProviderTransferEvents) @@ -856,6 +912,56 @@ func (response ListMetricsFiles500Response) VisitListMetricsFilesResponse(ctx fi return nil } +type BulkMetricsFilesRequestObject struct { + Params BulkMetricsFilesParams + Body *BulkMetricsFilesJSONRequestBody +} + +type BulkMetricsFilesResponseObject interface { + VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error +} + +type BulkMetricsFiles200JSONResponse MetricsListResponse + +func (response BulkMetricsFiles200JSONResponse) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type BulkMetricsFiles400Response struct { +} + +func (response BulkMetricsFiles400Response) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type BulkMetricsFiles401Response struct { +} + +func (response BulkMetricsFiles401Response) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type BulkMetricsFiles403Response struct { +} + +func (response BulkMetricsFiles403Response) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type BulkMetricsFiles500Response struct { +} + +func (response BulkMetricsFiles500Response) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + type GetMetricsFileRequestObject struct { ObjectId string `json:"object_id"` Params GetMetricsFileParams @@ -1117,6 +1223,9 @@ type StrictServerInterface interface { // (GET /index/v1/metrics/files) ListMetricsFiles(ctx context.Context, request ListMetricsFilesRequestObject) (ListMetricsFilesResponseObject, error) + // (POST /index/v1/metrics/files/bulk) + BulkMetricsFiles(ctx context.Context, request BulkMetricsFilesRequestObject) (BulkMetricsFilesResponseObject, error) + // (GET /index/v1/metrics/files/{object_id}) GetMetricsFile(ctx context.Context, request GetMetricsFileRequestObject) (GetMetricsFileResponseObject, error) // Record provider-observed transfer events @@ -1173,6 +1282,39 @@ func (sh *strictHandler) ListMetricsFiles(ctx fiber.Ctx, params ListMetricsFiles return nil } +// BulkMetricsFiles operation middleware +func (sh *strictHandler) BulkMetricsFiles(ctx fiber.Ctx, params BulkMetricsFilesParams) error { + var request BulkMetricsFilesRequestObject + + request.Params = params + + var body BulkMetricsFilesJSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.BulkMetricsFiles(ctx.Context(), request.(BulkMetricsFilesRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "BulkMetricsFiles") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(BulkMetricsFilesResponseObject); ok { + if err := validResponse.VisitBulkMetricsFilesResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + // GetMetricsFile operation middleware func (sh *strictHandler) GetMetricsFile(ctx fiber.Ctx, objectId string, params GetMetricsFileParams) error { var request GetMetricsFileRequestObject diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index 97cd6d95..0d818a82 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -102,34 +102,39 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { requestStart := time.Now() listStart := time.Now() - var ids []string + var objs []models.InternalObject if objectURL != "" { + var ids []string ids, err = om.ListObjectIDsPageByURL(c.Context(), objectURL, filterOrg, filterProject, "read", start, limit, offset) + if err != nil { + return apiutil.HandleError(c, err) + } + bulkStart := time.Now() + objs, err = om.GetBulkObjects(c.Context(), ids, "read") + if err != nil { + return apiutil.HandleError(c, err) + } + bulkDuration := time.Since(bulkStart) + prepareStart := time.Now() + objs, err = om.PrepareScopedObjects(c.Context(), objs, filterOrg, filterProject, "read") + if err != nil { + return apiutil.HandleError(c, err) + } + listDuration := time.Since(listStart) + prepareDuration := time.Since(prepareStart) + log.Printf("INFO: syfon_internal_index_list organization=%s project=%s url_filter=%t start_after=%t limit=%d offset=%d ids=%d records=%d list_ids_ms=%d bulk_objects_ms=%d prepare_scoped_ms=%d duration_ms=%d", filterOrg, filterProject, true, strings.TrimSpace(start) != "", limit, offset, len(ids), len(objs), listDuration.Milliseconds(), bulkDuration.Milliseconds(), prepareDuration.Milliseconds(), time.Since(requestStart).Milliseconds()) } else { - ids, err = om.ListObjectIDsPageByScope(c.Context(), filterOrg, filterProject, "read", start, limit, offset) - } - if err != nil { - return apiutil.HandleError(c, err) - } - listDuration := time.Since(listStart) - - bulkStart := time.Now() - objs, err := om.GetBulkObjects(c.Context(), ids, "read") - if err != nil { - return apiutil.HandleError(c, err) - } - bulkDuration := time.Since(bulkStart) - prepareStart := time.Now() - objs, err = om.PrepareScopedObjects(c.Context(), objs, filterOrg, filterProject, "read") - if err != nil { - return apiutil.HandleError(c, err) + objs, err = om.ListPreparedObjectsPageByScope(c.Context(), filterOrg, filterProject, "read", start, limit, offset) + if err != nil { + return apiutil.HandleError(c, err) + } + listDuration := time.Since(listStart) + log.Printf("INFO: syfon_internal_index_list organization=%s project=%s url_filter=%t start_after=%t limit=%d offset=%d records=%d list_prepared_ms=%d duration_ms=%d", filterOrg, filterProject, false, strings.TrimSpace(start) != "", limit, offset, len(objs), listDuration.Milliseconds(), time.Since(requestStart).Milliseconds()) } - prepareDuration := time.Since(prepareStart) records := make([]internalapi.InternalRecord, 0, len(objs)) for _, obj := range objs { records = append(records, core.InternalObjectToInternalRecord(obj)) } - log.Printf("INFO: syfon_internal_index_list organization=%s project=%s url_filter=%t start_after=%t limit=%d offset=%d ids=%d records=%d list_ids_ms=%d bulk_objects_ms=%d prepare_scoped_ms=%d duration_ms=%d", filterOrg, filterProject, objectURL != "", strings.TrimSpace(start) != "", limit, offset, len(ids), len(records), listDuration.Milliseconds(), bulkDuration.Milliseconds(), prepareDuration.Milliseconds(), time.Since(requestStart).Milliseconds()) return c.JSON(internalapi.ListRecordsResponse{Records: &records}) } } diff --git a/internal/api/internaldrs/index_test.go b/internal/api/internaldrs/index_test.go index e827550b..d465f570 100644 --- a/internal/api/internaldrs/index_test.go +++ b/internal/api/internaldrs/index_test.go @@ -202,6 +202,104 @@ func TestHandleInternalList_CanonicalizesProjectChecksumDuplicates(t *testing.T) } } +func TestHandleInternalList_FillsLimitAfterCanonicalizingDuplicates(t *testing.T) { + database := db.NewInMemoryDB() + om := core.NewObjectManager(database, &testutils.MockUrlManager{}) + now := time.Now().UTC() + later := now.Add(time.Minute) + duplicateSHA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + uniqueSHA := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + + for _, obj := range []models.InternalObject{ + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-1", + Name: stringPtr("older.tsv"), + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: duplicateSHA}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/old/" + duplicateSHA}, + }}, + }, + }, + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-2", + Name: stringPtr("newer.tsv"), + CreatedTime: later, + UpdatedTime: &later, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: duplicateSHA}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/new/" + duplicateSHA}, + }}, + }, + }, + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-3", + Name: stringPtr("unique.tsv"), + CreatedTime: later, + UpdatedTime: &later, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: uniqueSHA}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/unique/" + uniqueSHA}, + }}, + }, + }, + } { + if err := om.RegisterObjects(context.Background(), []models.InternalObject{obj}); err != nil { + t.Fatalf("RegisterObjects failed: %v", err) + } + } + + req := httptest.NewRequest(http.MethodGet, "/index?organization=org&project=p1&limit=2", nil) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var payload struct { + Records []internalapi.InternalRecord `json:"records"` + } + if err := json.NewDecoder(rr.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload.Records) != 2 { + t.Fatalf("expected filled canonical page of 2 records, got %d: %+v", len(payload.Records), payload.Records) + } + if payload.Records[0].Did != "did-1" || payload.Records[1].Did != "did-3" { + t.Fatalf("unexpected canonical page ids: %+v", []string{payload.Records[0].Did, payload.Records[1].Did}) + } + + req = httptest.NewRequest(http.MethodGet, "/index?organization=org&project=p1&limit=1&start=did-1", nil) + rr = doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + payload.Records = nil + if err := json.NewDecoder(rr.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload.Records) != 1 || payload.Records[0].Did != "did-3" { + t.Fatalf("expected pagination to skip duplicate sibling and return did-3, got %+v", payload.Records) + } +} + func TestHandleInternalList_MergesSiblingAccessMethodsFromLegacyDuplicateRows(t *testing.T) { database := db.NewInMemoryDB() om := core.NewObjectManager(database, &testutils.MockUrlManager{}) diff --git a/internal/api/internaldrs/inspect.go b/internal/api/internaldrs/inspect.go index 60c69ba0..62467dbe 100644 --- a/internal/api/internaldrs/inspect.go +++ b/internal/api/internaldrs/inspect.go @@ -291,26 +291,7 @@ func handleInternalInspectProjectBucketFiber(om *core.ObjectManager) fiber.Handl log.Printf("INFO: syfon_project_bucket_handler organization=%s project=%s mode=%s path_prefix=%q include_head=%t duration_ms=%d error=%q", req.Organization, req.Project, req.Mode, req.PathPrefix, req.IncludeHead, time.Since(started).Milliseconds(), err.Error()) return handleInspectStorageError(c, err) } - out := internalInspectProjectBucketResponse{ - Summary: projectBucketSummaryFromCore(result.Summary), - Items: make([]internalInspectProjectBucketItem, 0, len(result.Items)), - } - for _, item := range result.Items { - row := internalInspectProjectBucketItem{ - ObjectURL: item.ObjectURL, - Provider: item.Provider, - Bucket: item.Bucket, - Key: item.Key, - Path: item.Path, - SizeBytes: item.SizeBytes, - MetaSHA256: item.MetaSHA256, - ETag: item.ETag, - } - if !item.LastModTime.IsZero() { - row.LastModTime = item.LastModTime.Format(time.RFC3339) - } - out.Items = append(out.Items, row) - } + out := projectBucketInventoryResponseFromCore(result) exists := false objectCount := 0 totalBytes := int64(0) @@ -326,6 +307,40 @@ func handleInternalInspectProjectBucketFiber(om *core.ObjectManager) fiber.Handl } } +func handleInternalInspectProjectBucketInventoryFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + started := time.Now() + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectProjectBucketRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + result, err := om.InspectProjectStorage(c.Context(), strings.TrimSpace(req.Organization), strings.TrimSpace(req.Project), core.ProjectStorageInspectOptions{ + Mode: core.ProjectStorageInspectItems, + PathPrefix: strings.TrimSpace(req.PathPrefix), + }) + if err != nil { + log.Printf("INFO: syfon_project_bucket_inventory_handler organization=%s project=%s path_prefix=%q duration_ms=%d error=%q", req.Organization, req.Project, req.PathPrefix, time.Since(started).Milliseconds(), err.Error()) + return handleInspectStorageError(c, err) + } + out := projectBucketInventoryResponseFromCore(result) + objectCount := 0 + totalBytes := int64(0) + bucket := "" + prefix := "" + if out.Summary != nil { + objectCount = out.Summary.ObjectCount + totalBytes = out.Summary.TotalBytes + bucket = out.Summary.Bucket + prefix = out.Summary.Prefix + } + log.Printf("INFO: syfon_project_bucket_inventory_handler organization=%s project=%s path_prefix=%q bucket=%s prefix=%q object_count=%d returned_items=%d total_bytes=%d duration_ms=%d", req.Organization, req.Project, req.PathPrefix, bucket, prefix, objectCount, len(out.Items), totalBytes, time.Since(started).Milliseconds()) + return c.JSON(out) + } +} + func handleInternalInspectProjectRecordsFiber(om *core.ObjectManager) fiber.Handler { return func(c fiber.Ctx) error { if apimiddleware.MissingGen3AuthHeader(c.Context()) { @@ -527,6 +542,33 @@ func projectBucketSummaryFromCore(summary core.ProjectStorageSummary) *internalI return out } +func projectBucketInventoryResponseFromCore(result *core.ProjectStorageInspectResult) internalInspectProjectBucketResponse { + if result == nil { + return internalInspectProjectBucketResponse{Items: []internalInspectProjectBucketItem{}} + } + out := internalInspectProjectBucketResponse{ + Summary: projectBucketSummaryFromCore(result.Summary), + Items: make([]internalInspectProjectBucketItem, 0, len(result.Items)), + } + for _, item := range result.Items { + row := internalInspectProjectBucketItem{ + ObjectURL: item.ObjectURL, + Provider: item.Provider, + Bucket: item.Bucket, + Key: item.Key, + Path: item.Path, + SizeBytes: item.SizeBytes, + MetaSHA256: item.MetaSHA256, + ETag: item.ETag, + } + if !item.LastModTime.IsZero() { + row.LastModTime = item.LastModTime.Format(time.RFC3339) + } + out.Items = append(out.Items, row) + } + return out +} + func projectRecordAuditItemFromObject(obj models.InternalObject, organization, project string) (internalInspectProjectRecordItem, bool) { checksum := primarySHA256Checksum(obj.Checksums) if checksum == "" { diff --git a/internal/api/internaldrs/inspect_routes_test.go b/internal/api/internaldrs/inspect_routes_test.go index 9c8ffaca..0ea9be93 100644 --- a/internal/api/internaldrs/inspect_routes_test.go +++ b/internal/api/internaldrs/inspect_routes_test.go @@ -338,6 +338,65 @@ func TestHandleInternalInspectProjectBucketModesUsePrefixList(t *testing.T) { } } +func TestHandleInternalInspectProjectBucketInventoryListsProjectScope(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|": {Organization: "syfon", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "program-root"}, + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + inspectCalls := 0 + om.SetS3ObjectInspector(func(ctx context.Context, cred models.S3Credential, bucket string, key string) (*core.StorageObjectMetadata, error) { + inspectCalls++ + return nil, nil + }) + var listCalls []core.StoragePrefixListOptions + var listedPrefixes []string + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + if bucket != "bucket-a" { + t.Fatalf("unexpected list target bucket=%q prefix=%q", bucket, prefix) + } + listedPrefixes = append(listedPrefixes, prefix) + listCalls = append(listCalls, options) + return []core.StorageBucketObject{ + {Provider: "s3", Bucket: bucket, Key: prefix + "/a.bin", Path: "CONFIG/a.bin", SizeBytes: 10}, + {Provider: "s3", Bucket: bucket, Key: prefix + "/nested/b.bin", Path: "CONFIG/nested/b.bin", SizeBytes: 15}, + }, nil + }) + + body, _ := json.Marshal(internalInspectProjectBucketRequest{Organization: "syfon", Project: "e2e", PathPrefix: "CONFIG"}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-bucket/inventory", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectProjectBucketResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.Summary == nil || resp.Summary.Mode != "items" || resp.Summary.Prefix != "program-root/project-root/CONFIG" { + t.Fatalf("unexpected inventory summary: %+v", resp.Summary) + } + if resp.Summary.ObjectCount != 2 || resp.Summary.TotalBytes != 25 || len(resp.Items) != 2 { + t.Fatalf("unexpected inventory response summary=%+v items=%+v", resp.Summary, resp.Items) + } + if len(listCalls) != 1 || listCalls[0].IncludeHead || listCalls[0].MaxKeys != 0 || listCalls[0].ExactPrefix { + t.Fatalf("expected one recursive LIST without HEAD, got %+v", listCalls) + } + if len(listedPrefixes) != 1 || listedPrefixes[0] != "program-root/project-root/CONFIG" { + t.Fatalf("expected path_prefix to scope recursive inventory, got %+v", listedPrefixes) + } + if inspectCalls != 0 { + t.Fatalf("expected inventory route not to HEAD objects, got %d inspector calls", inspectCalls) + } +} + func TestHandleInternalInspectObjectBulkListValidatesExactKeyWithoutHead(t *testing.T) { db := &testutils.MockDatabase{ Credentials: map[string]models.S3Credential{ diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go index 07ccdbbf..75fa6f98 100644 --- a/internal/api/internaldrs/repair_cleanup.go +++ b/internal/api/internaldrs/repair_cleanup.go @@ -105,15 +105,7 @@ type scopeRepairIndexAdapter struct { } func (a scopeRepairIndexAdapter) List(ctx context.Context, opts repair.ListRecordsOptions) (clientinternalapi.ListRecordsResponse, error) { - ids, err := a.om.ListObjectIDsPageByScope(ctx, strings.TrimSpace(opts.Organization), strings.TrimSpace(opts.Project), "read", strings.TrimSpace(opts.Start), opts.Limit, 0) - if err != nil { - return clientinternalapi.ListRecordsResponse{}, err - } - objects, err := a.om.GetBulkObjects(ctx, ids, "read") - if err != nil { - return clientinternalapi.ListRecordsResponse{}, err - } - objects, err = a.om.PrepareScopedObjects(ctx, objects, strings.TrimSpace(opts.Organization), strings.TrimSpace(opts.Project), "read") + objects, err := a.om.ListPreparedObjectsPageByScope(ctx, strings.TrimSpace(opts.Organization), strings.TrimSpace(opts.Project), "read", strings.TrimSpace(opts.Start), opts.Limit, 0) if err != nil { return clientinternalapi.ListRecordsResponse{}, err } diff --git a/internal/api/internaldrs/transfer.go b/internal/api/internaldrs/transfer.go index 50b6b779..f2a65302 100644 --- a/internal/api/internaldrs/transfer.go +++ b/internal/api/internaldrs/transfer.go @@ -35,6 +35,7 @@ func registerInternalTransferRoutes(router fiber.Router, om *core.ObjectManager) router.Post(common.RouteInternalInspectObjectBulk, handleInternalInspectObjectBulkFiber(om)) router.Post(common.RouteInternalInspectObjectBulkList, handleInternalInspectObjectBulkListFiber(om)) router.Post(common.RouteInternalInspectProjectBucket, handleInternalInspectProjectBucketFiber(om)) + router.Post(common.RouteInternalInspectProjectBucketInventory, handleInternalInspectProjectBucketInventoryFiber(om)) router.Post(common.RouteInternalInspectProjectRecords, handleInternalInspectProjectRecordsFiber(om)) router.Post(common.RouteInternalInspectProjectScopes, handleInternalInspectProjectScopesFiber(om)) router.Post(common.RouteInternalDeleteProjectBucketObjects, handleInternalDeleteProjectBucketObjectsFiber(om)) diff --git a/internal/api/metrics/file_usage.go b/internal/api/metrics/file_usage.go index ae48b9a1..73e01c25 100644 --- a/internal/api/metrics/file_usage.go +++ b/internal/api/metrics/file_usage.go @@ -3,6 +3,7 @@ package metrics import ( "context" "errors" + "log" "net/http" "sort" "strings" @@ -77,6 +78,62 @@ func (s *MetricsServer) ListMetricsFiles(ctx context.Context, request metricsapi }, nil } +func (s *MetricsServer) BulkMetricsFiles(ctx context.Context, request metricsapi.BulkMetricsFilesRequestObject) (metricsapi.BulkMetricsFilesResponseObject, error) { + started := time.Now() + if request.Body == nil { + return metricsapi.BulkMetricsFiles400Response{}, nil + } + objectIDs := uniqueNonEmptyStrings(request.Body.ObjectIds) + if len(objectIDs) == 0 { + return metricsapi.BulkMetricsFiles400Response{}, nil + } + inactiveSince, err := parseInactiveSince(request.Body.InactiveDays) + if err != nil { + return metricsapi.BulkMetricsFiles400Response{}, nil + } + + access, statusCode, ok := s.checkAuth(ctx) + if !ok { + switch statusCode { + case http.StatusUnauthorized: + return metricsapi.BulkMetricsFiles401Response{}, nil + case http.StatusForbidden: + return metricsapi.BulkMetricsFiles403Response{}, nil + default: + return metricsapi.BulkMetricsFiles400Response{}, nil + } + } + + readableObjectIDs, err := s.readableBulkObjectIDs(ctx, access, objectIDs) + if err != nil { + return metricsapi.BulkMetricsFiles500Response{}, nil + } + data, err := s.database.ListFileUsageByObjectIDs(ctx, readableObjectIDs) + if err != nil { + return metricsapi.BulkMetricsFiles500Response{}, nil + } + items := make([]metricsapi.FileUsage, 0, len(data)) + for _, usage := range data { + if !usageMatchesInactiveFilter(usage, inactiveSince) { + continue + } + items = append(items, toMetricsFileUsage(usage)) + } + + log.Printf( + "INFO: syfon_metrics_files_bulk requested=%d returned=%d scoped=%t aggregate_scopes=%d inactive_days=%t duration_ms=%d", + len(objectIDs), + len(items), + access.isScoped(), + len(access.scopes), + request.Body.InactiveDays != nil, + time.Since(started).Milliseconds(), + ) + return metricsapi.BulkMetricsFiles200JSONResponse{ + Data: &items, + }, nil +} + func (s *MetricsServer) GetMetricsFile(ctx context.Context, request metricsapi.GetMetricsFileRequestObject) (metricsapi.GetMetricsFileResponseObject, error) { objectID := request.ObjectId if objectID == "" { @@ -146,8 +203,18 @@ func (s *MetricsServer) GetMetricsSummary(ctx context.Context, request metricsap if access.isScoped() { if scopedStore, ok := s.database.(db.FileUsageScopedLister); ok { summary, err = scopedStore.GetFileUsageSummaryByScope(ctx, access.organization, access.project, inactiveSince) + if err == nil { + recordSummary, recordErr := scopedStore.GetProjectRecordSummaryByScope(ctx, access.organization, access.project) + if recordErr != nil { + err = recordErr + } else { + summary.RecordCount = recordSummary.RecordCount + summary.RecordLatestUpdatedTime = recordSummary.RecordLatestUpdatedTime + } + } } else { _, summary, err = listScopedFileUsage(ctx, s.database, s.objects, access.organization, access.project, 0, 0, inactiveSince) + summary.RecordCount = summary.TotalFiles } } else if access.hasScopeAggregate() { if scopedStore, ok := s.database.(db.FileUsageScopedLister); ok { @@ -163,13 +230,86 @@ func (s *MetricsServer) GetMetricsSummary(ctx context.Context, request metricsap } return metricsapi.GetMetricsSummary200JSONResponse{ - TotalFiles: &summary.TotalFiles, - TotalUploads: &summary.TotalUploads, - TotalDownloads: &summary.TotalDownloads, - InactiveFileCount: &summary.InactiveFileCount, + TotalFiles: &summary.TotalFiles, + TotalUploads: &summary.TotalUploads, + TotalDownloads: &summary.TotalDownloads, + InactiveFileCount: &summary.InactiveFileCount, + RecordCount: scopedSummaryInt64(access, summary.RecordCount), + RecordLatestUpdatedTime: scopedSummaryTime(access, summary.RecordLatestUpdatedTime), }, nil } +func (s *MetricsServer) readableBulkObjectIDs(ctx context.Context, access metricsAccess, objectIDs []string) ([]string, error) { + if !access.isScoped() && !access.hasScopeAggregate() { + return objectIDs, nil + } + readable := make(map[string]struct{}, len(objectIDs)) + if access.isScoped() { + ids, err := s.objects.ListObjectIDsByScope(ctx, access.organization, access.project, "read") + if err != nil { + return nil, err + } + for _, id := range ids { + readable[strings.TrimSpace(id)] = struct{}{} + } + } else { + for _, scope := range access.scopes { + ids, err := s.objects.ListObjectIDsByScope(ctx, scope.organization, scope.project, "read") + if err != nil { + return nil, err + } + for _, id := range ids { + readable[strings.TrimSpace(id)] = struct{}{} + } + } + } + out := make([]string, 0, len(objectIDs)) + for _, objectID := range objectIDs { + if _, ok := readable[objectID]; ok { + out = append(out, objectID) + } + } + return out, nil +} + +func usageMatchesInactiveFilter(usage models.FileUsage, inactiveSince *time.Time) bool { + if inactiveSince == nil { + return true + } + return usage.LastDownloadTime == nil || usage.LastDownloadTime.Before(*inactiveSince) +} + +func uniqueNonEmptyStrings(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func scopedSummaryInt64(access metricsAccess, value int64) *int64 { + if !access.isScoped() { + return nil + } + return &value +} + +func scopedSummaryTime(access metricsAccess, value *time.Time) *time.Time { + if !access.isScoped() || value == nil { + return nil + } + return value +} + func parseInactiveSince(inactiveDays *int) (*time.Time, error) { if inactiveDays == nil { return nil, nil diff --git a/internal/api/metrics/file_usage_test.go b/internal/api/metrics/file_usage_test.go index ac5f5771..1d412807 100644 --- a/internal/api/metrics/file_usage_test.go +++ b/internal/api/metrics/file_usage_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -125,6 +126,86 @@ func TestMetricsRoutes_GetNotFoundAndValidation(t *testing.T) { } } +func TestMetricsRoutes_BulkFiles(t *testing.T) { + oldDownload := time.Now().UTC().AddDate(0, 0, -40) + recentDownload := time.Now().UTC().AddDate(0, 0, -2) + db := &testutils.MockDatabase{ + Objects: map[string]*drs.DrsObject{ + "obj-a": {Id: "obj-a", Name: common.Ptr("a.txt"), Size: 10}, + "obj-b": {Id: "obj-b", Name: common.Ptr("b.txt"), Size: 20}, + "obj-c": {Id: "obj-c", Name: common.Ptr("c.txt"), Size: 30}, + }, + ObjectAuthz: map[string]map[string][]string{ + "obj-a": {"cbds": {"end_to_end_test"}}, + "obj-b": {"cbds": {"end_to_end_test"}}, + "obj-c": {"other": {"project"}}, + }, + Usage: map[string]models.FileUsage{ + "obj-a": { + ObjectID: "obj-a", + Name: "a.txt", + Size: 10, + DownloadCount: 3, + LastDownloadTime: &oldDownload, + }, + "obj-b": { + ObjectID: "obj-b", + Name: "b.txt", + Size: 20, + DownloadCount: 7, + LastDownloadTime: &recentDownload, + }, + "obj-c": { + ObjectID: "obj-c", + Name: "c.txt", + Size: 30, + DownloadCount: 11, + }, + }, + } + app := fiber.New() + app.Use(func(c fiber.Ctx) error { + if mode := c.Get("X-Test-Auth-Mode"); mode != "" { + var privs map[string]map[string]bool + if privsJSON := c.Get("X-Test-Privileges"); privsJSON != "" { + _ = json.Unmarshal([]byte(privsJSON), &privs) + } + headerRaw := c.Get("X-Test-Auth-Header") + ctx := metricsTestContext(c.Context(), mode, headerRaw != "", headerRaw == "true", privs) + c.SetContext(ctx) + } + return c.Next() + }) + RegisterMetricsRoutes(app, db) + + req := httptest.NewRequest(http.MethodPost, "/index/v1/metrics/files/bulk?organization=cbds&project=end_to_end_test", strings.NewReader(`{"object_ids":["obj-a","obj-b","obj-c","missing","obj-a"],"inactive_days":30}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Test-Auth-Mode", "gen3") + req.Header.Set("X-Test-Auth-Header", "true") + privs, _ := json.Marshal(map[string]map[string]bool{ + "/programs/cbds/projects/end_to_end_test": {"read": true}, + }) + req.Header.Set("X-Test-Privileges", string(privs)) + httpResp, err := app.Test(req) + if err != nil { + t.Fatalf("test request failed: %v", err) + } + body, _ := io.ReadAll(httpResp.Body) + if httpResp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", httpResp.StatusCode, string(body)) + } + var resp metricsapi.MetricsListResponse + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.Data == nil || len(*resp.Data) != 1 { + t.Fatalf("expected only inactive scoped object, got %s", string(body)) + } + if got := *(*resp.Data)[0].ObjectId; got != "obj-a" { + t.Fatalf("expected obj-a, got %q", got) + } +} + func TestMetricsSummaryAuthzAndScope(t *testing.T) { db := &testutils.MockDatabase{ Objects: map[string]*drs.DrsObject{ @@ -187,6 +268,9 @@ func TestMetricsSummaryAuthzAndScope(t *testing.T) { if resp.TotalFiles == nil || resp.TotalUploads == nil || resp.TotalDownloads == nil || *resp.TotalFiles != 1 || *resp.TotalUploads != 2 || *resp.TotalDownloads != 3 { t.Fatalf("unexpected scoped summary: %+v", resp) } + if resp.RecordCount == nil || *resp.RecordCount != 1 { + t.Fatalf("expected exact scoped record count, got %+v", resp.RecordCount) + } }) t.Run("missing auth header returns 401", func(t *testing.T) { @@ -395,4 +479,3 @@ func TestListMultiScopedFileUsage_DeduplicatesAcrossScopes(t *testing.T) { t.Fatalf("expected summary total files > 0, got %+v", summary) } } - diff --git a/internal/common/routes.go b/internal/common/routes.go index d61051e4..98940825 100644 --- a/internal/common/routes.go +++ b/internal/common/routes.go @@ -1,27 +1,28 @@ package common const ( - RouteInternalDownload = "/data/download/{file_id}" - RouteInternalDownloadPart = "/data/download/{file_id}/part" - RouteInternalUpload = "/data/upload" - RouteInternalUploadURL = "/data/upload/{file_id}" - RouteInternalUploadBulk = "/data/upload/bulk" - RouteInternalMultipartInit = "/data/multipart/init" - RouteInternalMultipartUpload = "/data/multipart/upload" - RouteInternalMultipartComplete = "/data/multipart/complete" - RouteInternalInspectObject = "/data/inspect" - RouteInternalInspectObjectBulk = "/data/inspect/bulk" - RouteInternalInspectObjectBulkList = "/data/inspect/bulk-list" - RouteInternalInspectProjectBucket = "/data/inspect/project-bucket" - RouteInternalInspectProjectRecords = "/data/inspect/project-records" - RouteInternalInspectProjectScopes = "/data/inspect/project-scopes" - RouteInternalDeleteProjectBucketObjects = "/data/inspect/project-bucket/delete" - RouteInternalBuckets = "/data/buckets" - RouteInternalBucketDetail = "/data/buckets/{bucket}" - RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" - RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" - RouteInternalRepairScopeAudit = "/data/repair/project-scope/audit" - RouteInternalRepairScopeApply = "/data/repair/project-scope/apply" + RouteInternalDownload = "/data/download/{file_id}" + RouteInternalDownloadPart = "/data/download/{file_id}/part" + RouteInternalUpload = "/data/upload" + RouteInternalUploadURL = "/data/upload/{file_id}" + RouteInternalUploadBulk = "/data/upload/bulk" + RouteInternalMultipartInit = "/data/multipart/init" + RouteInternalMultipartUpload = "/data/multipart/upload" + RouteInternalMultipartComplete = "/data/multipart/complete" + RouteInternalInspectObject = "/data/inspect" + RouteInternalInspectObjectBulk = "/data/inspect/bulk" + RouteInternalInspectObjectBulkList = "/data/inspect/bulk-list" + RouteInternalInspectProjectBucket = "/data/inspect/project-bucket" + RouteInternalInspectProjectBucketInventory = "/data/inspect/project-bucket/inventory" + RouteInternalInspectProjectRecords = "/data/inspect/project-records" + RouteInternalInspectProjectScopes = "/data/inspect/project-scopes" + RouteInternalDeleteProjectBucketObjects = "/data/inspect/project-bucket/delete" + RouteInternalBuckets = "/data/buckets" + RouteInternalBucketDetail = "/data/buckets/{bucket}" + RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" + RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" + RouteInternalRepairScopeAudit = "/data/repair/project-scope/audit" + RouteInternalRepairScopeApply = "/data/repair/project-scope/apply" RouteInternalIndex = "/index" RouteInternalIndexDetail = "/index/{id}" diff --git a/internal/core/object_reads.go b/internal/core/object_reads.go index cb201bae..90bf8d51 100644 --- a/internal/core/object_reads.go +++ b/internal/core/object_reads.go @@ -146,6 +146,83 @@ func (m *ObjectManager) PrepareScopedObjects(ctx context.Context, objects []mode return canonical, nil } +func (m *ObjectManager) ListPreparedObjectsPageByScope(ctx context.Context, organization, project, requiredMethod, startAfter string, limit, offset int) ([]models.InternalObject, error) { + if limit <= 0 { + return []models.InternalObject{}, nil + } + if offset < 0 { + offset = 0 + } + + startAfter = strings.TrimSpace(startAfter) + skip := offset + if startAfter != "" { + skip = 0 + } + target := limit + skip + batchSize := target + if batchSize < 100 { + batchSize = 100 + } + + rawStart := startAfter + collected := make([]models.InternalObject, 0, target) + seen := make(map[string]struct{}, target) + started := time.Now() + rawPages := 0 + rawIDs := 0 + + for len(collected) < target { + ids, err := m.ListObjectIDsPageByScope(ctx, organization, project, requiredMethod, rawStart, batchSize, 0) + if err != nil { + return nil, err + } + if len(ids) == 0 { + break + } + rawPages++ + rawIDs += len(ids) + rawStart = ids[len(ids)-1] + + objects, err := m.GetBulkObjects(ctx, ids, requiredMethod) + if err != nil { + return nil, err + } + prepared, err := m.PrepareScopedObjects(ctx, objects, organization, project, requiredMethod) + if err != nil { + return nil, err + } + for _, obj := range prepared { + if startAfter != "" && obj.Id <= startAfter { + continue + } + if _, ok := seen[obj.Id]; ok { + continue + } + seen[obj.Id] = struct{}{} + collected = append(collected, obj) + if len(collected) >= target { + break + } + } + if len(ids) < batchSize { + break + } + } + + if skip >= len(collected) { + log.Printf("INFO: syfon_list_prepared_objects_page_by_scope organization=%s project=%s start_after=%t limit=%d offset=%d raw_pages=%d raw_ids=%d records=0 duration_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), startAfter != "", limit, offset, rawPages, rawIDs, time.Since(started).Milliseconds()) + return []models.InternalObject{}, nil + } + end := skip + limit + if end > len(collected) { + end = len(collected) + } + out := collected[skip:end] + log.Printf("INFO: syfon_list_prepared_objects_page_by_scope organization=%s project=%s start_after=%t limit=%d offset=%d raw_pages=%d raw_ids=%d records=%d duration_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), startAfter != "", limit, offset, rawPages, rawIDs, len(out), time.Since(started).Milliseconds()) + return out, nil +} + func (m *ObjectManager) ListObjectIDsPageByChecksum(ctx context.Context, checksum, checksumType, organization, project, requiredMethod, startAfter string, limit, offset int) ([]string, error) { if limit <= 0 { return []string{}, nil diff --git a/internal/db/interface.go b/internal/db/interface.go index 36f9e4a1..c03dd652 100644 --- a/internal/db/interface.go +++ b/internal/db/interface.go @@ -60,6 +60,7 @@ type FileUsageScopedLister interface { ListFileUsagePageByResources(ctx context.Context, resources []string, includeUnscoped bool, limit, offset int, inactiveSince *time.Time) ([]models.FileUsage, error) GetFileUsageSummaryByScope(ctx context.Context, organization, project string, inactiveSince *time.Time) (models.FileUsageSummary, error) GetFileUsageSummaryByResources(ctx context.Context, resources []string, includeUnscoped bool, inactiveSince *time.Time) (models.FileUsageSummary, error) + GetProjectRecordSummaryByScope(ctx context.Context, organization, project string) (models.FileUsageSummary, error) } type TransferAttributionScopedStore interface { diff --git a/internal/db/postgres/metrics.go b/internal/db/postgres/metrics.go index 67cd5c1c..1fb75481 100644 --- a/internal/db/postgres/metrics.go +++ b/internal/db/postgres/metrics.go @@ -363,6 +363,27 @@ func (db *PostgresDB) GetFileUsageSummaryByResources(ctx context.Context, resour return db.getScopedFileUsageSummary(ctx, resources, includeUnscoped, inactiveSince) } +func (db *PostgresDB) GetProjectRecordSummaryByScope(ctx context.Context, organization, project string) (models.FileUsageSummary, error) { + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return models.FileUsageSummary{}, err + } + var summary models.FileUsageSummary + var latest sql.NullTime + if err := db.db.QueryRowContext(ctx, ` + SELECT COUNT(DISTINCT o.id), MAX(o.updated_time) + FROM drs_object o + INNER JOIN drs_object_controlled_access ca ON ca.object_id = o.id + WHERE ca.resource = $1`, resource).Scan(&summary.RecordCount, &latest); err != nil { + return models.FileUsageSummary{}, err + } + if latest.Valid { + t := latest.Time.UTC() + summary.RecordLatestUpdatedTime = &t + } + return summary, nil +} + func scanFileUsageRows(rows *sql.Rows, capacity int) ([]models.FileUsage, error) { out := make([]models.FileUsage, 0, capacity) for rows.Next() { @@ -615,8 +636,6 @@ func nullableInt64(v *int64) any { return *v } - - func (db *PostgresDB) backfillAccessGrants(ctx context.Context) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -1028,8 +1047,6 @@ func postgresTransferResourceClause(resources []string, startIndex int) (string, return strings.Join(clauses, " OR "), args } - - func transferAttributionGroupExpr(groupBy string) (string, string) { switch strings.ToLower(strings.TrimSpace(groupBy)) { case "user": diff --git a/internal/db/scripts/normalize_object_names_to_basename_postgres.sql b/internal/db/scripts/normalize_object_names_to_basename_postgres.sql new file mode 100644 index 00000000..de075253 --- /dev/null +++ b/internal/db/scripts/normalize_object_names_to_basename_postgres.sql @@ -0,0 +1,85 @@ +-- Normalize legacy drs_object.name values from full paths to basenames. +-- +-- Usage: +-- psql "$DATABASE_URL" -f internal/db/scripts/normalize_object_names_to_basename_postgres.sql +-- +-- The update only touches non-empty names whose slash/backslash-normalized +-- basename differs from the stored value. Access URLs, aliases, checksums, and +-- controlled_access rows are intentionally untouched. + +BEGIN; + +WITH candidates AS ( + SELECT + id, + name, + regexp_replace(replace(btrim(name), E'\\', '/'), '^.*/', '') AS basename + FROM drs_object + WHERE name IS NOT NULL + AND btrim(name) <> '' +), +changed AS ( + SELECT id, name, basename + FROM candidates + WHERE basename <> '' + AND name <> basename +) +SELECT COUNT(*) AS rows_to_update +FROM changed; + +WITH candidates AS ( + SELECT + id, + name, + regexp_replace(replace(btrim(name), E'\\', '/'), '^.*/', '') AS basename + FROM drs_object + WHERE name IS NOT NULL + AND btrim(name) <> '' +), +changed AS ( + SELECT id, name, basename + FROM candidates + WHERE basename <> '' + AND name <> basename +) +SELECT id, name AS old_name, basename AS new_name +FROM changed +ORDER BY id +LIMIT 20; + +WITH candidates AS ( + SELECT + id, + regexp_replace(replace(btrim(name), E'\\', '/'), '^.*/', '') AS basename + FROM drs_object + WHERE name IS NOT NULL + AND btrim(name) <> '' +), +changed AS ( + SELECT id, basename + FROM candidates + WHERE basename <> '' +) +UPDATE drs_object AS o +SET + name = changed.basename, + updated_time = now() +FROM changed +WHERE o.id = changed.id + AND o.name <> changed.basename; + +WITH candidates AS ( + SELECT + id, + name, + regexp_replace(replace(btrim(name), E'\\', '/'), '^.*/', '') AS basename + FROM drs_object + WHERE name IS NOT NULL + AND btrim(name) <> '' +) +SELECT COUNT(*) AS rows_still_path_like +FROM candidates +WHERE basename <> '' + AND name <> basename; + +COMMIT; diff --git a/internal/db/sqlite/metrics.go b/internal/db/sqlite/metrics.go index 4e7032df..ff4b0af4 100644 --- a/internal/db/sqlite/metrics.go +++ b/internal/db/sqlite/metrics.go @@ -462,6 +462,27 @@ func (db *SqliteDB) GetFileUsageSummaryByResources(ctx context.Context, resource return db.getScopedFileUsageSummary(ctx, resources, includeUnscoped, inactiveSince) } +func (db *SqliteDB) GetProjectRecordSummaryByScope(ctx context.Context, organization, project string) (models.FileUsageSummary, error) { + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return models.FileUsageSummary{}, err + } + var summary models.FileUsageSummary + var latest any + if err := db.db.QueryRowContext(ctx, ` + SELECT COUNT(DISTINCT o.id), MAX(o.updated_time) + FROM drs_object o + INNER JOIN drs_object_controlled_access ca ON ca.object_id = o.id + WHERE ca.resource = ?`, resource).Scan(&summary.RecordCount, &latest); err != nil { + return models.FileUsageSummary{}, err + } + if parsed, ok := parseSQLiteTransferTime(latest); ok { + t := parsed.UTC() + summary.RecordLatestUpdatedTime = &t + } + return summary, nil +} + func scanFileUsageRows(rows *sql.Rows, capacity int) ([]models.FileUsage, error) { out := make([]models.FileUsage, 0, capacity) for rows.Next() { @@ -677,8 +698,6 @@ func nullableInt64(v *int64) any { return *v } - - func (db *SqliteDB) backfillAccessGrants(ctx context.Context) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -1068,8 +1087,6 @@ func sqliteTransferResourceClause(resources []string) (string, []any) { return strings.Join(clauses, " OR "), args } - - func transferAttributionGroupExpr(groupBy string) (string, string) { switch strings.ToLower(strings.TrimSpace(groupBy)) { case "user": diff --git a/internal/db/sqlite/sqlite_test.go b/internal/db/sqlite/sqlite_test.go index 6c40fb87..1e594cc5 100644 --- a/internal/db/sqlite/sqlite_test.go +++ b/internal/db/sqlite/sqlite_test.go @@ -1528,6 +1528,17 @@ func TestSqliteDB_ScopedFileUsageQueries(t *testing.T) { if summary.TotalFiles != 2 || summary.TotalUploads != 1 || summary.TotalDownloads != 1 || summary.InactiveFileCount != 0 { t.Fatalf("unexpected scoped summary: %+v", summary) } + + recordSummary, err := db.GetProjectRecordSummaryByScope(ctx, "org", "p1") + if err != nil { + t.Fatalf("GetProjectRecordSummaryByScope failed: %v", err) + } + if recordSummary.RecordCount != 2 { + t.Fatalf("unexpected scoped record count: %+v", recordSummary) + } + if recordSummary.RecordLatestUpdatedTime == nil || !recordSummary.RecordLatestUpdatedTime.Equal(now) { + t.Fatalf("unexpected scoped record latest updated time: %+v", recordSummary.RecordLatestUpdatedTime) + } } func TestSqliteDB_TransferAttributionByResources(t *testing.T) { diff --git a/internal/models/storage.go b/internal/models/storage.go index fd193123..e1be2f64 100644 --- a/internal/models/storage.go +++ b/internal/models/storage.go @@ -48,10 +48,12 @@ type FileUsage struct { // FileUsageSummary aggregates transfer activity for a set of objects. type FileUsageSummary struct { - TotalFiles int64 - TotalUploads int64 - TotalDownloads int64 - InactiveFileCount int64 + TotalFiles int64 + TotalUploads int64 + TotalDownloads int64 + InactiveFileCount int64 + RecordCount int64 + RecordLatestUpdatedTime *time.Time } // BucketVisibilityRow is the minimum storage projection needed to build bucket From 660a8fe41e9bea11a31b08e5d04c2d6b2d0d068b Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 7 Jul 2026 13:58:17 -0700 Subject: [PATCH 17/24] fix tests --- client/services/services_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/services/services_test.go b/client/services/services_test.go index 24c5492c..723fe6ef 100644 --- a/client/services/services_test.go +++ b/client/services/services_test.go @@ -146,6 +146,14 @@ func (f *fakeMetricsClient) ListMetricsFilesWithResponse(ctx context.Context, pa return f.filesResp, f.filesErr } +func (f *fakeMetricsClient) BulkMetricsFilesWithBodyWithResponse(ctx context.Context, params *metricsapi.BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.BulkMetricsFilesResponse, error) { + return &metricsapi.BulkMetricsFilesResponse{HTTPResponse: &http.Response{StatusCode: http.StatusNotImplemented}}, nil +} + +func (f *fakeMetricsClient) BulkMetricsFilesWithResponse(ctx context.Context, params *metricsapi.BulkMetricsFilesParams, body metricsapi.BulkMetricsFilesJSONRequestBody, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.BulkMetricsFilesResponse, error) { + return &metricsapi.BulkMetricsFilesResponse{HTTPResponse: &http.Response{StatusCode: http.StatusNotImplemented}}, nil +} + func (f *fakeMetricsClient) GetMetricsFileWithResponse(ctx context.Context, objectId string, params *metricsapi.GetMetricsFileParams, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.GetMetricsFileResponse, error) { f.fileObjectID = objectId return f.fileResp, f.fileErr From 5bf339e70fa1ee4ed41321aa9d0ce706aad85475 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 7 Jul 2026 15:10:01 -0700 Subject: [PATCH 18/24] updtae routes --- .../api/internaldrs/bucket_routes_test.go | 31 +++++++++++++ internal/api/internaldrs/buckets.go | 14 +++--- internal/core/service_test.go | 46 ++++++++++++++++++- internal/core/storage_signing.go | 3 ++ 4 files changed, 86 insertions(+), 8 deletions(-) diff --git a/internal/api/internaldrs/bucket_routes_test.go b/internal/api/internaldrs/bucket_routes_test.go index fe14d42e..fea0a263 100644 --- a/internal/api/internaldrs/bucket_routes_test.go +++ b/internal/api/internaldrs/bucket_routes_test.go @@ -376,6 +376,37 @@ func TestHandleInternalListBucketScopes_FiltersUnauthorizedScopesOnSharedBucket( } } +func TestHandleInternalListBucketScopes_RendersRootScopeAsBucketURL(t *testing.T) { + mockDB := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "gdcdata": {CredentialID: "gdcdata", Bucket: "gdcdata", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "gdc|": {Organization: "gdc", ProjectID: "", CredentialID: "gdcdata", Bucket: "gdcdata", PathPrefix: ""}, + }, + } + req := httptest.NewRequest(http.MethodGet, "/data/buckets/gdcdata/scopes", nil) + req = req.WithContext(dataTestAuthContext(req.Context(), "gen3", true, map[string]map[string]bool{ + "/programs/gdc": {"read": true}, + })) + + rr := doInternalDRSTestRequest(req, core.NewObjectManager(mockDB, &testutils.MockUrlManager{})) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + + var resp []bucketapi.BucketScopeResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp) != 1 { + t.Fatalf("expected 1 scope, got %d: %+v", len(resp), resp) + } + if resp[0].Path == nil || *resp[0].Path != "s3://gdcdata" { + t.Fatalf("expected root scope path s3://gdcdata, got %+v", resp[0].Path) + } +} + func TestHandleInternalListBucketScopes_RequiresGen3Auth(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/data/buckets/bucket-a/scopes", nil) req = req.WithContext(dataTestAuthContext(req.Context(), "gen3", false, nil)) diff --git a/internal/api/internaldrs/buckets.go b/internal/api/internaldrs/buckets.go index 64d3eac0..01b4d08f 100644 --- a/internal/api/internaldrs/buckets.go +++ b/internal/api/internaldrs/buckets.go @@ -361,13 +361,13 @@ func handleInternalListBucketScopesFiber(c fiber.Ctx, om *core.ObjectManager) er if !bucketScopeAllowed(c.Context(), scope, "read") { continue } - path := "" - if scope.PathPrefix != "" { - scheme := "s3" - if cred, err := om.GetS3Credential(c.Context(), scope.CredentialID); err == nil && cred != nil { - scheme = common.ProviderToScheme(cred.Provider) - } - path = fmt.Sprintf("%s://%s/%s", scheme, scope.Bucket, scope.PathPrefix) + scheme := "s3" + if cred, err := om.GetS3Credential(c.Context(), scope.CredentialID); err == nil && cred != nil { + scheme = common.ProviderToScheme(cred.Provider) + } + path := fmt.Sprintf("%s://%s", scheme, scope.Bucket) + if strings.TrimSpace(scope.PathPrefix) != "" { + path = fmt.Sprintf("%s/%s", path, strings.Trim(strings.TrimSpace(scope.PathPrefix), "/")) } result = append(result, bucketapi.BucketScopeResponse{ Organization: scope.Organization, diff --git a/internal/core/service_test.go b/internal/core/service_test.go index 18e9b9f6..fb6d7a87 100644 --- a/internal/core/service_test.go +++ b/internal/core/service_test.go @@ -964,6 +964,51 @@ func TestObjectManagerDeleteResolveAndSignDelegation(t *testing.T) { }) }) + t.Run("root bucket scope preserves existing physical storage key", func(t *testing.T) { + db := &coreTestDB{ + MockDatabase: &testutils.MockDatabase{ + BucketScopes: map[string]models.BucketScope{ + "gdc_mirror|gdc_mirror": { + Organization: "gdc_mirror", + ProjectID: "gdc_mirror", + Bucket: "gdcdata", + PathPrefix: "", + }, + }, + }, + } + om := NewObjectManager(db, &capturingURLManager{}) + obj := &models.InternalObject{ + DrsObject: drs.DrsObject{ + Id: "00664eeb-830c-5fe4-b48c-054cd9c8e02f", + Checksums: []drs.Checksum{{ + Type: "sha256", + Checksum: "239f8402efd37b62bfb892aa4becb0692b3ca5f58015083d8567e8d7fbdd1843", + }}, + ControlledAccess: &[]string{"/organization/gdc_mirror/project/gdc_mirror"}, + }, + } + sourceURL := "s3://gdcdata/00664eeb-830c-5fe4-b48c-054cd9c8e02f/239f8402efd37b62bfb892aa4becb0692b3ca5f58015083d8567e8d7fbdd1843" + + target, err := om.ResolveCanonicalStorageTarget(context.Background(), CanonicalStorageTargetRequest{ + Object: obj, + AccessURL: sourceURL, + PreferChecksum: true, + }) + if err != nil { + t.Fatalf("ResolveCanonicalStorageTarget failed: %v", err) + } + if target.URL != sourceURL { + t.Fatalf("expected root scoped physical URL to remain %q, got %q", sourceURL, target.URL) + } + if target.Bucket != "gdcdata" { + t.Fatalf("expected bucket gdcdata, got %q", target.Bucket) + } + if target.Key != "00664eeb-830c-5fe4-b48c-054cd9c8e02f/239f8402efd37b62bfb892aa4becb0692b3ca5f58015083d8567e8d7fbdd1843" { + t.Fatalf("unexpected target key: %q", target.Key) + } + }) + t.Run("create bucket scope updates signing cache", func(t *testing.T) { mockDB := &testutils.MockDatabase{} db := &coreTestDB{MockDatabase: mockDB} @@ -991,5 +1036,4 @@ func TestObjectManagerDeleteResolveAndSignDelegation(t *testing.T) { } }) - } diff --git a/internal/core/storage_signing.go b/internal/core/storage_signing.go index ac085729..6be8acd0 100644 --- a/internal/core/storage_signing.go +++ b/internal/core/storage_signing.go @@ -69,6 +69,9 @@ func (m *ObjectManager) ResolveCanonicalStorageTarget(ctx context.Context, req C return CanonicalStorageTarget{}, fmt.Errorf("unable to resolve scoped storage bucket for object %s", obj.Id) } targetKey := m.canonicalObjectKey(obj, req.Key, existingKey, req.PreferChecksum) + if existingOK && strings.EqualFold(strings.TrimSpace(existingBucket), targetBucket) && len(normalizedScopePrefixes(scopes)) == 0 && strings.TrimSpace(existingKey) != "" { + targetKey = existingKey + } targetKey = normalizeScopedStorageKey(targetKey, scopes) if strings.TrimSpace(targetKey) == "" { return CanonicalStorageTarget{}, fmt.Errorf("unable to resolve scoped storage key for object %s", obj.Id) From cd948fc2ca8fb5bf7f59e87ffd2ff31d50232465 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 7 Jul 2026 16:30:20 -0700 Subject: [PATCH 19/24] fix bugs --- apigen/client/internalapi/internal.gen.go | 299 ++++++++++++++++++ apigen/openapi/internal.openapi.yaml | 86 +++++ apigen/server/internalapi/internal.gen.go | 241 ++++++++++++++ internal/api/internaldrs/inspect.go | 10 +- .../api/internaldrs/inspect_routes_test.go | 51 ++- internal/api/internaldrs/transfer.go | 1 + 6 files changed, 672 insertions(+), 16 deletions(-) diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index 4143f1ca..10f63bc1 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -139,6 +139,25 @@ type InternalProjectBucketInventorySummary struct { TotalBytes *int64 `json:"total_bytes,omitempty"` } +// InternalProjectScopeItem defines model for InternalProjectScopeItem. +type InternalProjectScopeItem struct { + Bucket *string `json:"bucket,omitempty"` + Organization *string `json:"organization,omitempty"` + Path *string `json:"path,omitempty"` + ProjectId *string `json:"project_id,omitempty"` +} + +// InternalProjectScopesRequest defines model for InternalProjectScopesRequest. +type InternalProjectScopesRequest struct { + Organization string `json:"organization"` + Project string `json:"project"` +} + +// InternalProjectScopesResponse defines model for InternalProjectScopesResponse. +type InternalProjectScopesResponse struct { + Items *[]InternalProjectScopeItem `json:"items,omitempty"` +} + // InternalRecord defines model for InternalRecord. type InternalRecord struct { AccessMethods *[]externalRef0.AccessMethod `json:"access_methods,omitempty"` @@ -247,6 +266,12 @@ type InternalDownloadPartParams struct { End int64 `form:"end" json:"end"` } +// InternalInspectProjectScopesParams defines parameters for InternalInspectProjectScopes. +type InternalInspectProjectScopesParams struct { + Organization string `form:"organization" json:"organization"` + Project string `form:"project" json:"project"` +} + // InternalUploadURLParams defines parameters for InternalUploadURL. type InternalUploadURLParams struct { Organization *string `form:"organization,omitempty" json:"organization,omitempty"` @@ -280,6 +305,9 @@ type InternalListParams struct { // InternalInspectProjectBucketInventoryJSONRequestBody defines body for InternalInspectProjectBucketInventory for application/json ContentType. type InternalInspectProjectBucketInventoryJSONRequestBody = InternalProjectBucketInventoryRequest +// InternalInspectProjectScopesPostJSONRequestBody defines body for InternalInspectProjectScopesPost for application/json ContentType. +type InternalInspectProjectScopesPostJSONRequestBody = InternalProjectScopesRequest + // InternalMultipartCompleteJSONRequestBody defines body for InternalMultipartComplete for application/json ContentType. type InternalMultipartCompleteJSONRequestBody = InternalMultipartCompleteRequest @@ -465,6 +493,14 @@ type ClientInterface interface { InternalInspectProjectBucketInventory(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalInspectProjectScopes request + InternalInspectProjectScopes(ctx context.Context, params *InternalInspectProjectScopesParams, reqEditors ...RequestEditorFn) (*http.Response, error) + + // InternalInspectProjectScopesPostWithBody request with any body + InternalInspectProjectScopesPostWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + InternalInspectProjectScopesPost(ctx context.Context, body InternalInspectProjectScopesPostJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalMultipartCompleteWithBody request with any body InternalMultipartCompleteWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -594,6 +630,42 @@ func (c *Client) InternalInspectProjectBucketInventory(ctx context.Context, body return c.Client.Do(req) } +func (c *Client) InternalInspectProjectScopes(ctx context.Context, params *InternalInspectProjectScopesParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectScopesRequest(c.Server, params) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalInspectProjectScopesPostWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectScopesPostRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalInspectProjectScopesPost(ctx context.Context, body InternalInspectProjectScopesPostJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectScopesPostRequest(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) InternalMultipartCompleteWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewInternalMultipartCompleteRequestWithBody(c.Server, contentType, body) if err != nil { @@ -1142,6 +1214,103 @@ func NewInternalInspectProjectBucketInventoryRequestWithBody(server string, cont return req, nil } +// NewInternalInspectProjectScopesRequest generates requests for InternalInspectProjectScopes +func NewInternalInspectProjectScopesRequest(server string, params *InternalInspectProjectScopesParams) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/data/inspect/project-scopes") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "organization", runtime.ParamLocationQuery, params.Organization); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "project", runtime.ParamLocationQuery, params.Project); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewInternalInspectProjectScopesPostRequest calls the generic InternalInspectProjectScopesPost builder with application/json body +func NewInternalInspectProjectScopesPostRequest(server string, body InternalInspectProjectScopesPostJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewInternalInspectProjectScopesPostRequestWithBody(server, "application/json", bodyReader) +} + +// NewInternalInspectProjectScopesPostRequestWithBody generates requests for InternalInspectProjectScopesPost with any type of body +func NewInternalInspectProjectScopesPostRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/data/inspect/project-scopes") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewInternalMultipartCompleteRequest calls the generic InternalMultipartComplete builder with application/json body func NewInternalMultipartCompleteRequest(server string, body InternalMultipartCompleteJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -2192,6 +2361,14 @@ type ClientWithResponsesInterface interface { InternalInspectProjectBucketInventoryWithResponse(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalInspectProjectBucketInventoryResponse, error) + // InternalInspectProjectScopesWithResponse request + InternalInspectProjectScopesWithResponse(ctx context.Context, params *InternalInspectProjectScopesParams, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesResponse, error) + + // InternalInspectProjectScopesPostWithBodyWithResponse request with any body + InternalInspectProjectScopesPostWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesPostResponse, error) + + InternalInspectProjectScopesPostWithResponse(ctx context.Context, body InternalInspectProjectScopesPostJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesPostResponse, error) + // InternalMultipartCompleteWithBodyWithResponse request with any body InternalMultipartCompleteWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalMultipartCompleteResponse, error) @@ -2339,6 +2516,50 @@ func (r InternalInspectProjectBucketInventoryResponse) StatusCode() int { return 0 } +type InternalInspectProjectScopesResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *InternalProjectScopesResponse +} + +// Status returns HTTPResponse.Status +func (r InternalInspectProjectScopesResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalInspectProjectScopesResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type InternalInspectProjectScopesPostResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *InternalProjectScopesResponse +} + +// Status returns HTTPResponse.Status +func (r InternalInspectProjectScopesPostResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalInspectProjectScopesPostResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type InternalMultipartCompleteResponse struct { Body []byte HTTPResponse *http.Response @@ -2769,6 +2990,32 @@ func (c *ClientWithResponses) InternalInspectProjectBucketInventoryWithResponse( return ParseInternalInspectProjectBucketInventoryResponse(rsp) } +// InternalInspectProjectScopesWithResponse request returning *InternalInspectProjectScopesResponse +func (c *ClientWithResponses) InternalInspectProjectScopesWithResponse(ctx context.Context, params *InternalInspectProjectScopesParams, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesResponse, error) { + rsp, err := c.InternalInspectProjectScopes(ctx, params, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectScopesResponse(rsp) +} + +// InternalInspectProjectScopesPostWithBodyWithResponse request with arbitrary body returning *InternalInspectProjectScopesPostResponse +func (c *ClientWithResponses) InternalInspectProjectScopesPostWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesPostResponse, error) { + rsp, err := c.InternalInspectProjectScopesPostWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectScopesPostResponse(rsp) +} + +func (c *ClientWithResponses) InternalInspectProjectScopesPostWithResponse(ctx context.Context, body InternalInspectProjectScopesPostJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesPostResponse, error) { + rsp, err := c.InternalInspectProjectScopesPost(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectScopesPostResponse(rsp) +} + // InternalMultipartCompleteWithBodyWithResponse request with arbitrary body returning *InternalMultipartCompleteResponse func (c *ClientWithResponses) InternalMultipartCompleteWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalMultipartCompleteResponse, error) { rsp, err := c.InternalMultipartCompleteWithBody(ctx, contentType, body, reqEditors...) @@ -3113,6 +3360,58 @@ func ParseInternalInspectProjectBucketInventoryResponse(rsp *http.Response) (*In return response, nil } +// ParseInternalInspectProjectScopesResponse parses an HTTP response from a InternalInspectProjectScopesWithResponse call +func ParseInternalInspectProjectScopesResponse(rsp *http.Response) (*InternalInspectProjectScopesResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalInspectProjectScopesResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest InternalProjectScopesResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseInternalInspectProjectScopesPostResponse parses an HTTP response from a InternalInspectProjectScopesPostWithResponse call +func ParseInternalInspectProjectScopesPostResponse(rsp *http.Response) (*InternalInspectProjectScopesPostResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalInspectProjectScopesPostResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest InternalProjectScopesResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseInternalMultipartCompleteResponse parses an HTTP response from a InternalMultipartCompleteWithResponse call func ParseInternalMultipartCompleteResponse(rsp *http.Response) (*InternalMultipartCompleteResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/openapi/internal.openapi.yaml b/apigen/openapi/internal.openapi.yaml index a8160810..c7b1a34c 100644 --- a/apigen/openapi/internal.openapi.yaml +++ b/apigen/openapi/internal.openapi.yaml @@ -368,6 +368,64 @@ paths: description: Content too large '500': description: Internal server error + /data/inspect/project-scopes: + get: + tags: + - internal-data + operationId: internalInspectProjectScopes + description: List bucket scopes visible for a project. + parameters: + - in: query + name: organization + required: true + schema: + type: string + - in: query + name: project + required: true + schema: + type: string + responses: + '200': + description: Project bucket scopes + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectScopesResponse' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error + post: + tags: + - internal-data + operationId: internalInspectProjectScopesPost + description: Compatibility request-body form of project scope listing. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectScopesRequest' + responses: + '200': + description: Project bucket scopes + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectScopesResponse' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error /data/inspect/project-bucket/inventory: post: tags: @@ -851,6 +909,34 @@ components: type: string last_modified: type: string + InternalProjectScopesRequest: + type: object + required: + - organization + - project + properties: + organization: + type: string + project: + type: string + InternalProjectScopesResponse: + type: object + properties: + items: + type: array + items: + $ref: '#/components/schemas/InternalProjectScopeItem' + InternalProjectScopeItem: + type: object + properties: + bucket: + type: string + organization: + type: string + project_id: + type: string + path: + type: string InternalUploadBlankRequest: type: object properties: diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 6b66c97b..1a830c35 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -137,6 +137,25 @@ type InternalProjectBucketInventorySummary struct { TotalBytes *int64 `json:"total_bytes,omitempty"` } +// InternalProjectScopeItem defines model for InternalProjectScopeItem. +type InternalProjectScopeItem struct { + Bucket *string `json:"bucket,omitempty"` + Organization *string `json:"organization,omitempty"` + Path *string `json:"path,omitempty"` + ProjectId *string `json:"project_id,omitempty"` +} + +// InternalProjectScopesRequest defines model for InternalProjectScopesRequest. +type InternalProjectScopesRequest struct { + Organization string `json:"organization"` + Project string `json:"project"` +} + +// InternalProjectScopesResponse defines model for InternalProjectScopesResponse. +type InternalProjectScopesResponse struct { + Items *[]InternalProjectScopeItem `json:"items,omitempty"` +} + // InternalRecord defines model for InternalRecord. type InternalRecord struct { AccessMethods *[]externalRef0.AccessMethod `json:"access_methods,omitempty"` @@ -245,6 +264,12 @@ type InternalDownloadPartParams struct { End int64 `form:"end" json:"end"` } +// InternalInspectProjectScopesParams defines parameters for InternalInspectProjectScopes. +type InternalInspectProjectScopesParams struct { + Organization string `form:"organization" json:"organization"` + Project string `form:"project" json:"project"` +} + // InternalUploadURLParams defines parameters for InternalUploadURL. type InternalUploadURLParams struct { Organization *string `form:"organization,omitempty" json:"organization,omitempty"` @@ -278,6 +303,9 @@ type InternalListParams struct { // InternalInspectProjectBucketInventoryJSONRequestBody defines body for InternalInspectProjectBucketInventory for application/json ContentType. type InternalInspectProjectBucketInventoryJSONRequestBody = InternalProjectBucketInventoryRequest +// InternalInspectProjectScopesPostJSONRequestBody defines body for InternalInspectProjectScopesPost for application/json ContentType. +type InternalInspectProjectScopesPostJSONRequestBody = InternalProjectScopesRequest + // InternalMultipartCompleteJSONRequestBody defines body for InternalMultipartComplete for application/json ContentType. type InternalMultipartCompleteJSONRequestBody = InternalMultipartCompleteRequest @@ -391,6 +419,12 @@ type ServerInterface interface { // (POST /data/inspect/project-bucket/inventory) InternalInspectProjectBucketInventory(c fiber.Ctx) error + // (GET /data/inspect/project-scopes) + InternalInspectProjectScopes(c fiber.Ctx, params InternalInspectProjectScopesParams) error + + // (POST /data/inspect/project-scopes) + InternalInspectProjectScopesPost(c fiber.Ctx) error + // (POST /data/multipart/complete) InternalMultipartComplete(c fiber.Ctx) error @@ -541,6 +575,47 @@ func (siw *ServerInterfaceWrapper) InternalInspectProjectBucketInventory(c fiber return siw.Handler.InternalInspectProjectBucketInventory(c) } +// InternalInspectProjectScopes operation middleware +func (siw *ServerInterfaceWrapper) InternalInspectProjectScopes(c fiber.Ctx) error { + var err error + var params InternalInspectProjectScopesParams + + // ------------- Required query parameter "organization" ------------- + if paramValue := c.Query("organization"); paramValue != "" { + + var value string + err = runtime.BindStyledParameterWithOptions("form", "organization", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter organization: %w", err).Error()) + } + params.Organization = value + + } else { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument organization is required, but not found").Error()) + } + // ------------- Required query parameter "project" ------------- + if paramValue := c.Query("project"); paramValue != "" { + + var value string + err = runtime.BindStyledParameterWithOptions("form", "project", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter project: %w", err).Error()) + } + params.Project = value + + } else { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument project is required, but not found").Error()) + } + + return siw.Handler.InternalInspectProjectScopes(c, params) +} + +// InternalInspectProjectScopesPost operation middleware +func (siw *ServerInterfaceWrapper) InternalInspectProjectScopesPost(c fiber.Ctx) error { + + return siw.Handler.InternalInspectProjectScopesPost(c) +} + // InternalMultipartComplete operation middleware func (siw *ServerInterfaceWrapper) InternalMultipartComplete(c fiber.Ctx) error { @@ -927,6 +1002,10 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Post(options.BaseURL+"/data/inspect/project-bucket/inventory", wrapper.InternalInspectProjectBucketInventory) + router.Get(options.BaseURL+"/data/inspect/project-scopes", wrapper.InternalInspectProjectScopes) + + router.Post(options.BaseURL+"/data/inspect/project-scopes", wrapper.InternalInspectProjectScopesPost) + router.Post(options.BaseURL+"/data/multipart/complete", wrapper.InternalMultipartComplete) router.Post(options.BaseURL+"/data/multipart/init", wrapper.InternalMultipartInit) @@ -1154,6 +1233,104 @@ func (response InternalInspectProjectBucketInventory500Response) VisitInternalIn return nil } +type InternalInspectProjectScopesRequestObject struct { + Params InternalInspectProjectScopesParams +} + +type InternalInspectProjectScopesResponseObject interface { + VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error +} + +type InternalInspectProjectScopes200JSONResponse InternalProjectScopesResponse + +func (response InternalInspectProjectScopes200JSONResponse) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalInspectProjectScopes400Response struct { +} + +func (response InternalInspectProjectScopes400Response) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalInspectProjectScopes401Response struct { +} + +func (response InternalInspectProjectScopes401Response) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalInspectProjectScopes403Response struct { +} + +func (response InternalInspectProjectScopes403Response) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalInspectProjectScopes500Response struct { +} + +func (response InternalInspectProjectScopes500Response) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + +type InternalInspectProjectScopesPostRequestObject struct { + Body *InternalInspectProjectScopesPostJSONRequestBody +} + +type InternalInspectProjectScopesPostResponseObject interface { + VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error +} + +type InternalInspectProjectScopesPost200JSONResponse InternalProjectScopesResponse + +func (response InternalInspectProjectScopesPost200JSONResponse) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalInspectProjectScopesPost400Response struct { +} + +func (response InternalInspectProjectScopesPost400Response) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalInspectProjectScopesPost401Response struct { +} + +func (response InternalInspectProjectScopesPost401Response) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalInspectProjectScopesPost403Response struct { +} + +func (response InternalInspectProjectScopesPost403Response) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalInspectProjectScopesPost500Response struct { +} + +func (response InternalInspectProjectScopesPost500Response) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + type InternalMultipartCompleteRequestObject struct { Body *InternalMultipartCompleteJSONRequestBody } @@ -2018,6 +2195,12 @@ type StrictServerInterface interface { // (POST /data/inspect/project-bucket/inventory) InternalInspectProjectBucketInventory(ctx context.Context, request InternalInspectProjectBucketInventoryRequestObject) (InternalInspectProjectBucketInventoryResponseObject, error) + // (GET /data/inspect/project-scopes) + InternalInspectProjectScopes(ctx context.Context, request InternalInspectProjectScopesRequestObject) (InternalInspectProjectScopesResponseObject, error) + + // (POST /data/inspect/project-scopes) + InternalInspectProjectScopesPost(ctx context.Context, request InternalInspectProjectScopesPostRequestObject) (InternalInspectProjectScopesPostResponseObject, error) + // (POST /data/multipart/complete) InternalMultipartComplete(ctx context.Context, request InternalMultipartCompleteRequestObject) (InternalMultipartCompleteResponseObject, error) @@ -2173,6 +2356,64 @@ func (sh *strictHandler) InternalInspectProjectBucketInventory(ctx fiber.Ctx) er return nil } +// InternalInspectProjectScopes operation middleware +func (sh *strictHandler) InternalInspectProjectScopes(ctx fiber.Ctx, params InternalInspectProjectScopesParams) error { + var request InternalInspectProjectScopesRequestObject + + request.Params = params + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalInspectProjectScopes(ctx.Context(), request.(InternalInspectProjectScopesRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalInspectProjectScopes") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalInspectProjectScopesResponseObject); ok { + if err := validResponse.VisitInternalInspectProjectScopesResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + +// InternalInspectProjectScopesPost operation middleware +func (sh *strictHandler) InternalInspectProjectScopesPost(ctx fiber.Ctx) error { + var request InternalInspectProjectScopesPostRequestObject + + var body InternalInspectProjectScopesPostJSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalInspectProjectScopesPost(ctx.Context(), request.(InternalInspectProjectScopesPostRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalInspectProjectScopesPost") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalInspectProjectScopesPostResponseObject); ok { + if err := validResponse.VisitInternalInspectProjectScopesPostResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + // InternalMultipartComplete operation middleware func (sh *strictHandler) InternalMultipartComplete(ctx fiber.Ctx) error { var request InternalMultipartCompleteRequestObject diff --git a/internal/api/internaldrs/inspect.go b/internal/api/internaldrs/inspect.go index 62467dbe..213091ba 100644 --- a/internal/api/internaldrs/inspect.go +++ b/internal/api/internaldrs/inspect.go @@ -388,8 +388,14 @@ func handleInternalInspectProjectScopesFiber(om *core.ObjectManager) fiber.Handl return c.SendStatus(fiber.StatusUnauthorized) } var req internalInspectProjectScopesRequest - if err := decodeStrictJSON(c.Body(), &req); err != nil { - return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + switch c.Method() { + case fiber.MethodGet: + req.Organization = c.Query("organization") + req.Project = c.Query("project") + default: + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } } organization := strings.TrimSpace(req.Organization) project := strings.TrimSpace(req.Project) diff --git a/internal/api/internaldrs/inspect_routes_test.go b/internal/api/internaldrs/inspect_routes_test.go index 0ea9be93..148ba021 100644 --- a/internal/api/internaldrs/inspect_routes_test.go +++ b/internal/api/internaldrs/inspect_routes_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "strings" @@ -198,7 +199,6 @@ func TestProjectRecordMatchesAnyPathPrefixAvoidsFalsePrefixMatches(t *testing.T) } func TestHandleInternalInspectProjectScopes(t *testing.T) { - body, _ := json.Marshal(internalInspectProjectScopesRequest{Organization: "syfon", Project: "e2e"}) db := &testutils.MockDatabase{ Credentials: map[string]models.S3Credential{ "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, @@ -209,23 +209,46 @@ func TestHandleInternalInspectProjectScopes(t *testing.T) { }, } om := core.NewObjectManager(db, &testutils.MockUrlManager{}) - req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-scopes", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + authz := map[string]map[string]bool{ "/organization/syfon": {"read": true}, "/organization/syfon/project/e2e": {"read": true}, - }) - rr := doInternalDRSTestRequest(req, om) - if rr.Code != http.StatusOK { - t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) } - var resp internalInspectProjectScopesResponse - if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { - t.Fatalf("decode response: %v", err) - } - if len(resp.Items) != 2 { - t.Fatalf("expected 2 items, got %d", len(resp.Items)) + cases := []struct { + name string + method string + path string + body io.Reader + }{ + { + name: "get query", + method: http.MethodGet, + path: "/data/inspect/project-scopes?organization=syfon&project=e2e", + }, + { + name: "post body", + method: http.MethodPost, + path: "/data/inspect/project-scopes", + body: bytes.NewBufferString(`{"organization":"syfon","project":"e2e"}`), + }, } - if resp.Items[0].Bucket != "bucket-a" || !strings.HasPrefix(resp.Items[0].Path, "s3://bucket-a/") { - t.Fatalf("unexpected first scope: %+v", resp.Items[0]) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := withTestAuthzContext(httptest.NewRequest(tc.method, tc.path, tc.body), "gen3", authz) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectProjectScopesResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Items) != 2 { + t.Fatalf("expected 2 items, got %d", len(resp.Items)) + } + if resp.Items[0].Bucket != "bucket-a" || !strings.HasPrefix(resp.Items[0].Path, "s3://bucket-a/") { + t.Fatalf("unexpected first scope: %+v", resp.Items[0]) + } + }) } } diff --git a/internal/api/internaldrs/transfer.go b/internal/api/internaldrs/transfer.go index f2a65302..72e1797c 100644 --- a/internal/api/internaldrs/transfer.go +++ b/internal/api/internaldrs/transfer.go @@ -37,6 +37,7 @@ func registerInternalTransferRoutes(router fiber.Router, om *core.ObjectManager) router.Post(common.RouteInternalInspectProjectBucket, handleInternalInspectProjectBucketFiber(om)) router.Post(common.RouteInternalInspectProjectBucketInventory, handleInternalInspectProjectBucketInventoryFiber(om)) router.Post(common.RouteInternalInspectProjectRecords, handleInternalInspectProjectRecordsFiber(om)) + router.Get(common.RouteInternalInspectProjectScopes, handleInternalInspectProjectScopesFiber(om)) router.Post(common.RouteInternalInspectProjectScopes, handleInternalInspectProjectScopesFiber(om)) router.Post(common.RouteInternalDeleteProjectBucketObjects, handleInternalDeleteProjectBucketObjectsFiber(om)) router.Post(common.RouteInternalUploadBulk, handleInternalUploadBulkFiber(om)) From e2a6838a271f2e520e9f8c2d991e323a1b1fc846 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 10 Jul 2026 13:02:59 -0700 Subject: [PATCH 20/24] update syfon to not trust bucket outputs --- internal/api/internaldrs/inspect.go | 36 +- .../api/internaldrs/inspect_routes_test.go | 5 + internal/core/object_storage_inspect.go | 46 +- internal/core/object_storage_inspect_test.go | 66 ++- internal/core/object_storage_inventory.go | 447 ++++++++++++++++-- .../core/object_storage_inventory_test.go | 339 +++++++++++++ internal/core/s3_probe_limiter.go | 71 +++ internal/core/service.go | 2 + 8 files changed, 940 insertions(+), 72 deletions(-) create mode 100644 internal/core/object_storage_inventory_test.go create mode 100644 internal/core/s3_probe_limiter.go diff --git a/internal/api/internaldrs/inspect.go b/internal/api/internaldrs/inspect.go index 213091ba..a11b5716 100644 --- a/internal/api/internaldrs/inspect.go +++ b/internal/api/internaldrs/inspect.go @@ -123,15 +123,16 @@ type internalInspectProjectBucketSummary struct { } type internalInspectProjectBucketItem struct { - ObjectURL string `json:"object_url"` - Provider string `json:"provider"` - Bucket string `json:"bucket"` - Key string `json:"key"` - Path string `json:"path"` - SizeBytes int64 `json:"size_bytes"` - MetaSHA256 string `json:"meta_sha256,omitempty"` - ETag string `json:"etag,omitempty"` - LastModTime string `json:"last_modified,omitempty"` + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + MetaSHA256 string `json:"meta_sha256,omitempty"` + ETag string `json:"etag,omitempty"` + LastModTime string `json:"last_modified,omitempty"` + InventoryComplete bool `json:"inventory_complete,omitempty"` } type internalInspectProjectRecordItem struct { @@ -558,14 +559,15 @@ func projectBucketInventoryResponseFromCore(result *core.ProjectStorageInspectRe } for _, item := range result.Items { row := internalInspectProjectBucketItem{ - ObjectURL: item.ObjectURL, - Provider: item.Provider, - Bucket: item.Bucket, - Key: item.Key, - Path: item.Path, - SizeBytes: item.SizeBytes, - MetaSHA256: item.MetaSHA256, - ETag: item.ETag, + ObjectURL: item.ObjectURL, + Provider: item.Provider, + Bucket: item.Bucket, + Key: item.Key, + Path: item.Path, + SizeBytes: item.SizeBytes, + MetaSHA256: item.MetaSHA256, + ETag: item.ETag, + InventoryComplete: true, } if !item.LastModTime.IsZero() { row.LastModTime = item.LastModTime.Format(time.RFC3339) diff --git a/internal/api/internaldrs/inspect_routes_test.go b/internal/api/internaldrs/inspect_routes_test.go index 148ba021..b58d8d0f 100644 --- a/internal/api/internaldrs/inspect_routes_test.go +++ b/internal/api/internaldrs/inspect_routes_test.go @@ -409,6 +409,11 @@ func TestHandleInternalInspectProjectBucketInventoryListsProjectScope(t *testing if resp.Summary.ObjectCount != 2 || resp.Summary.TotalBytes != 25 || len(resp.Items) != 2 { t.Fatalf("unexpected inventory response summary=%+v items=%+v", resp.Summary, resp.Items) } + for _, item := range resp.Items { + if !item.InventoryComplete { + t.Fatalf("expected successful inventory item to be marked complete, got %+v", item) + } + } if len(listCalls) != 1 || listCalls[0].IncludeHead || listCalls[0].MaxKeys != 0 || listCalls[0].ExactPrefix { t.Fatalf("expected one recursive LIST without HEAD, got %+v", listCalls) } diff --git a/internal/core/object_storage_inspect.go b/internal/core/object_storage_inspect.go index eae8c6e0..cf4f8ee8 100644 --- a/internal/core/object_storage_inspect.go +++ b/internal/core/object_storage_inspect.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log" "path" "sort" "strings" @@ -94,16 +95,26 @@ const ( StorageInspectPermissionDenied StorageInspectErrorKind = "permission_denied" StorageInspectObjectNotFound StorageInspectErrorKind = "object_not_found" StorageInspectBucketUnavailable StorageInspectErrorKind = "bucket_unavailable" + StorageInspectListingIncomplete StorageInspectErrorKind = "listing_incomplete" StorageInspectUnsupported StorageInspectErrorKind = "unsupported" ) -const maxStorageInspectWorkers = 16 +const maxStorageInspectWorkers = 8 + +const ( + defaultS3HeadMaxAttempts = 3 + envS3HeadMaxAttempts = "SYFON_S3_HEAD_MAX_ATTEMPTS" +) type StorageInspectError struct { Kind StorageInspectErrorKind Message string } +type s3HeadObjectClient interface { + HeadObject(context.Context, *awss3.HeadObjectInput, ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) +} + var storageInspectCacheKey contextKey = "storageInspectCache" type storageInspectCredentialCacheEntry struct { @@ -155,6 +166,7 @@ func (m *ObjectManager) SetS3ObjectInspector(fn func(context.Context, models.S3C } func (m *ObjectManager) InspectStorageObject(ctx context.Context, req InspectStorageRequest) (*StorageObjectMetadata, error) { + ctx = withS3ProbeLimiter(ctx, m.s3ProbeLimiter) if strings.TrimSpace(req.ObjectURL) != "" { return m.inspectRawStorageObject(ctx, req) } @@ -162,6 +174,7 @@ func (m *ObjectManager) InspectStorageObject(ctx context.Context, req InspectSto } func (m *ObjectManager) InspectStorageObjects(ctx context.Context, items []InspectStorageRequest) []StorageProbeResult { + ctx = withS3ProbeLimiter(ctx, m.s3ProbeLimiter) ctx = WithStorageInspectCache(ctx) if len(items) == 0 { return []StorageProbeResult{} @@ -617,18 +630,17 @@ func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, buc if err != nil { return nil, err } - out, err := client.HeadObject(ctx, &awss3.HeadObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) + out, err := headS3ObjectWithRetry(ctx, client, bucket, key) if err != nil { var apiErr smithy.APIError if errors.As(err, &apiErr) { switch strings.ToLower(strings.TrimSpace(apiErr.ErrorCode())) { case "forbidden", "accessdenied", "permissiondenied": return nil, &StorageInspectError{Kind: StorageInspectBucketUnavailable, Message: fmt.Sprintf("provider rejected object probe for s3://%s/%s; mapped bucket target may be missing or inaccessible", bucket, key)} - case "notfound", "nosuchkey", "nosuchbucket": + case "notfound", "nosuchkey": return nil, &StorageInspectError{Kind: StorageInspectObjectNotFound, Message: fmt.Sprintf("provider could not find s3://%s/%s", bucket, key)} + case "nosuchbucket": + return nil, &StorageInspectError{Kind: StorageInspectBucketUnavailable, Message: fmt.Sprintf("provider could not find bucket %q", bucket)} } } return nil, fmt.Errorf("inspect s3 object %s/%s: %w", bucket, key, err) @@ -652,3 +664,25 @@ func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, buc LastModTime: lastMod, }, nil } + +func headS3ObjectWithRetry(ctx context.Context, client s3HeadObjectClient, bucket, key string) (*awss3.HeadObjectOutput, error) { + policy := s3ListPageRetryPolicyFromEnv() + maxAttempts := intEnvOrDefault(envS3HeadMaxAttempts, defaultS3HeadMaxAttempts, 1) + for attempt := 1; attempt <= maxAttempts; attempt++ { + release, acquireErr := acquireS3Probe(ctx, "head", bucket, key) + if acquireErr != nil { + return nil, acquireErr + } + out, err := client.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}) + release() + if err == nil || !isRetryableS3ListPageError(err) || attempt == maxAttempts { + return out, err + } + backoff := policy.backoff(attempt) + log.Printf("INFO: syfon_s3_head_retry request_id=%s bucket=%s key=%q attempt=%d max_attempts=%d backoff_ms=%d error=%q", common.GetRequestID(ctx), bucket, key, attempt+1, maxAttempts, backoff.Milliseconds(), err.Error()) + if err := sleepS3ListPageRetry(ctx, backoff); err != nil { + return nil, err + } + } + return nil, nil +} diff --git a/internal/core/object_storage_inspect_test.go b/internal/core/object_storage_inspect_test.go index e443f312..fe6e0661 100644 --- a/internal/core/object_storage_inspect_test.go +++ b/internal/core/object_storage_inspect_test.go @@ -1,6 +1,15 @@ package core -import "testing" +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" +) func TestClassifyProbeErrorCredentialMissingIsNotBucketMiss(t *testing.T) { status, kind := classifyStorageProbeError(&StorageInspectError{ @@ -21,3 +30,58 @@ func TestClassifyProbeErrorObjectNotFoundRemainsBucketMiss(t *testing.T) { t.Fatalf("expected object_not_found to classify as not_found, got status=%q kind=%q", status, kind) } } + +type fakeS3HeadClient struct { + calls atomic.Int32 +} + +func (client *fakeS3HeadClient) HeadObject(_ context.Context, _ *awss3.HeadObjectInput, _ ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) { + if client.calls.Add(1) == 1 { + return nil, &smithy.GenericAPIError{Code: "SlowDown", Message: "retry"} + } + return &awss3.HeadObjectOutput{ContentLength: aws.Int64(42)}, nil +} + +func TestHeadS3ObjectWithRetryRecoversTransientFailure(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3HeadMaxAttempts, "2") + client := &fakeS3HeadClient{} + + out, err := headS3ObjectWithRetry(context.Background(), client, "bucket", "key") + if err != nil { + t.Fatalf("head object: %v", err) + } + if out == nil || aws.ToInt64(out.ContentLength) != 42 || client.calls.Load() != 2 { + t.Fatalf("expected retry then successful HEAD, output=%+v calls=%d", out, client.calls.Load()) + } +} + +func TestS3ProbeLimiterSharesCapacityAcrossContexts(t *testing.T) { + limiter := newS3ProbeLimiter(1) + ctx := withS3ProbeLimiter(context.Background(), limiter) + release, err := acquireS3Probe(ctx, "head", "bucket", "first") + if err != nil { + t.Fatalf("acquire first permit: %v", err) + } + + acquired := make(chan struct{}) + go func() { + secondRelease, err := acquireS3Probe(ctx, "list", "bucket", "second") + if err == nil { + secondRelease() + close(acquired) + } + }() + select { + case <-acquired: + t.Fatal("second operation acquired before the first released its permit") + case <-time.After(20 * time.Millisecond): + } + release() + select { + case <-acquired: + case <-time.After(time.Second): + t.Fatal("second operation did not acquire after release") + } +} diff --git a/internal/core/object_storage_inventory.go b/internal/core/object_storage_inventory.go index 396918d3..b37b2da9 100644 --- a/internal/core/object_storage_inventory.go +++ b/internal/core/object_storage_inventory.go @@ -2,17 +2,26 @@ package core import ( "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "io" "log" + "math/rand" + "net" + "os" "path" "sort" + "strconv" "strings" "sync" "time" "github.com/aws/aws-sdk-go-v2/aws" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" syfoncommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/authz" @@ -54,6 +63,20 @@ const ( storagePrefixListLoggingDisabled = "disabled" ) +const ( + defaultS3ListPageSize = int32(1000) + defaultS3ListPageMaxAttempts = 12 + defaultS3ExactProbeMaxAttempts = 3 + defaultS3ListPageInitialBackoff = 500 * time.Millisecond + defaultS3ListPageMaxBackoff = 10 * time.Second + defaultS3InventoryTerminalReplays = 3 + envS3ListPageMaxAttempts = "SYFON_S3_LIST_PAGE_MAX_ATTEMPTS" + envS3ExactProbeMaxAttempts = "SYFON_S3_EXACT_PROBE_MAX_ATTEMPTS" + envS3ListPageInitialBackoffMillis = "SYFON_S3_LIST_PAGE_INITIAL_BACKOFF_MS" + envS3ListPageMaxBackoffMillis = "SYFON_S3_LIST_PAGE_MAX_BACKOFF_MS" + envS3InventoryTerminalReplays = "SYFON_S3_INVENTORY_TERMINAL_REPLAY_ATTEMPTS" +) + type ProjectStorageSummary struct { Provider string Bucket string @@ -86,6 +109,7 @@ type storageListTargetWork struct { } type storageListRunStats struct { + mu sync.Mutex inputItemCount int distinctTargetCount int duplicateCount int @@ -96,6 +120,12 @@ type storageListRunStats struct { fallbackCount int } +func (s *storageListRunStats) incrementExactListCalls() { + s.mu.Lock() + s.exactListCallCount++ + s.mu.Unlock() +} + type StorageListValidationRequest struct { ID string ObjectURL string @@ -136,6 +166,26 @@ type ProjectStorageDeleteResult struct { Error string } +type s3ListObjectsV2Client interface { + ListObjectsV2(context.Context, *awss3.ListObjectsV2Input, ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) +} + +type s3PrefixListStats struct { + Pages int + Retries int + LastKey string + FailedPage int + LastTokenID string + TerminalReplayAttempts int + TerminalDisagreements int +} + +type s3ListPageRetryPolicy struct { + MaxAttempts int + InitialBackoff time.Duration + MaxBackoff time.Duration +} + func (m *ObjectManager) SetS3PrefixLister(fn func(context.Context, models.S3Credential, string, string, bool) ([]StorageBucketObject, error)) { if fn == nil { m.listS3Prefix = defaultS3PrefixLister @@ -164,6 +214,7 @@ func (m *ObjectManager) ListProjectStorageObjects(ctx context.Context, organizat func (m *ObjectManager) InspectProjectStorage(ctx context.Context, organization, project string, inspectOptions ProjectStorageInspectOptions) (*ProjectStorageInspectResult, error) { started := time.Now() + ctx = withS3ProbeLimiter(ctx, m.s3ProbeLimiter) ctx = WithStorageInspectCache(ctx) target, err := m.resolveProjectStorageScopeTarget(ctx, organization, project) if err != nil { @@ -420,6 +471,7 @@ func uniqueStorageObjectURLs(values []string) []string { func (m *ObjectManager) ListValidateStorageObjects(ctx context.Context, items []StorageListValidationRequest) []StorageListValidationResult { started := time.Now() + ctx = withS3ProbeLimiter(ctx, m.s3ProbeLimiter) ctx = WithStorageInspectCache(ctx) ctx = withStoragePrefixListLogging(ctx, storagePrefixListLoggingDisabled) if len(items) == 0 { @@ -715,14 +767,14 @@ func (m *ObjectManager) runExactStorageListTarget(ctx context.Context, work *sto started := time.Now() matches, err := m.listS3Prefix(ctx, work.cred, work.bucket, work.key, StoragePrefixListOptions{ExactPrefix: true, MaxKeys: 1}) if err != nil { - log.Printf("INFO: syfon_bulk_list_validate_exact bucket=%s key=%q duration_ms=%d error=%q", work.bucket, work.key, time.Since(started).Milliseconds(), err.Error()) + log.Printf("INFO: syfon_bulk_list_validate_exact request_id=%s bucket=%s key=%q duration_ms=%d error=%q", common.GetRequestID(ctx), work.bucket, work.key, time.Since(started).Milliseconds(), err.Error()) result := work.baseResult result.Status, result.ErrorKind = classifyStorageProbeError(err) result.Error = strings.TrimSpace(err.Error()) - stats.exactListCallCount++ + stats.incrementExactListCalls() return result, StorageBucketObject{}, false } - stats.exactListCallCount++ + stats.incrementExactListCalls() for _, item := range matches { key := strings.Trim(strings.TrimSpace(item.Key), "/") if key != work.key { @@ -733,7 +785,7 @@ func (m *ObjectManager) runExactStorageListTarget(ctx context.Context, work *sto break } if duration := time.Since(started); duration > storageListSlowExactThreshold { - log.Printf("INFO: syfon_bulk_list_validate_exact bucket=%s key=%q duration_ms=%d status=present", work.bucket, work.key, duration.Milliseconds()) + log.Printf("INFO: syfon_bulk_list_validate_exact request_id=%s bucket=%s key=%q duration_ms=%d status=present", common.GetRequestID(ctx), work.bucket, work.key, duration.Milliseconds()) } result := work.baseResult result.Exists = true @@ -743,7 +795,7 @@ func (m *ObjectManager) runExactStorageListTarget(ctx context.Context, work *sto return result, normalized[0], true } if duration := time.Since(started); duration > storageListSlowExactThreshold { - log.Printf("INFO: syfon_bulk_list_validate_exact bucket=%s key=%q duration_ms=%d status=not_found", work.bucket, work.key, duration.Milliseconds()) + log.Printf("INFO: syfon_bulk_list_validate_exact request_id=%s bucket=%s key=%q duration_ms=%d status=not_found", common.GetRequestID(ctx), work.bucket, work.key, duration.Milliseconds()) } result := work.baseResult result.Status = StorageProbeStatusNotFound @@ -808,7 +860,8 @@ func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket return nil, err } input := &awss3.ListObjectsV2Input{ - Bucket: aws.String(bucket), + Bucket: aws.String(bucket), + MaxKeys: aws.Int32(defaultS3ListPageSize), } if trimmedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); trimmedPrefix != "" { if options.ExactPrefix { @@ -823,51 +876,21 @@ func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket requestPrefix := aws.ToString(input.Prefix) logEnabled := storagePrefixListLoggingEnabled(ctx) if logEnabled { - log.Printf("INFO: syfon_s3_prefix_list_start bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead) + log.Printf("INFO: syfon_s3_prefix_list_start request_id=%s bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t", common.GetRequestID(ctx), bucket, prefix, requestPrefix, options.ExactPrefix, aws.ToInt32(input.MaxKeys), options.IncludeHead) } - paginator := awss3.NewListObjectsV2Paginator(client, input) - out := make([]StorageBucketObject, 0) - pageCount := 0 - firstKeys := make([]string, 0, 5) - for paginator.HasMorePages() { - page, err := paginator.NextPage(ctx) - if err != nil { - if logEnabled { - log.Printf("INFO: syfon_s3_prefix_list_done bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d duration_ms=%d error=%q", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead, pageCount, len(out), time.Since(started).Milliseconds(), err.Error()) - } - return nil, classifyS3ListError(bucket, prefix, err) + out, stats, firstKeys, err := listS3PrefixPagesWithExactProbeRetry(ctx, client, input, bucket, prefix, requestPrefix, options, logEnabled) + if err != nil { + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_done request_id=%s bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d retries=%d terminal_replay_attempts=%d terminal_disagreements=%d failed_page=%d token=%s last_key=%q duration_ms=%d error=%q", common.GetRequestID(ctx), bucket, prefix, requestPrefix, options.ExactPrefix, aws.ToInt32(input.MaxKeys), options.IncludeHead, stats.Pages, len(out), stats.Retries, stats.TerminalReplayAttempts, stats.TerminalDisagreements, stats.FailedPage, stats.LastTokenID, stats.LastKey, time.Since(started).Milliseconds(), err.Error()) } - pageCount++ - for _, item := range page.Contents { - key := strings.Trim(strings.TrimSpace(aws.ToString(item.Key)), "/") - if key == "" { - continue - } - if len(firstKeys) < cap(firstKeys) { - firstKeys = append(firstKeys, key) - } - var size int64 - if item.Size != nil { - size = *item.Size - } - lastMod := time.Time{} - if item.LastModified != nil { - lastMod = *item.LastModified - } - out = append(out, StorageBucketObject{ - Provider: common.S3Provider, - Bucket: bucket, - Key: key, - ObjectURL: common.BucketToURL(bucket, key), - Path: path.Base(key), - SizeBytes: size, - ETag: strings.Trim(strings.TrimSpace(aws.ToString(item.ETag)), "\""), - LastModTime: lastMod, - }) + var inspectErr *StorageInspectError + if errors.As(err, &inspectErr) { + return nil, inspectErr } + return nil, classifyS3ListError(bucket, prefix, err) } if logEnabled { - log.Printf("INFO: syfon_s3_prefix_list_done bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d first_keys=%q duration_ms=%d", bucket, prefix, requestPrefix, options.ExactPrefix, options.MaxKeys, options.IncludeHead, pageCount, len(out), strings.Join(firstKeys, ","), time.Since(started).Milliseconds()) + log.Printf("INFO: syfon_s3_prefix_list_done request_id=%s bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d retries=%d terminal_replay_attempts=%d terminal_disagreements=%d first_keys=%q duration_ms=%d", common.GetRequestID(ctx), bucket, prefix, requestPrefix, options.ExactPrefix, aws.ToInt32(input.MaxKeys), options.IncludeHead, stats.Pages, len(out), stats.Retries, stats.TerminalReplayAttempts, stats.TerminalDisagreements, strings.Join(firstKeys, ","), time.Since(started).Milliseconds()) } if !options.IncludeHead || len(out) == 0 { return out, nil @@ -919,6 +942,332 @@ func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket return out, nil } +// Full-prefix inventories use one token-only scan per request. Exact probes +// retain their cheaper bounded retry behavior because a transient omission is +// more common than a real deletion for a single known key. +func listS3PrefixPagesWithExactProbeRetry(ctx context.Context, client s3ListObjectsV2Client, input *awss3.ListObjectsV2Input, bucket, prefix, requestPrefix string, options StoragePrefixListOptions, logEnabled bool) ([]StorageBucketObject, s3PrefixListStats, []string, error) { + if options.MaxKeys != 1 { + return listS3PrefixPages(ctx, client, input, bucket, prefix, requestPrefix, options, logEnabled) + } + if !isExactProbeList(options) { + return listS3PrefixPages(ctx, client, input, bucket, prefix, requestPrefix, options, logEnabled) + } + policy := s3ListPageRetryPolicyFromEnv() + maxAttempts := intEnvOrDefault(envS3ExactProbeMaxAttempts, defaultS3ExactProbeMaxAttempts, 1) + for attempt := 1; attempt <= maxAttempts; attempt++ { + out, stats, firstKeys, err := listS3PrefixPages(ctx, client, input, bucket, prefix, requestPrefix, options, logEnabled) + if err != nil || hasExactListedKey(out, prefix) || attempt == maxAttempts { + return out, stats, firstKeys, err + } + backoff := policy.backoff(attempt) + if logEnabled { + log.Printf("INFO: syfon_s3_exact_probe_retry request_id=%s bucket=%s key=%q attempt=%d max_attempts=%d backoff_ms=%d objects=%d", common.GetRequestID(ctx), bucket, prefix, attempt+1, maxAttempts, backoff.Milliseconds(), len(out)) + } + if err := sleepS3ListPageRetry(ctx, backoff); err != nil { + return nil, stats, firstKeys, err + } + } + return nil, s3PrefixListStats{}, nil, nil +} + +func isExactProbeList(options StoragePrefixListOptions) bool { + return options.ExactPrefix && options.MaxKeys == 1 +} + +func hasExactListedKey(items []StorageBucketObject, key string) bool { + want := strings.Trim(strings.TrimSpace(key), "/") + for _, item := range items { + if strings.Trim(strings.TrimSpace(item.Key), "/") == want { + return true + } + } + return false +} + +func listS3PrefixPages(ctx context.Context, client s3ListObjectsV2Client, input *awss3.ListObjectsV2Input, bucket string, prefix string, requestPrefix string, options StoragePrefixListOptions, logEnabled bool) ([]StorageBucketObject, s3PrefixListStats, []string, error) { + policy := s3ListPageRetryPolicyFromEnv() + out := make([]StorageBucketObject, 0) + stats := s3PrefixListStats{} + firstKeys := make([]string, 0, 5) + seenKeys := make(map[string]struct{}) + continuationToken := "" + baseInput := cloneListObjectsV2Input(input, "") + baseInput.StartAfter = nil + for { + pageNumber := stats.Pages + 1 + page, tokenID, retries, err := listS3PrefixPageWithRetry(ctx, client, baseInput, continuationToken, bucket, prefix, requestPrefix, policy, pageNumber, len(out), stats.LastKey, logEnabled) + stats.LastTokenID = tokenID + stats.Retries += retries + if err != nil { + stats.FailedPage = pageNumber + return out, stats, firstKeys, err + } + stats.Pages++ + added := appendS3ListPageObjects(&out, page, bucket, &firstKeys, seenKeys) + if len(out) > 0 { + stats.LastKey = out[len(out)-1].Key + } + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_page_done request_id=%s bucket=%s requested_prefix=%q input_prefix=%q page=%d token=%s objects_added=%d objects_total=%d last_key=%q truncated=%t", common.GetRequestID(ctx), bucket, prefix, requestPrefix, pageNumber, tokenID, added, len(out), stats.LastKey, aws.ToBool(page.IsTruncated)) + } + if !aws.ToBool(page.IsTruncated) { + if options.MaxKeys != 1 { + replays, replayRetries, replayErr := replayS3TerminalPage(ctx, client, baseInput, continuationToken, page, bucket, prefix, requestPrefix, policy, pageNumber, len(out), stats.LastKey, logEnabled) + stats.TerminalReplayAttempts += replays + stats.Retries += replayRetries + if replayErr != nil { + stats.TerminalDisagreements++ + stats.FailedPage = pageNumber + return out, stats, firstKeys, replayErr + } + } + return out, stats, firstKeys, nil + } + continuationToken = strings.TrimSpace(aws.ToString(page.NextContinuationToken)) + if continuationToken == "" { + err := fmt.Errorf("list s3 objects for %s/%s stopped at page %d after %d objects: provider returned truncated page without next continuation token", bucket, strings.Trim(strings.TrimSpace(prefix), "/"), pageNumber, len(out)) + stats.FailedPage = pageNumber + return out, stats, firstKeys, err + } + baseInput.StartAfter = nil + } +} + +func replayS3TerminalPage(ctx context.Context, client s3ListObjectsV2Client, baseInput *awss3.ListObjectsV2Input, continuationToken string, firstPage *awss3.ListObjectsV2Output, bucket, prefix, requestPrefix string, policy s3ListPageRetryPolicy, pageNumber, objectCount int, lastKey string, logEnabled bool) (int, int, error) { + maxAttempts := intEnvOrDefault(envS3InventoryTerminalReplays, defaultS3InventoryTerminalReplays, 2) + firstFingerprint := s3ListPageFingerprint(firstPage) + replays := 0 + retries := 0 + for attempt := 2; attempt <= maxAttempts; attempt++ { + page, tokenID, pageRetries, err := listS3PrefixPageWithRetry(ctx, client, baseInput, continuationToken, bucket, prefix, requestPrefix, policy, pageNumber, objectCount, lastKey, logEnabled) + replays++ + if err != nil { + return replays, retries + pageRetries, incompleteS3ListingError(bucket, prefix, lastKey, "terminal replay failed", err) + } + retries += pageRetries + if logEnabled { + log.Printf("INFO: syfon_s3_inventory_terminal_replay request_id=%s bucket=%s requested_prefix=%q page=%d token=%s replay=%d max_replays=%d objects=%d truncated=%t", common.GetRequestID(ctx), bucket, prefix, pageNumber, tokenID, attempt-1, maxAttempts-1, len(page.Contents), aws.ToBool(page.IsTruncated)) + } + if s3ListPageFingerprint(page) != firstFingerprint { + return replays, retries, incompleteS3ListingError(bucket, prefix, lastKey, "terminal replay returned different page content", nil) + } + } + return replays, retries, nil +} + +func s3ListPageFingerprint(page *awss3.ListObjectsV2Output) string { + if page == nil { + return "nil" + } + objects := append([]types.Object(nil), page.Contents...) + sort.Slice(objects, func(i, j int) bool { + return aws.ToString(objects[i].Key) < aws.ToString(objects[j].Key) + }) + var builder strings.Builder + fmt.Fprintf(&builder, "truncated=%t\n", aws.ToBool(page.IsTruncated)) + for _, object := range objects { + lastModified := "" + if object.LastModified != nil { + lastModified = object.LastModified.UTC().Format(time.RFC3339Nano) + } + fmt.Fprintf(&builder, "%s\x00%d\x00%s\x00%s\n", aws.ToString(object.Key), aws.ToInt64(object.Size), aws.ToString(object.ETag), lastModified) + } + sum := sha256.Sum256([]byte(builder.String())) + return hex.EncodeToString(sum[:])[:16] +} + +func incompleteS3ListingError(bucket, prefix, lastKey, reason string, cause error) error { + message := fmt.Sprintf("provider returned an incomplete listing for s3://%s/%s after key %q: %s", bucket, strings.Trim(strings.TrimSpace(prefix), "/"), lastKey, reason) + if cause != nil { + message += ": " + cause.Error() + } + return &StorageInspectError{Kind: StorageInspectListingIncomplete, Message: message} +} + +func listS3PrefixPageWithRetry(ctx context.Context, client s3ListObjectsV2Client, baseInput *awss3.ListObjectsV2Input, continuationToken string, bucket string, prefix string, requestPrefix string, policy s3ListPageRetryPolicy, pageNumber int, objectCount int, lastKey string, logEnabled bool) (*awss3.ListObjectsV2Output, string, int, error) { + tokenID := s3ContinuationTokenFingerprint(continuationToken) + retries := 0 + for attempt := 1; ; attempt++ { + release, acquireErr := acquireS3Probe(ctx, "list", bucket, prefix) + if acquireErr != nil { + return nil, tokenID, retries, acquireErr + } + page, err := client.ListObjectsV2(ctx, cloneListObjectsV2Input(baseInput, continuationToken)) + release() + if err == nil { + if page == nil { + err = errors.New("provider returned an empty list page") + } else if aws.ToBool(page.IsTruncated) && strings.TrimSpace(aws.ToString(page.NextContinuationToken)) == "" { + err = errors.New("provider returned a malformed truncated list page without next continuation token") + } else if aws.ToBool(page.IsTruncated) && len(page.Contents) == 0 { + err = errors.New("provider returned an empty malformed truncated list page") + } else { + return page, tokenID, retries, nil + } + } + if !isRetryableS3ListPageError(err) || attempt >= policy.MaxAttempts { + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_page_failed request_id=%s bucket=%s requested_prefix=%q input_prefix=%q page=%d token=%s objects=%d attempt=%d max_attempts=%d last_key=%q retryable=%t error=%q", common.GetRequestID(ctx), bucket, prefix, requestPrefix, pageNumber, tokenID, objectCount, attempt, policy.MaxAttempts, lastKey, isRetryableS3ListPageError(err), err.Error()) + } + return nil, tokenID, retries, fmt.Errorf("list s3 objects for %s/%s failed at page %d after %d objects and %d attempts: %w", bucket, strings.Trim(strings.TrimSpace(prefix), "/"), pageNumber, objectCount, attempt, err) + } + retries++ + backoff := policy.backoff(attempt) + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_page_retry request_id=%s bucket=%s requested_prefix=%q input_prefix=%q page=%d token=%s objects=%d attempt=%d max_attempts=%d backoff_ms=%d last_key=%q error=%q", common.GetRequestID(ctx), bucket, prefix, requestPrefix, pageNumber, tokenID, objectCount, attempt, policy.MaxAttempts, backoff.Milliseconds(), lastKey, err.Error()) + } + if err := sleepS3ListPageRetry(ctx, backoff); err != nil { + return nil, tokenID, retries, err + } + } +} + +func appendS3ListPageObjects(out *[]StorageBucketObject, page *awss3.ListObjectsV2Output, bucket string, firstKeys *[]string, seenKeys map[string]struct{}) int { + before := len(*out) + for _, item := range page.Contents { + key := strings.Trim(strings.TrimSpace(aws.ToString(item.Key)), "/") + if key == "" { + continue + } + if _, seen := seenKeys[key]; seen { + continue + } + seenKeys[key] = struct{}{} + if len(*firstKeys) < cap(*firstKeys) { + *firstKeys = append(*firstKeys, key) + } + var size int64 + if item.Size != nil { + size = *item.Size + } + lastMod := time.Time{} + if item.LastModified != nil { + lastMod = *item.LastModified + } + *out = append(*out, StorageBucketObject{ + Provider: common.S3Provider, + Bucket: bucket, + Key: key, + ObjectURL: common.BucketToURL(bucket, key), + Path: path.Base(key), + SizeBytes: size, + ETag: strings.Trim(strings.TrimSpace(aws.ToString(item.ETag)), "\""), + LastModTime: lastMod, + }) + } + return len(*out) - before +} + +func cloneListObjectsV2Input(input *awss3.ListObjectsV2Input, continuationToken string) *awss3.ListObjectsV2Input { + clone := *input + clone.StartAfter = nil + if strings.TrimSpace(continuationToken) == "" { + clone.ContinuationToken = nil + } else { + clone.ContinuationToken = aws.String(continuationToken) + } + return &clone +} + +func isRetryableS3ListPageError(err error) bool { + if err == nil { + return false + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch strings.ToLower(strings.TrimSpace(apiErr.ErrorCode())) { + case "internalerror", "slowdown", "serviceunavailable", "requesttimeout", "requesttimeoutexception", "toomanyrequests", "throttling", "throttlingexception", "requestlimitexceeded": + return true + default: + return false + } + } + if errors.Is(err, io.EOF) { + return true + } + var netErr net.Error + if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "eof") || strings.Contains(msg, "connection reset") || strings.Contains(msg, "timeout") || + strings.Contains(msg, "malformed truncated list page") || strings.Contains(msg, "empty list page") +} + +func s3ListPageRetryPolicyFromEnv() s3ListPageRetryPolicy { + return s3ListPageRetryPolicy{ + MaxAttempts: intEnvOrDefault(envS3ListPageMaxAttempts, defaultS3ListPageMaxAttempts, 1), + InitialBackoff: millisEnvOrDefault(envS3ListPageInitialBackoffMillis, defaultS3ListPageInitialBackoff, 0), + MaxBackoff: millisEnvOrDefault(envS3ListPageMaxBackoffMillis, defaultS3ListPageMaxBackoff, 0), + } +} + +func intEnvOrDefault(name string, fallback int, minimum int) int { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + value, err := strconv.Atoi(raw) + if err != nil || value < minimum { + return fallback + } + return value +} + +func millisEnvOrDefault(name string, fallback time.Duration, minimumMillis int) time.Duration { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + value, err := strconv.Atoi(raw) + if err != nil || value < minimumMillis { + return fallback + } + return time.Duration(value) * time.Millisecond +} + +func (policy s3ListPageRetryPolicy) backoff(attempt int) time.Duration { + backoff := policy.InitialBackoff + for i := 1; i < attempt; i++ { + backoff *= 2 + if backoff >= policy.MaxBackoff { + backoff = policy.MaxBackoff + break + } + } + if backoff <= 0 { + return 0 + } + jitterMax := backoff / 4 + if jitterMax <= 0 { + return backoff + } + return backoff + time.Duration(rand.Int63n(int64(jitterMax)+1)) +} + +func s3ContinuationTokenFingerprint(token string) string { + token = strings.TrimSpace(token) + if token == "" { + return "start" + } + sum := sha1.Sum([]byte(token)) + return hex.EncodeToString(sum[:])[:12] +} + +var sleepS3ListPageRetry = func(ctx context.Context, duration time.Duration) error { + if duration <= 0 { + return nil + } + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + func classifyS3ListError(bucket string, prefix string, err error) error { var apiErr smithy.APIError if errors.As(err, &apiErr) { @@ -932,8 +1281,10 @@ func classifyS3ListError(bucket string, prefix string, err error) error { strings.Trim(strings.TrimSpace(prefix), "/"), ), } - case "notfound", "nosuchbucket": - return &StorageInspectError{Kind: StorageInspectObjectNotFound, Message: fmt.Sprintf("provider could not find bucket %q", bucket)} + case "nosuchbucket": + return &StorageInspectError{Kind: StorageInspectBucketUnavailable, Message: fmt.Sprintf("provider could not find bucket %q", bucket)} + case "notfound": + return &StorageInspectError{Kind: StorageInspectBucketUnavailable, Message: fmt.Sprintf("provider could not resolve bucket inventory target %q", bucket)} } } return fmt.Errorf("list s3 objects for %s/%s: %w", bucket, strings.Trim(strings.TrimSpace(prefix), "/"), err) diff --git a/internal/core/object_storage_inventory_test.go b/internal/core/object_storage_inventory_test.go new file mode 100644 index 00000000..cdbf0796 --- /dev/null +++ b/internal/core/object_storage_inventory_test.go @@ -0,0 +1,339 @@ +package core + +import ( + "context" + "errors" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" +) + +type fakeS3ListPage struct { + output *awss3.ListObjectsV2Output + err error +} + +type fakeS3ListClient struct { + pages map[string][]fakeS3ListPage + startAfterPages map[string][]fakeS3ListPage + calls []string + startAfters []string +} + +func (client *fakeS3ListClient) ListObjectsV2(_ context.Context, input *awss3.ListObjectsV2Input, _ ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) { + token := strings.TrimSpace(aws.ToString(input.ContinuationToken)) + startAfter := strings.TrimSpace(aws.ToString(input.StartAfter)) + client.calls = append(client.calls, token) + client.startAfters = append(client.startAfters, startAfter) + if startAfter != "" { + queue := client.startAfterPages[startAfter] + if len(queue) == 0 { + return finalListPage(), nil + } + next := queue[0] + client.startAfterPages[startAfter] = queue[1:] + return next.output, next.err + } + queue := client.pages[token] + if len(queue) == 0 { + return nil, errors.New("unexpected token " + token) + } + next := queue[0] + client.pages[token] = queue[1:] + return next.output, next.err +} + +func TestListS3PrefixPagesRetriesTransientPageFailure(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "4") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: listPage("token-2", "prefix/one.txt")}, + }, + "token-2": { + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {output: finalListPage("prefix/two.txt")}, + {output: finalListPage("prefix/two.txt")}, + {output: finalListPage("prefix/two.txt")}, + }, + }} + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err != nil { + t.Fatalf("list pages: %v", err) + } + if got, want := keys(items), []string{"prefix/one.txt", "prefix/two.txt"}; strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("unexpected keys: got %v want %v", got, want) + } + if got := countString(client.calls, "token-2"); got != 5 { + t.Fatalf("expected page retry and terminal replays to reuse continuation token five times, got calls %q", strings.Join(client.calls, ",")) + } + if stats.Pages != 2 { + t.Fatalf("expected 2 successful pages, got %+v", stats) + } +} + +func TestListS3PrefixPagesRecoversObservedPageSeventeenInternalError(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "4") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{}} + for page := 1; page < 17; page++ { + token := "" + if page > 1 { + token = tokenForPage(page) + } + client.pages[token] = []fakeS3ListPage{ + {output: listPage(tokenForPage(page+1), "prefix/page-"+strconv.Itoa(page)+".txt")}, + } + } + client.pages[tokenForPage(17)] = []fakeS3ListPage{ + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {output: finalListPage("prefix/page-17.txt")}, + {output: finalListPage("prefix/page-17.txt")}, + {output: finalListPage("prefix/page-17.txt")}, + } + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err != nil { + t.Fatalf("list pages: %v", err) + } + if len(items) != 17 { + t.Fatalf("expected 17 inventory objects after recovered page 17, got %d", len(items)) + } + if stats.Pages != 17 || stats.Retries != 2 { + t.Fatalf("expected recovered page 17 with 2 retries, got %+v", stats) + } + if got := countString(client.calls, tokenForPage(17)); got != 5 { + t.Fatalf("expected page 17 token to be retried and replayed five times, got calls %v", client.calls) + } +} + +func TestListS3PrefixPagesExhaustsTransientRetriesWithProgress(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "3") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: listPage("token-2", "prefix/one.txt")}, + }, + "token-2": { + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + }, + }} + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err == nil { + t.Fatal("expected retry exhaustion error") + } + if len(items) != 1 || items[0].Key != "prefix/one.txt" { + t.Fatalf("expected collected page to be preserved for error reporting, got %+v", items) + } + if stats.FailedPage != 2 { + t.Fatalf("expected failed page 2, got %+v", stats) + } + if !strings.Contains(err.Error(), "page 2") || !strings.Contains(err.Error(), "after 1 objects and 3 attempts") { + t.Fatalf("expected progress in error, got %q", err.Error()) + } +} + +func TestListS3PrefixPagesDoesNotRetryAccessDenied(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "4") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {err: &smithy.GenericAPIError{Code: "AccessDenied", Message: "no"}}, + }, + }} + + _, _, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err == nil { + t.Fatal("expected access denied error") + } + if len(client.calls) != 1 { + t.Fatalf("expected no retries for access denied, got calls %v", client.calls) + } +} + +func TestListS3PrefixPagesErrorsOnMissingContinuationToken(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "1") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: &awss3.ListObjectsV2Output{IsTruncated: aws.Bool(true), Contents: []types.Object{{Key: aws.String("prefix/one.txt")}}}}, + }, + }} + + _, _, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err == nil || !strings.Contains(err.Error(), "without next continuation token") { + t.Fatalf("expected missing token error, got %v", err) + } +} + +func TestListS3PrefixPagesRetriesMalformedTruncatedPage(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "2") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: &awss3.ListObjectsV2Output{IsTruncated: aws.Bool(true)}}, + {output: finalListPage("prefix/one.txt")}, + {output: finalListPage("prefix/one.txt")}, + {output: finalListPage("prefix/one.txt")}, + }, + }} + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err != nil { + t.Fatalf("list pages: %v", err) + } + if len(items) != 1 || stats.Retries != 1 { + t.Fatalf("expected malformed page retry, items=%v stats=%+v", items, stats) + } +} + +func TestListS3PrefixPagesRetriesExactProbeMiss(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ExactProbeMaxAttempts, "2") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: finalListPage("prefix/other.txt")}, + {output: finalListPage("prefix/target.txt")}, + }, + }} + + items, _, _, err := listS3PrefixPagesWithExactProbeRetry(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/target.txt")}, "bucket", "prefix/target.txt", "prefix/target.txt", StoragePrefixListOptions{ExactPrefix: true, MaxKeys: 1}, false) + if err != nil { + t.Fatalf("exact probe: %v", err) + } + if len(items) != 1 || items[0].Key != "prefix/target.txt" { + t.Fatalf("expected exact probe retry to find target, got %v", items) + } +} + +func TestListS3PrefixPagesTerminalReplayUsesContinuationTokenOnly(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": {{output: listPage("token-2", "prefix/one.txt")}}, + "token-2": { + {output: finalListPage("prefix/two.txt")}, + {output: finalListPage("prefix/two.txt")}, + {output: finalListPage("prefix/two.txt")}, + }, + }} + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err != nil { + t.Fatalf("list pages: %v", err) + } + if got, want := strings.Join(keys(items), ","), "prefix/one.txt,prefix/two.txt"; got != want { + t.Fatalf("unexpected keys: got %q want %q", got, want) + } + if stats.TerminalReplayAttempts != 2 { + t.Fatalf("expected two terminal replays, got %+v", stats) + } + if len(client.startAfters) != 4 || containsNonEmpty(client.startAfters) { + t.Fatalf("expected no StartAfter requests, got %q", client.startAfters) + } +} + +func TestListS3PrefixPagesRejectsContradictoryTerminalReplay(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": {{output: listPage("token-2", "prefix/one.txt")}}, + "token-2": { + {output: finalListPage("prefix/two.txt")}, + {output: listPage("token-3", "prefix/two.txt")}, + }, + }} + + _, _, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err == nil || !strings.Contains(err.Error(), "terminal replay") { + t.Fatalf("expected contradictory terminal replay error, got %v", err) + } +} + +func listPage(nextToken string, keys ...string) *awss3.ListObjectsV2Output { + return &awss3.ListObjectsV2Output{ + IsTruncated: aws.Bool(true), + NextContinuationToken: aws.String(nextToken), + Contents: objects(keys...), + } +} + +func finalListPage(keys ...string) *awss3.ListObjectsV2Output { + return &awss3.ListObjectsV2Output{ + IsTruncated: aws.Bool(false), + Contents: objects(keys...), + } +} + +func objects(keys ...string) []types.Object { + out := make([]types.Object, 0, len(keys)) + for _, key := range keys { + out = append(out, types.Object{Key: aws.String(key), Size: aws.Int64(12)}) + } + return out +} + +func tokenForPage(page int) string { + return "token-" + strconv.Itoa(page) +} + +func keys(items []StorageBucketObject) []string { + out := make([]string, 0, len(items)) + for _, item := range items { + out = append(out, item.Key) + } + return out +} + +func countString(values []string, target string) int { + count := 0 + for _, value := range values { + if value == target { + count++ + } + } + return count +} + +func containsString(values []string, target string) bool { + return countString(values, target) > 0 +} + +func containsNonEmpty(values []string) bool { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return true + } + } + return false +} + +func disableS3ListRetrySleep(t *testing.T) func() { + t.Helper() + previous := sleepS3ListPageRetry + sleepS3ListPageRetry = func(context.Context, time.Duration) error { + return nil + } + return func() { + sleepS3ListPageRetry = previous + } +} diff --git a/internal/core/s3_probe_limiter.go b/internal/core/s3_probe_limiter.go new file mode 100644 index 00000000..625202b6 --- /dev/null +++ b/internal/core/s3_probe_limiter.go @@ -0,0 +1,71 @@ +package core + +import ( + "context" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/calypr/syfon/internal/common" +) + +const ( + defaultS3ProbeConcurrency = 8 + envS3ProbeConcurrency = "SYFON_S3_PROBE_CONCURRENCY" +) + +var s3ProbeLimiterContextKey contextKey = "s3ProbeLimiter" + +type s3ProbeLimiter struct { + permits chan struct{} +} + +func newS3ProbeLimiter(limit int) *s3ProbeLimiter { + if limit < 1 { + limit = defaultS3ProbeConcurrency + } + return &s3ProbeLimiter{permits: make(chan struct{}, limit)} +} + +func newS3ProbeLimiterFromEnv() *s3ProbeLimiter { + return newS3ProbeLimiter(s3ProbeConcurrencyFromEnv()) +} + +func s3ProbeConcurrencyFromEnv() int { + raw := strings.TrimSpace(os.Getenv(envS3ProbeConcurrency)) + if raw == "" { + return defaultS3ProbeConcurrency + } + limit, err := strconv.Atoi(raw) + if err != nil || limit < 1 { + return defaultS3ProbeConcurrency + } + return limit +} + +func withS3ProbeLimiter(ctx context.Context, limiter *s3ProbeLimiter) context.Context { + if limiter == nil || ctx.Value(s3ProbeLimiterContextKey) != nil { + return ctx + } + return context.WithValue(ctx, s3ProbeLimiterContextKey, limiter) +} + +func acquireS3Probe(ctx context.Context, operation, bucket, key string) (func(), error) { + limiter, _ := ctx.Value(s3ProbeLimiterContextKey).(*s3ProbeLimiter) + if limiter == nil { + return func() {}, nil + } + started := time.Now() + select { + case limiter.permits <- struct{}{}: + waited := time.Since(started) + if waited >= 100*time.Millisecond { + log.Printf("INFO: syfon_s3_probe_limiter_wait request_id=%s operation=%s bucket=%s key=%q wait_ms=%d", common.GetRequestID(ctx), operation, bucket, key, waited.Milliseconds()) + } + return func() { <-limiter.permits }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} diff --git a/internal/core/service.go b/internal/core/service.go index 901a0a2d..b525228c 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -38,6 +38,7 @@ type ObjectManager struct { bucketScopeCache *bucketScopeCache inspectS3Object func(context.Context, models.S3Credential, string, string) (*StorageObjectMetadata, error) listS3Prefix func(context.Context, models.S3Credential, string, string, StoragePrefixListOptions) ([]StorageBucketObject, error) + s3ProbeLimiter *s3ProbeLimiter } type VisibleBucket struct { @@ -52,5 +53,6 @@ func NewObjectManager(db db.DatabaseInterface, uM urlmanager.UrlManager) *Object bucketScopeCache: newBucketScopeCache(30 * time.Second), inspectS3Object: defaultS3ObjectInspector, listS3Prefix: defaultS3PrefixLister, + s3ProbeLimiter: newS3ProbeLimiterFromEnv(), } } From 08e36ebde29736c6bc2cc5a3f553da592c549886 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 10 Jul 2026 13:17:56 -0700 Subject: [PATCH 21/24] update go version to avoid vuln check issues --- Dockerfile | 2 +- go.mod | 2 +- go.work | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8692dbcf..d831c8ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.7 -FROM golang:1.26.4-alpine AS builder +FROM golang:1.26.5-alpine AS builder RUN apk add --no-cache build-base git WORKDIR /src diff --git a/go.mod b/go.mod index 5644d2ae..b62bddc5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/calypr/syfon -go 1.26.4 +go 1.26.5 require ( cloud.google.com/go/storage v1.62.1 diff --git a/go.work b/go.work index 269df952..fb0d2bbe 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.26.4 +go 1.26.5 use ( . From 2bd19a199e983083c663ab0599c35573bb4014ed Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 15 Jul 2026 10:31:32 -0700 Subject: [PATCH 22/24] add new route for bulk existance checks in git-drs --- apigen/client/internalapi/internal.gen.go | 155 ++++++++++++++++++++++ apigen/openapi/internal.openapi.yaml | 56 ++++++++ apigen/server/internalapi/internal.gen.go | 118 ++++++++++++++++ client/services/index_service.go | 14 ++ client/services/index_service_test.go | 34 +++++ internal/api/internaldrs/index.go | 60 ++++++++- internal/api/internaldrs/index_test.go | 53 ++++++++ internal/common/routes.go | 1 + internal/config/config.go | 15 ++- internal/core/object_reads.go | 27 ++++ internal/db/sqlite/objects.go | 20 +++ internal/db/sqlite/sqlite_test.go | 14 ++ 12 files changed, 558 insertions(+), 9 deletions(-) diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index 10f63bc1..9600b4ae 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -40,6 +40,19 @@ type BulkHashesRequest struct { Hashes []string `json:"hashes"` } +// BulkMissingSHA256Request defines model for BulkMissingSHA256Request. +type BulkMissingSHA256Request struct { + Organization string `json:"organization"` + Project string `json:"project"` + Sha256 []string `json:"sha256"` +} + +// BulkMissingSHA256Response defines model for BulkMissingSHA256Response. +type BulkMissingSHA256Response struct { + Checked int32 `json:"checked"` + MissingSha256 []string `json:"missing_sha256"` +} + // BulkSHA256ValidityRequest defines model for BulkSHA256ValidityRequest. type BulkSHA256ValidityRequest struct { Hashes *[]string `json:"hashes,omitempty"` @@ -338,6 +351,9 @@ type InternalBulkDocumentsJSONRequestBody = BulkDocumentsRequest // InternalBulkHashesJSONRequestBody defines body for InternalBulkHashes for application/json ContentType. type InternalBulkHashesJSONRequestBody = BulkHashesRequest +// InternalBulkMissingSHA256JSONRequestBody defines body for InternalBulkMissingSHA256 for application/json ContentType. +type InternalBulkMissingSHA256JSONRequestBody = BulkMissingSHA256Request + // InternalBulkSHA256ValidityJSONRequestBody defines body for InternalBulkSHA256Validity for application/json ContentType. type InternalBulkSHA256ValidityJSONRequestBody = BulkSHA256ValidityRequest @@ -560,6 +576,11 @@ type ClientInterface interface { InternalBulkHashes(ctx context.Context, body InternalBulkHashesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalBulkMissingSHA256WithBody request with any body + InternalBulkMissingSHA256WithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + InternalBulkMissingSHA256(ctx context.Context, body InternalBulkMissingSHA256JSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalBulkSHA256ValidityWithBody request with any body InternalBulkSHA256ValidityWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -942,6 +963,30 @@ func (c *Client) InternalBulkHashes(ctx context.Context, body InternalBulkHashes return c.Client.Do(req) } +func (c *Client) InternalBulkMissingSHA256WithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalBulkMissingSHA256RequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalBulkMissingSHA256(ctx context.Context, body InternalBulkMissingSHA256JSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalBulkMissingSHA256Request(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) InternalBulkSHA256ValidityWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewInternalBulkSHA256ValidityRequestWithBody(c.Server, contentType, body) if err != nil { @@ -2105,6 +2150,46 @@ func NewInternalBulkHashesRequestWithBody(server string, contentType string, bod return req, nil } +// NewInternalBulkMissingSHA256Request calls the generic InternalBulkMissingSHA256 builder with application/json body +func NewInternalBulkMissingSHA256Request(server string, body InternalBulkMissingSHA256JSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewInternalBulkMissingSHA256RequestWithBody(server, "application/json", bodyReader) +} + +// NewInternalBulkMissingSHA256RequestWithBody generates requests for InternalBulkMissingSHA256 with any type of body +func NewInternalBulkMissingSHA256RequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/index/bulk/sha256/missing") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewInternalBulkSHA256ValidityRequest calls the generic InternalBulkSHA256Validity builder with application/json body func NewInternalBulkSHA256ValidityRequest(server string, body InternalBulkSHA256ValidityJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -2428,6 +2513,11 @@ type ClientWithResponsesInterface interface { InternalBulkHashesWithResponse(ctx context.Context, body InternalBulkHashesJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalBulkHashesResponse, error) + // InternalBulkMissingSHA256WithBodyWithResponse request with any body + InternalBulkMissingSHA256WithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalBulkMissingSHA256Response, error) + + InternalBulkMissingSHA256WithResponse(ctx context.Context, body InternalBulkMissingSHA256JSONRequestBody, reqEditors ...RequestEditorFn) (*InternalBulkMissingSHA256Response, error) + // InternalBulkSHA256ValidityWithBodyWithResponse request with any body InternalBulkSHA256ValidityWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalBulkSHA256ValidityResponse, error) @@ -2846,6 +2936,28 @@ func (r InternalBulkHashesResponse) StatusCode() int { return 0 } +type InternalBulkMissingSHA256Response struct { + Body []byte + HTTPResponse *http.Response + JSON200 *BulkMissingSHA256Response +} + +// Status returns HTTPResponse.Status +func (r InternalBulkMissingSHA256Response) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalBulkMissingSHA256Response) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type InternalBulkSHA256ValidityResponse struct { Body []byte HTTPResponse *http.Response @@ -3213,6 +3325,23 @@ func (c *ClientWithResponses) InternalBulkHashesWithResponse(ctx context.Context return ParseInternalBulkHashesResponse(rsp) } +// InternalBulkMissingSHA256WithBodyWithResponse request with arbitrary body returning *InternalBulkMissingSHA256Response +func (c *ClientWithResponses) InternalBulkMissingSHA256WithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalBulkMissingSHA256Response, error) { + rsp, err := c.InternalBulkMissingSHA256WithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalBulkMissingSHA256Response(rsp) +} + +func (c *ClientWithResponses) InternalBulkMissingSHA256WithResponse(ctx context.Context, body InternalBulkMissingSHA256JSONRequestBody, reqEditors ...RequestEditorFn) (*InternalBulkMissingSHA256Response, error) { + rsp, err := c.InternalBulkMissingSHA256(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalBulkMissingSHA256Response(rsp) +} + // InternalBulkSHA256ValidityWithBodyWithResponse request with arbitrary body returning *InternalBulkSHA256ValidityResponse func (c *ClientWithResponses) InternalBulkSHA256ValidityWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalBulkSHA256ValidityResponse, error) { rsp, err := c.InternalBulkSHA256ValidityWithBody(ctx, contentType, body, reqEditors...) @@ -3747,6 +3876,32 @@ func ParseInternalBulkHashesResponse(rsp *http.Response) (*InternalBulkHashesRes return response, nil } +// ParseInternalBulkMissingSHA256Response parses an HTTP response from a InternalBulkMissingSHA256WithResponse call +func ParseInternalBulkMissingSHA256Response(rsp *http.Response) (*InternalBulkMissingSHA256Response, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalBulkMissingSHA256Response{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest BulkMissingSHA256Response + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseInternalBulkSHA256ValidityResponse parses an HTTP response from a InternalBulkSHA256ValidityWithResponse call func ParseInternalBulkSHA256ValidityResponse(rsp *http.Response) (*InternalBulkSHA256ValidityResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/openapi/internal.openapi.yaml b/apigen/openapi/internal.openapi.yaml index c7b1a34c..c4509d38 100644 --- a/apigen/openapi/internal.openapi.yaml +++ b/apigen/openapi/internal.openapi.yaml @@ -342,6 +342,34 @@ paths: description: Content too large '500': description: Internal server error + /index/bulk/sha256/missing: + post: + tags: + - internal-index + operationId: internalBulkMissingSHA256 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/BulkMissingSHA256Request' + responses: + '200': + description: SHA-256 values that are not registered in the requested project + content: + application/json: + schema: + $ref: '#/components/schemas/BulkMissingSHA256Response' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '413': + description: Content too large + '500': + description: Internal server error /index/bulk/documents: post: tags: @@ -826,6 +854,34 @@ components: type: array items: type: string + BulkMissingSHA256Request: + type: object + required: + - organization + - project + - sha256 + properties: + organization: + type: string + project: + type: string + sha256: + type: array + items: + type: string + BulkMissingSHA256Response: + type: object + required: + - checked + - missing_sha256 + properties: + checked: + type: integer + format: int32 + missing_sha256: + type: array + items: + type: string BulkDocumentsRequest: oneOf: - type: array diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 1a830c35..776b88fc 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -38,6 +38,19 @@ type BulkHashesRequest struct { Hashes []string `json:"hashes"` } +// BulkMissingSHA256Request defines model for BulkMissingSHA256Request. +type BulkMissingSHA256Request struct { + Organization string `json:"organization"` + Project string `json:"project"` + Sha256 []string `json:"sha256"` +} + +// BulkMissingSHA256Response defines model for BulkMissingSHA256Response. +type BulkMissingSHA256Response struct { + Checked int32 `json:"checked"` + MissingSha256 []string `json:"missing_sha256"` +} + // BulkSHA256ValidityRequest defines model for BulkSHA256ValidityRequest. type BulkSHA256ValidityRequest struct { Hashes *[]string `json:"hashes,omitempty"` @@ -336,6 +349,9 @@ type InternalBulkDocumentsJSONRequestBody = BulkDocumentsRequest // InternalBulkHashesJSONRequestBody defines body for InternalBulkHashes for application/json ContentType. type InternalBulkHashesJSONRequestBody = BulkHashesRequest +// InternalBulkMissingSHA256JSONRequestBody defines body for InternalBulkMissingSHA256 for application/json ContentType. +type InternalBulkMissingSHA256JSONRequestBody = BulkMissingSHA256Request + // InternalBulkSHA256ValidityJSONRequestBody defines body for InternalBulkSHA256Validity for application/json ContentType. type InternalBulkSHA256ValidityJSONRequestBody = BulkSHA256ValidityRequest @@ -464,6 +480,9 @@ type ServerInterface interface { // (POST /index/bulk/hashes) InternalBulkHashes(c fiber.Ctx) error + // (POST /index/bulk/sha256/missing) + InternalBulkMissingSHA256(c fiber.Ctx) error + // (POST /index/bulk/sha256/validity) InternalBulkSHA256Validity(c fiber.Ctx) error @@ -909,6 +928,12 @@ func (siw *ServerInterfaceWrapper) InternalBulkHashes(c fiber.Ctx) error { return siw.Handler.InternalBulkHashes(c) } +// InternalBulkMissingSHA256 operation middleware +func (siw *ServerInterfaceWrapper) InternalBulkMissingSHA256(c fiber.Ctx) error { + + return siw.Handler.InternalBulkMissingSHA256(c) +} + // InternalBulkSHA256Validity operation middleware func (siw *ServerInterfaceWrapper) InternalBulkSHA256Validity(c fiber.Ctx) error { @@ -1032,6 +1057,8 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Post(options.BaseURL+"/index/bulk/hashes", wrapper.InternalBulkHashes) + router.Post(options.BaseURL+"/index/bulk/sha256/missing", wrapper.InternalBulkMissingSHA256) + router.Post(options.BaseURL+"/index/bulk/sha256/validity", wrapper.InternalBulkSHA256Validity) router.Delete(options.BaseURL+"/index/:id", wrapper.InternalDelete) @@ -1993,6 +2020,63 @@ func (response InternalBulkHashes500Response) VisitInternalBulkHashesResponse(ct return nil } +type InternalBulkMissingSHA256RequestObject struct { + Body *InternalBulkMissingSHA256JSONRequestBody +} + +type InternalBulkMissingSHA256ResponseObject interface { + VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error +} + +type InternalBulkMissingSHA256200JSONResponse BulkMissingSHA256Response + +func (response InternalBulkMissingSHA256200JSONResponse) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalBulkMissingSHA256400Response struct { +} + +func (response InternalBulkMissingSHA256400Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalBulkMissingSHA256401Response struct { +} + +func (response InternalBulkMissingSHA256401Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalBulkMissingSHA256403Response struct { +} + +func (response InternalBulkMissingSHA256403Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalBulkMissingSHA256413Response struct { +} + +func (response InternalBulkMissingSHA256413Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(413) + return nil +} + +type InternalBulkMissingSHA256500Response struct { +} + +func (response InternalBulkMissingSHA256500Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + type InternalBulkSHA256ValidityRequestObject struct { Body *InternalBulkSHA256ValidityJSONRequestBody } @@ -2240,6 +2324,9 @@ type StrictServerInterface interface { // (POST /index/bulk/hashes) InternalBulkHashes(ctx context.Context, request InternalBulkHashesRequestObject) (InternalBulkHashesResponseObject, error) + // (POST /index/bulk/sha256/missing) + InternalBulkMissingSHA256(ctx context.Context, request InternalBulkMissingSHA256RequestObject) (InternalBulkMissingSHA256ResponseObject, error) + // (POST /index/bulk/sha256/validity) InternalBulkSHA256Validity(ctx context.Context, request InternalBulkSHA256ValidityRequestObject) (InternalBulkSHA256ValidityResponseObject, error) @@ -2809,6 +2896,37 @@ func (sh *strictHandler) InternalBulkHashes(ctx fiber.Ctx) error { return nil } +// InternalBulkMissingSHA256 operation middleware +func (sh *strictHandler) InternalBulkMissingSHA256(ctx fiber.Ctx) error { + var request InternalBulkMissingSHA256RequestObject + + var body InternalBulkMissingSHA256JSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalBulkMissingSHA256(ctx.Context(), request.(InternalBulkMissingSHA256RequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalBulkMissingSHA256") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalBulkMissingSHA256ResponseObject); ok { + if err := validResponse.VisitInternalBulkMissingSHA256Response(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + // InternalBulkSHA256Validity operation middleware func (sh *strictHandler) InternalBulkSHA256Validity(ctx fiber.Ctx) error { var request InternalBulkSHA256ValidityRequestObject diff --git a/client/services/index_service.go b/client/services/index_service.go index 6db2e888..95ae507b 100644 --- a/client/services/index_service.go +++ b/client/services/index_service.go @@ -195,6 +195,20 @@ func (s *IndexService) BulkSHA256Validity(ctx context.Context, req internalapi.B return *resp.JSON200, nil } +// MissingSHA256 returns the SHA-256 values that are not registered in a +// project. Syfon performs this as an indexed existence check and does not +// return complete DRS records. +func (s *IndexService) MissingSHA256(ctx context.Context, req internalapi.BulkMissingSHA256Request) (internalapi.BulkMissingSHA256Response, error) { + resp, err := s.gen.InternalBulkMissingSHA256WithResponse(ctx, internalapi.InternalBulkMissingSHA256JSONRequestBody(req)) + if err != nil { + return internalapi.BulkMissingSHA256Response{}, err + } + if resp.JSON200 == nil { + return internalapi.BulkMissingSHA256Response{}, fmt.Errorf("failed to find missing sha256 values: %d", resp.StatusCode()) + } + return *resp.JSON200, nil +} + func (s *IndexService) BulkDocuments(ctx context.Context, dids []string) ([]internalapi.InternalRecordResponse, error) { var body internalapi.BulkDocumentsRequest if err := body.FromBulkDocumentsRequest0(internalapi.BulkDocumentsRequest0(dids)); err != nil { diff --git a/client/services/index_service_test.go b/client/services/index_service_test.go index 57c9e885..8c5e18ae 100644 --- a/client/services/index_service_test.go +++ b/client/services/index_service_test.go @@ -289,6 +289,40 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { } } +func TestIndexServiceMissingSHA256(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/index/bulk/sha256/missing" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + var req internalapi.BulkMissingSHA256Request + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if req.Organization != "org" || req.Project != "project" || len(req.Sha256) != 2 { + t.Fatalf("unexpected request body: %+v", req) + } + writeJSON(t, w, http.StatusOK, internalapi.BulkMissingSHA256Response{ + Checked: 2, + MissingSha256: []string{"missing"}, + }) + })) + defer server.Close() + + service := NewIndexService(mustInternalClient(t, server.URL), &fakeRequester{}) + got, err := service.MissingSHA256(context.Background(), internalapi.BulkMissingSHA256Request{ + Organization: "org", + Project: "project", + Sha256: []string{"present", "missing"}, + }) + if err != nil { + t.Fatalf("MissingSHA256 returned error: %v", err) + } + if got.Checked != 2 || len(got.MissingSha256) != 1 || got.MissingSha256[0] != "missing" { + t.Fatalf("unexpected response: %+v", got) + } +} + func TestIndexServiceRemoveControlledAccessRequiresJSON200(t *testing.T) { t.Parallel() diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index 0d818a82..500b5250 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -1,6 +1,7 @@ package internaldrs import ( + "encoding/hex" "fmt" "log" "strconv" @@ -18,8 +19,9 @@ import ( ) const ( - defaultInternalListLimit = 1000 - maxInternalListLimit = 10000 + defaultInternalListLimit = 1000 + maxInternalListLimit = 10000 + maxInternalBulkMissingSHA256 = 10000 ) func RegisterInternalRoutes(router fiber.Router, om *core.ObjectManager) { @@ -36,6 +38,7 @@ func RegisterInternalRoutes(router fiber.Router, om *core.ObjectManager) { router.Post(common.RouteInternalBulkHashes, handleInternalBulkHashesFiber(om)) router.Post(common.RouteInternalBulkSHA256, handleInternalBulkSHA256ValidityFiber(om)) + router.Post(common.RouteInternalBulkSHA256Missing, handleInternalBulkMissingSHA256Fiber(om)) router.Post(common.RouteInternalBulkCreate, handleInternalBulkCreateFiber(om)) router.Post(common.RouteInternalBulkDocs, handleInternalBulkDocumentsFiber(om)) router.Post(common.RouteInternalBulkDeleteHashes, handleInternalBulkDeleteFiber(om)) @@ -45,6 +48,59 @@ func RegisterInternalRoutes(router fiber.Router, om *core.ObjectManager) { registerInternalTransferRoutes(router, om) } +func handleInternalBulkMissingSHA256Fiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + var req internalapi.BulkMissingSHA256Request + if err := c.Bind().JSON(&req); err != nil { + return c.Status(fiber.StatusBadRequest).SendString("Invalid request body") + } + if strings.TrimSpace(req.Organization) == "" || strings.TrimSpace(req.Project) == "" || len(req.Sha256) == 0 { + return c.Status(fiber.StatusBadRequest).SendString("Invalid request body: organization, project, and sha256 values are required") + } + + normalized, err := normalizeMissingSHA256(req.Sha256) + if err != nil { + return c.Status(fiber.StatusBadRequest).SendString(err.Error()) + } + if len(normalized) > maxInternalBulkMissingSHA256 { + return c.Status(fiber.StatusRequestEntityTooLarge).SendString(fmt.Sprintf("too many sha256 values: maximum is %d", maxInternalBulkMissingSHA256)) + } + + missing, err := om.ListMissingScopedSHA256(c.Context(), req.Organization, req.Project, normalized) + if err != nil { + return apiutil.HandleError(c, err) + } + return c.JSON(internalapi.BulkMissingSHA256Response{Checked: int32(len(normalized)), MissingSha256: missing}) + } +} + +func normalizeMissingSHA256(values []string) ([]string, error) { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, raw := range values { + value := strings.TrimSpace(raw) + value = strings.TrimPrefix(strings.ToLower(value), "sha256:") + if value == "" { + continue + } + if len(value) != 64 { + return nil, fmt.Errorf("invalid sha256 checksum %q", raw) + } + if _, err := hex.DecodeString(value); err != nil { + return nil, fmt.Errorf("invalid sha256 checksum %q", raw) + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + if len(out) == 0 { + return nil, fmt.Errorf("Invalid request body: sha256 values are required") + } + return out, nil +} + func handleInternalGetFiber(om *core.ObjectManager) fiber.Handler { return func(c fiber.Ctx) error { id := c.Params("id") diff --git a/internal/api/internaldrs/index_test.go b/internal/api/internaldrs/index_test.go index d465f570..bfb6de0f 100644 --- a/internal/api/internaldrs/index_test.go +++ b/internal/api/internaldrs/index_test.go @@ -791,6 +791,59 @@ func TestHandleInternalBulkSHA256Validity(t *testing.T) { } } +func TestHandleInternalBulkMissingSHA256(t *testing.T) { + present := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + missing := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + now := time.Now().UTC() + mockDB := &testutils.MockDatabase{Objects: map[string]*drs.DrsObject{ + "obj-sha": { + Id: "obj-sha", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: present}}, + }, + }, ObjectAuthz: map[string]map[string][]string{ + "obj-sha": {"org": {"project"}}, + }} + om := core.NewObjectManager(mockDB, &testutils.MockUrlManager{}) + app := fiber.New() + app.Post("/index/bulk/sha256/missing", handleInternalBulkMissingSHA256Fiber(om)) + req := httptest.NewRequest(http.MethodPost, "/index/bulk/sha256/missing", strings.NewReader(`{"organization":"org","project":"project","sha256":["SHA256:`+present+`","`+missing+`","`+missing+`"]}`)) + req.Header.Set("Content-Type", "application/json") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 200, got %d body=%s", resp.StatusCode, string(body)) + } + var payload internalapi.BulkMissingSHA256Response + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if payload.Checked != 2 || !slices.Equal(payload.MissingSha256, []string{missing}) { + t.Fatalf("unexpected response: %+v", payload) + } +} + +func TestHandleInternalBulkMissingSHA256RejectsInvalidChecksum(t *testing.T) { + om := core.NewObjectManager(&testutils.MockDatabase{}, &testutils.MockUrlManager{}) + app := fiber.New() + app.Post("/index/bulk/sha256/missing", handleInternalBulkMissingSHA256Fiber(om)) + req := httptest.NewRequest(http.MethodPost, "/index/bulk/sha256/missing", strings.NewReader(`{"organization":"org","project":"project","sha256":["not-a-sha256"]}`)) + req.Header.Set("Content-Type", "application/json") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400, got %d", resp.StatusCode) + } +} + func TestHandleInternalCreate_PersistsControlledAccess(t *testing.T) { mockDB := &testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}} reqBody := `{"records":[{"did":"obj-1","size":42,"controlled_access":["https://calypr.org/program/test/project/p1"],"access_methods":[{"type":"s3","access_url":{"url":"s3://bucket/path/obj-1"}}]}]}` diff --git a/internal/common/routes.go b/internal/common/routes.go index 98940825..feaac418 100644 --- a/internal/common/routes.go +++ b/internal/common/routes.go @@ -30,6 +30,7 @@ const ( RouteInternalBulkHashes = "/index/bulk/hashes" RouteInternalBulkDeleteHashes = "/index/bulk/delete" RouteInternalBulkSHA256 = "/index/bulk/sha256/validity" + RouteInternalBulkSHA256Missing = "/index/bulk/sha256/missing" RouteInternalBulkCreate = "/index/bulk" RouteInternalBulkDocs = "/index/bulk/documents" ) diff --git a/internal/config/config.go b/internal/config/config.go index a0bef340..918c0df5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -59,13 +59,14 @@ const ( RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" // Internal DRS Index - RouteInternalIndex = "/index" - RouteInternalIndexDetail = "/index/{id}" - RouteInternalBulkHashes = "/index/bulk/hashes" - RouteInternalBulkDeleteHashes = "/index/bulk/delete" - RouteInternalBulkSHA256 = "/index/bulk/sha256/validity" - RouteInternalBulkCreate = "/index/bulk" - RouteInternalBulkDocs = "/index/bulk/documents" + RouteInternalIndex = "/index" + RouteInternalIndexDetail = "/index/{id}" + RouteInternalBulkHashes = "/index/bulk/hashes" + RouteInternalBulkDeleteHashes = "/index/bulk/delete" + RouteInternalBulkSHA256 = "/index/bulk/sha256/validity" + RouteInternalBulkSHA256Missing = "/index/bulk/sha256/missing" + RouteInternalBulkCreate = "/index/bulk" + RouteInternalBulkDocs = "/index/bulk/documents" // Core API RouteCoreSHA256 = "/index/v1/sha256/validity" diff --git a/internal/core/object_reads.go b/internal/core/object_reads.go index 90bf8d51..b74a4f0d 100644 --- a/internal/core/object_reads.go +++ b/internal/core/object_reads.go @@ -389,6 +389,33 @@ func (m *ObjectManager) ListObjectsByScope(ctx context.Context, organization, pr return m.PrepareScopedObjects(ctx, objects, organization, project, requiredMethod) } +// ListMissingScopedSHA256 returns the requested SHA-256 checksums that are not +// registered for the given project. It deliberately uses the indexed checksum +// lookup and does not hydrate complete DRS records or access methods. +func (m *ObjectManager) ListMissingScopedSHA256(ctx context.Context, organization, project string, checksums []string) ([]string, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" || project == "" || len(checksums) == 0 { + return nil, common.ErrUnauthorized + } + if err := m.requireScopeMethod(ctx, organization, project, objectMethodRead); err != nil { + return nil, err + } + + existingByChecksum, err := m.db.ListScopedObjectIDsByChecksums(ctx, organization, project, checksums) + if err != nil { + return nil, err + } + + missing := make([]string, 0, len(checksums)) + for _, checksum := range checksums { + if objectIDs := existingByChecksum[checksum]; len(objectIDs) == 0 { + missing = append(missing, checksum) + } + } + return missing, nil +} + func (m *ObjectManager) expandProjectChecksumSiblingObjects(ctx context.Context, objects []models.InternalObject, organization, project string) ([]models.InternalObject, error) { if len(objects) == 0 { return []models.InternalObject{}, nil diff --git a/internal/db/sqlite/objects.go b/internal/db/sqlite/objects.go index 09865cd2..1a1750c6 100644 --- a/internal/db/sqlite/objects.go +++ b/internal/db/sqlite/objects.go @@ -489,6 +489,26 @@ func (db *SqliteDB) ListScopedObjectIDsByChecksums(ctx context.Context, organiza if len(normalized) == 0 { return map[string][]string{}, nil } + // Keep the checksum predicate below within SQLite's bound-parameter limit. + // The two fixed parameters are the project resource and checksum type. + maxChecksums := sqliteMaxParams - 2 + if len(normalized) > maxChecksums { + out := make(map[string][]string, len(normalized)) + for start := 0; start < len(normalized); start += maxChecksums { + end := start + maxChecksums + if end > len(normalized) { + end = len(normalized) + } + part, err := db.ListScopedObjectIDsByChecksums(ctx, organization, project, normalized[start:end]) + if err != nil { + return nil, err + } + for checksum, ids := range part { + out[checksum] = append(out[checksum], ids...) + } + } + return out, nil + } args := make([]any, 0, len(normalized)+2) args = append(args, resource, "sha256") placeholders := makePlaceholders(len(normalized)) diff --git a/internal/db/sqlite/sqlite_test.go b/internal/db/sqlite/sqlite_test.go index 1e594cc5..51f4b6f7 100644 --- a/internal/db/sqlite/sqlite_test.go +++ b/internal/db/sqlite/sqlite_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "path/filepath" "slices" "strings" @@ -999,6 +1000,19 @@ func TestSqliteDB_ListScopedObjectIDsByChecksums(t *testing.T) { if len(emptyRes) != 0 { t.Fatalf("expected empty map for empty checksum input, got %+v", emptyRes) } + + large := make([]string, sqliteMaxParams) + for i := range large { + large[i] = fmt.Sprintf("large-%d", i) + } + large[len(large)-1] = "sha-b" + largeRes, err := db.ListScopedObjectIDsByChecksums(ctx, "org", "p1", large) + if err != nil { + t.Fatalf("ListScopedObjectIDsByChecksums large input failed: %v", err) + } + if got := largeRes["sha-b"]; len(got) != 1 || got[0] != "proj-b" { + t.Fatalf("unexpected large-input result for sha-b: %+v", got) + } } func TestSqliteDB_ListObjectIDsByScopeRootIncludesUnscoped(t *testing.T) { From 1800011696073ca17af857a8f1bcfafaae014b8a Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 15 Jul 2026 11:03:05 -0700 Subject: [PATCH 23/24] update all the deps --- go.mod | 149 +++++----- go.sum | 162 +++++++++++ go.work.sum | 775 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1012 insertions(+), 74 deletions(-) diff --git a/go.mod b/go.mod index b62bddc5..46075780 100644 --- a/go.mod +++ b/go.mod @@ -3,66 +3,66 @@ module github.com/calypr/syfon go 1.26.5 require ( - cloud.google.com/go/storage v1.62.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 + cloud.google.com/go/storage v1.63.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.32.14 - github.com/aws/aws-sdk-go-v2/credentials v1.19.14 - github.com/aws/aws-sdk-go-v2/service/kms v1.50.4 - github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 - github.com/aws/smithy-go v1.24.3 + github.com/aws/aws-sdk-go-v2 v1.42.1 + github.com/aws/aws-sdk-go-v2/config v1.32.30 + github.com/aws/aws-sdk-go-v2/credentials v1.19.29 + github.com/aws/aws-sdk-go-v2/service/kms v1.54.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1 + github.com/aws/smithy-go v1.27.3 github.com/calypr/syfon/apigen v0.2.8 github.com/calypr/syfon/client v0.3.0 - github.com/getkin/kin-openapi v0.135.0 - github.com/gofiber/fiber/v3 v3.1.0 + github.com/getkin/kin-openapi v0.142.0 + github.com/gofiber/fiber/v3 v3.4.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 - github.com/hashicorp/go-plugin v1.7.0 + github.com/hashicorp/go-plugin v1.8.0 github.com/lib/pq v1.12.3 - github.com/mattn/go-sqlite3 v1.14.42 + github.com/mattn/go-sqlite3 v1.14.48 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/testcontainers/testcontainers-go v0.42.0 - gocloud.dev v0.45.0 - google.golang.org/api v0.276.0 + gocloud.dev v0.46.0 + google.golang.org/api v0.288.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.2 // indirect cloud.google.com/go v0.123.0 // indirect - cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth v0.22.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.9.0 // indirect - cloud.google.com/go/monitoring v1.27.0 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect dario.cat/mergo v1.0.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/andybalholm/brotli v1.2.0 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.4.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.32.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.44.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect @@ -79,33 +79,33 @@ require ( github.com/ebitengine/purego v0.10.0 // indirect github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect - github.com/fatih/color v1.16.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/jsonpointer v0.22.5 // indirect - github.com/go-openapi/swag/jsonname v0.25.5 // indirect - github.com/gofiber/schema v1.7.0 // indirect - github.com/gofiber/utils/v2 v2.0.2 // indirect + github.com/go-openapi/jsonpointer v1.0.0 // indirect + github.com/go-openapi/swag/jsonname v0.27.0 // indirect + github.com/gofiber/schema v1.8.2 // indirect + github.com/gofiber/utils/v2 v2.2.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect - github.com/googleapis/gax-go/v2 v2.21.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/yamux v0.1.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.19.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mailru/easyjson v0.9.2 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.21 // indirect - github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-runewidth v0.0.24 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect github.com/moby/moby/api v1.54.1 // indirect @@ -116,10 +116,10 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect - github.com/oapi-codegen/runtime v1.4.0 // indirect - github.com/oasdiff/yaml v0.0.9 // indirect - github.com/oasdiff/yaml3 v0.0.9 // indirect - github.com/oklog/run v1.1.0 // indirect + github.com/oapi-codegen/runtime v1.5.0 // indirect + github.com/oasdiff/yaml v0.1.1 // indirect + github.com/oasdiff/yaml3 v0.0.14 // indirect + github.com/oklog/run v1.2.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect @@ -127,39 +127,40 @@ require ( github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/stretchr/testify v1.11.1 // indirect - github.com/tinylib/msgp v1.6.3 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.69.0 // indirect - github.com/vbauerster/mpb/v8 v8.12.0 // indirect + github.com/valyala/fasthttp v1.72.0 // indirect + github.com/vbauerster/mpb/v8 v8.12.1 // indirect github.com/woodsbury/decimal128 v1.4.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect - golang.org/x/crypto v0.51.0 // indirect - golang.org/x/net v0.55.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/grpc v1.80.0 // indirect + google.golang.org/genproto v0.0.0-20260713224248-f5fc221cf8c4 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260713224248-f5fc221cf8c4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260713224248-f5fc221cf8c4 // indirect + google.golang.org/grpc v1.82.0 // indirect google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/ini.v1 v1.67.1 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect ) diff --git a/go.sum b/go.sum index e8842c0e..efd7da46 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,48 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ= +cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute v1.64.0 h1:7MmuzeAxlG5MOG5PQD2NLtyYR6bWjkvGljRu7pByoRU= +cloud.google.com/go/compute v1.64.0/go.mod h1:eHhcRZ6vf70fQCS3VEsiWSh+nQ+tLvSMb7mwLQskgN0= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/iam v1.9.0 h1:89wyjxT6DL4b5rk/Nk8eBC9DHqf+JiMstrn5IEYxFw4= cloud.google.com/go/iam v1.9.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= +cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= cloud.google.com/go/logging v1.15.0 h1:6ooUEBNT6jdWh2b36+iuPn6b/R9qN/tHCbvGS5255gg= cloud.google.com/go/logging v1.15.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= +cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k= cloud.google.com/go/longrunning v0.10.0 h1:4OWvp1BjCvoeSZTog3sRFDu6j4IrI9TI4/Y9N+8h25g= cloud.google.com/go/longrunning v0.10.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= cloud.google.com/go/monitoring v1.27.0 h1:BhYwMqao+e5Nn7JtWMM9m6zRtKtVUK6kJWMizXChkLU= cloud.google.com/go/monitoring v1.27.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= cloud.google.com/go/storage v1.62.1 h1:Os0G3XbUbjZumkpDUf2Y0rLoXJTCF1kU2kWUujKYXD8= cloud.google.com/go/storage v1.62.1/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= +cloud.google.com/go/storage v1.63.1 h1:CYXILV9G4CH0C18IQ9+V0h4XiqD2LhKnMLO0o7uJWNs= +cloud.google.com/go/storage v1.63.1/go.mod h1:lWyAtwvDZHdL3k68WVKbESP6bmWaV23ZJJ/JEVw/ZaQ= cloud.google.com/go/trace v1.13.0 h1:RfqsqPOiSCG8ql50UZt5F65KrVa1zbY9mJrO7xvZfbE= cloud.google.com/go/trace v1.13.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= @@ -34,20 +51,30 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1 h1:edShSHV3DV90+kt+CMaEXEzR9QF7wFrPJxVGz2blMIU= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 h1:yzIYdwuro811Z27D3T80Wkd3rqZzb0K43nner7Eh1yE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 h1:O2sXMyJh8b7devAGdE+163xtRurt0RVpB6DIzX5vGfg= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0/go.mod h1:hEpiGU18xf70qb3jbTcIggWAiEfX/cOIVc2OTe4OegA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0 h1:ZYGajzJNcirVZpT1rltgf9iM+j9zZ4v8V9DrF+xKRJ8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0/go.mod h1:PDQyYBOzGtQgvshQI//UiXyzuMHCz0ndyu+4W8X82vM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0 h1:ZIT85vKP7LBS84XJ0WdJ3dPOX3iz4j3c0+lpajGQMyo= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0/go.mod h1:rqP9UEhOXv9WhQ7Gjz+G5y/pf8+BJZW5/Ts0AhE0PwE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.58.0 h1:IBF8BbhKJkMsON/eY+LMu3aF3XMiotCb9KvkUmEkOJo= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 h1:0YP0+/ixwu+Uqeu/FGiBZNQ19huiUxxiPXIc9WsLKuQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0/go.mod h1:6ZZMQhZKDvUvkJw2rc+oDP90tMMzuU/J+5HG1ZmPOmE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 h1:SBZzZCiPmDrUV7NSCWY54OnKikO/oTydPCvyEyYaDDE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= @@ -57,50 +84,90 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek= +github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= +github.com/aws/aws-sdk-go-v2/config v1.32.30 h1:XwsEzpTJfQYJbFicz/QMLwAZdyeNVVoOEkbF7R3gPJk= +github.com/aws/aws-sdk-go-v2/config v1.32.30/go.mod h1:Ud32SuMc+/9BGxfpSVld7HrE2o05JwKmXY4M3jOQNZU= github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29 h1:WHZGssHH887cO0ox07SIQZsFx3MKD4ps6w0xUEmnKYQ= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29/go.mod h1:Mhl0xR6zjguiuj00XRx2wMx22sAltk7oya39sT7fdg8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I= github.com/aws/aws-sdk-go-v2/service/kms v1.50.4 h1:PgD1y0ZagPokGIZPmejCBUySBzOFDN+leZxCOfb1OEQ= github.com/aws/aws-sdk-go-v2/service/kms v1.50.4/go.mod h1:FfXDb5nXrsoGgxsBFxwxr3vdHXheC2tV+6lmuLghhjQ= +github.com/aws/aws-sdk-go-v2/service/kms v1.54.1 h1:aeJAJyvWS3gQ679pJbz8ZdOh3MViD1zvEdoZMVEawbg= +github.com/aws/aws-sdk-go-v2/service/kms v1.54.1/go.mod h1:0RXNc6Yf3AvSMldGD6Lcch96Ojlw2TtGnHsqfD/L4u8= github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1 h1:LkBKxAOE5WXjlFuFZqPG1rREnl6I6QCMElcXFDEidos= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do= github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= +github.com/aws/aws-sdk-go-v2/service/signin v1.4.1 h1:V7ZZ300WPXGjvkyore5DGe0ljVPOxCXie/thWdtSBXE= +github.com/aws/aws-sdk-go-v2/service/signin v1.4.1/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg= github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/sso v1.32.1 h1:gYFYh4iLLcAOJRLNPY2aD2g9DIhKn4eof8UkIrr1rTk= +github.com/aws/aws-sdk-go-v2/service/sso v1.32.1/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1 h1:arjT9Cm3/WYbGmD5TUZHk4UQn4Lle1fUNZs5FC6CtF0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= +github.com/aws/aws-sdk-go-v2/service/sts v1.44.1 h1:RvfHDg+xvAeZ+5741vUEjpOVtYSIm93W2zhx10Xtydw= +github.com/aws/aws-sdk-go-v2/service/sts v1.44.1/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q= github.com/aws/smithy-go v1.24.3 h1:XgOAaUgx+HhVBoP4v8n6HCQoTRDhoMghKqw4LNHsDNg= github.com/aws/smithy-go v1.24.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= @@ -152,12 +219,19 @@ github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDD github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78= github.com/getkin/kin-openapi v0.135.0 h1:751SjYfbiwqukYuVjwYEIKNfrSwS5YpA7DZnKSwQgtg= github.com/getkin/kin-openapi v0.135.0/go.mod h1:6dd5FJl6RdX4usBtFBaQhk9q62Yb2J0Mk5IhUO/QqFI= +github.com/getkin/kin-openapi v0.142.0 h1:izj0vBdFprMhitfzaX8sTqztsEQyvwhssBoB6n8NO7w= +github.com/getkin/kin-openapi v0.142.0/go.mod h1:3BH9M9XDe/y9M5DSvEocVYAYq1w0qrhJHjC/vZi0AaY= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -169,18 +243,29 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= +github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= +github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= +github.com/go-openapi/swag/jsonname v0.27.0 h1:4QVB//CKOdE8IOiBg19JNY2wfDS48MhesIquYBy2rUE= +github.com/go-openapi/swag/jsonname v0.27.0/go.mod h1:I1YsyvvhBuZsFXSW6I7ODfdyq13p7hDil//1T9/pFFk= github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gofiber/fiber/v3 v3.1.0 h1:1p4I820pIa+FGxfwWuQZ5rAyX0WlGZbGT6Hnuxt6hKY= github.com/gofiber/fiber/v3 v3.1.0/go.mod h1:n2nYQovvL9z3Too/FGOfgtERjW3GQcAUqgfoezGBZdU= +github.com/gofiber/fiber/v3 v3.4.0 h1:F0aND4vwZF7dR7cbvSwFQQEpBU902XHKWxrLsFBkVqw= +github.com/gofiber/fiber/v3 v3.4.0/go.mod h1:nAhJfdxUIJJph2tPWPmqWf8QDIN2iiqQiQf3lENZpdk= github.com/gofiber/schema v1.7.0 h1:yNM+FNRZjyYEli9Ey0AXRBrAY9jTnb+kmGs3lJGPvKg= github.com/gofiber/schema v1.7.0/go.mod h1:A/X5Ffyru4p9eBdp99qu+nzviHzQiZ7odLT+TwxWhbk= +github.com/gofiber/schema v1.8.2 h1:wq+LO2xEGlsqma/8Akp9PUebQ6vcsYmF0xYQ4F2ijvU= +github.com/gofiber/schema v1.8.2/go.mod h1:iyAMJztdyky7Pk2U7bwUP3EHjzMqUNjZ/hglE0nYO/g= github.com/gofiber/utils/v2 v2.0.2 h1:ShRRssz0F3AhTlAQcuEj54OEDtWF7+HJDwEi/aa6QLI= github.com/gofiber/utils/v2 v2.0.2/go.mod h1:+9Ub4NqQ+IaJoTliq5LfdmOJAA/Hzwf4pXOxOa3RrJ0= +github.com/gofiber/utils/v2 v2.2.0 h1:YSSmCzQponq/f9uSOg2HtXC5qK1Dmor0o6DqaQVz8GE= +github.com/gofiber/utils/v2 v2.2.0/go.mod h1:Ieopk6sQh7rbhQ12aBNCJtJuG0gxAg0nz63sFCrrOmE= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -198,8 +283,12 @@ github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= +github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl/wMbiI= github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -208,6 +297,8 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-plugin v1.7.0 h1:YghfQH/0QmPNc/AZMTFE3ac8fipZyZECHdDPshfk+mA= github.com/hashicorp/go-plugin v1.7.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= +github.com/hashicorp/go-plugin v1.8.0 h1:ie8S6RRY8RvB2usYZv+AAZ/wBvx2AU5p5QeP5j/FORs= +github.com/hashicorp/go-plugin v1.8.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= @@ -222,6 +313,8 @@ github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPci github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -240,14 +333,22 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= +github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/mattn/go-sqlite3 v1.14.48 h1:7XHIgl0a8HwOaiK4E47ozLkST78rR9+OtNGx27D/TFs= +github.com/mattn/go-sqlite3 v1.14.48/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= @@ -270,12 +371,20 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/oapi-codegen/runtime v1.4.0 h1:KLOSFOp7UzkbS7Cs1ms6NBEKYr0WmH2wZG0KKbd2er4= github.com/oapi-codegen/runtime v1.4.0/go.mod h1:5sw5fxCDmnOzKNYmkVNF8d34kyUeejJEY8HNT2WaPec= +github.com/oapi-codegen/runtime v1.5.0 h1:aiil4QnH+eiWYSO60eaYZ4aur7sJH3rz6BvT5EBFnxc= +github.com/oapi-codegen/runtime v1.5.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= github.com/oasdiff/yaml v0.0.9 h1:zQOvd2UKoozsSsAknnWoDJlSK4lC0mpmjfDsfqNwX48= github.com/oasdiff/yaml v0.0.9/go.mod h1:8lvhgJG4xiKPj3HN5lDow4jZHPlx1i7dIwzkdAo6oAM= +github.com/oasdiff/yaml v0.1.1 h1:6nHx+pn9gBRM6YpBlFZFQGCCd1nuvqOBtTD3KKTgGxY= +github.com/oasdiff/yaml v0.1.1/go.mod h1:EYJNoyktvWMJ0Hmhx+6qTaqMOsalUaRGT8Sj1hNcegU= github.com/oasdiff/yaml3 v0.0.9 h1:rWPrKccrdUm8J0F3sGuU+fuh9+1K/RdJlWF7O/9yw2g= github.com/oasdiff/yaml3 v0.0.9/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= +github.com/oasdiff/yaml3 v0.0.14 h1:aLJee3hxBK2H5wdXd9iPcIXb93Nty1Ge0pT171eHtkw= +github.com/oasdiff/yaml3 v0.0.14/go.mod h1:csto2xfDjYccdUn/yw/bPjj/cYTdp6HtFA0J4TWG+gg= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/oklog/run v1.2.0 h1:O8x3yXwah4A73hJdlrwo/2X6J62gE5qTMusH0dvz60E= +github.com/oklog/run v1.2.0/go.mod h1:mgDbKRSwPhJfesJ4PntqFUbKQRZ50NgmZTSPlFA0YFk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -296,8 +405,11 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:Om github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/shamaton/msgpack/v3 v3.1.0 h1:jsk0vEAqVvvS9+fTZ5/EcQ9tz860c9pWxJ4Iwecz8gU= github.com/shamaton/msgpack/v3 v3.1.0/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc= +github.com/shamaton/msgpack/v3 v3.2.0 h1:1q2Ms+MWmuRju+PuDMSFDB7p7621npeX4zprJN5Zck8= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= @@ -309,6 +421,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -327,6 +441,8 @@ github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44Xt github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= @@ -337,8 +453,12 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI= github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw= +github.com/valyala/fasthttp v1.72.0 h1:R7kYdoWhn1ye1fVpP+cDHDJwYm3NkwLliwgzJ/Abg7M= +github.com/valyala/fasthttp v1.72.0/go.mod h1:zsbLTYqcpIktdQytlVBwIjY9La5d6bs990nBxWg8efk= github.com/vbauerster/mpb/v8 v8.12.0 h1:+gneY3ifzc88tKDzOtfG8k8gfngCx615S2ZmFM4liWg= github.com/vbauerster/mpb/v8 v8.12.0/go.mod h1:V02YIuMVo301Y1VE9VtZlD8s84OMsk+EKN6mwvf/588= +github.com/vbauerster/mpb/v8 v8.12.1 h1:pyj3yQ2ZGQJgUXm4h17QpR+eERaNz5OQ1ftPSEE/sMM= +github.com/vbauerster/mpb/v8 v8.12.1/go.mod h1:XLXRfStkw/6i5k0aQltijDHT1Z93fD1DVwmIdcFUp6k= github.com/woodsbury/decimal128 v1.4.0 h1:xJATj7lLu4f2oObouMt2tgGiElE5gO6mSWUjQsBgUlc= github.com/woodsbury/decimal128 v1.4.0/go.mod h1:BP46FUrVjVhdTbKT+XuQh2xfQaGki9LMIRJSFuh6THU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -351,33 +471,58 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0/go.mod h1:D7J12YRapIekYyPWgGPlA/23pRmpSEZC5xJC/TTLI9U= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0 h1:hqxVTu/GtBF+vJ8d1fzW7fRxZFvgoDjWcxwwCaFDYpU= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY= gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI= +gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= +gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -388,10 +533,15 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -401,14 +551,24 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.276.0 h1:nVArUtfLEihtW+b0DdcqRGK1xoEm2+ltAihyztq7MKY= google.golang.org/api v0.276.0/go.mod h1:Fnag/EWUPIcJXuIkP1pjoTgS5vdxlk3eeemL7Do6bvw= +google.golang.org/api v0.288.0 h1:glhO/J88obKP5I269W3hB73dvBKrjU56ZfmNlNXpgTU= +google.golang.org/api v0.288.0/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= google.golang.org/genproto v0.0.0-20260414002931-afd174a4e478 h1:aLsVTW0lZ8+IY5u/ERjZSCvAmhuR7slKzyha3YikDNA= google.golang.org/genproto v0.0.0-20260414002931-afd174a4e478/go.mod h1:YJAzKjfHIUHb9T+bfu8L7mthAp7VVXQBUs1PLdBWS7M= +google.golang.org/genproto v0.0.0-20260713224248-f5fc221cf8c4 h1:PWVFocb6Y93aqxY7m9ArHu5zQiLkIrXBLr+ajQxRHH0= +google.golang.org/genproto v0.0.0-20260713224248-f5fc221cf8c4/go.mod h1:v47/Kzm6KzB/2uGxuDjGww+7Dk7ny/VFIaIw4WxEtu0= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/api v0.0.0-20260713224248-f5fc221cf8c4 h1:lI0NbdWVmT6lOJJNDd7vyeTdfxP/7ouCLSJUKNNXa0k= +google.golang.org/genproto/googleapis/api v0.0.0-20260713224248-f5fc221cf8c4/go.mod h1:WRrQ7/7N19PypuT0fxLOL5Lq0waoiRri4FbtHDEKrGE= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260713224248-f5fc221cf8c4 h1:7RtFDizMtT9eZzHzKxifoMGfcDBBy+LYZlgfg24ZmOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260713224248-f5fc221cf8c4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -416,6 +576,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work.sum b/go.work.sum index df6e708c..be41eb9a 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,251 +1,445 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.82.0/go.mod h1:vlKccHJGuFBFufnAnuB08dfEH9Y3H7dzDzRECFdC2TA= cloud.google.com/go/accessapproval v1.10.0 h1:AkUX4ayuxplE7ZSbya9bBvXFoRu8E9bJYmulNVcex6I= cloud.google.com/go/accessapproval v1.10.0/go.mod h1:7bmInw17bQX+ZPi7YmReC3xKymDrMmxXaUnaI6zQOqI= +cloud.google.com/go/accessapproval v1.13.0/go.mod h1:7bmInw17bQX+ZPi7YmReC3xKymDrMmxXaUnaI6zQOqI= cloud.google.com/go/accesscontextmanager v1.11.0 h1:41YvtDKbyIVwhpzDUJ7TI8Ca3BRACvT4cPQSeg+lXsM= cloud.google.com/go/accesscontextmanager v1.11.0/go.mod h1:VO15iVnsM0FO9Dt8hSFPgkuHRZjq6LEYZq1szJ27U2k= +cloud.google.com/go/accesscontextmanager v1.14.0/go.mod h1:VO15iVnsM0FO9Dt8hSFPgkuHRZjq6LEYZq1szJ27U2k= cloud.google.com/go/aiplatform v1.123.0 h1:Gh/SjaxI5wu/jeYQPPn8m+iGECBxlvsoyCWv8MqJ6xs= cloud.google.com/go/aiplatform v1.123.0/go.mod h1:yWTZiCunYDnyxeWWD14tDo6+BMlvAUCC5VxuxhvbrVI= +cloud.google.com/go/aiplatform v1.125.0/go.mod h1:yWTZiCunYDnyxeWWD14tDo6+BMlvAUCC5VxuxhvbrVI= cloud.google.com/go/analytics v0.32.0 h1:dGhOv5gmWi0sq/5oQ9Ojfx/R8daCAjv5CuvxVU0Zs8g= cloud.google.com/go/analytics v0.32.0/go.mod h1:V9Qef2N0y8GDqQ9FTlmM2XpDEMYonZJRPSUNGZlPCcc= +cloud.google.com/go/analytics v0.36.0/go.mod h1:q/KfbXopU5Ad7LThQDrcx/B6A6kAQNhQU7zX4gD+JLQ= cloud.google.com/go/apigateway v1.9.0 h1:7wOtsGpsfgbUNMfw1D4CHpz6oLchn4vWxoVi2uAjW2Y= cloud.google.com/go/apigateway v1.9.0/go.mod h1:f3Sk8Tdh1Ty5HR7kgbWB6Yu1M82LM+nIr5DTMZnLZWk= +cloud.google.com/go/apigateway v1.12.0/go.mod h1:f3Sk8Tdh1Ty5HR7kgbWB6Yu1M82LM+nIr5DTMZnLZWk= cloud.google.com/go/apigeeconnect v1.9.0 h1:7Dicdaf2uRZnxYxgcgyVY4IIs0rlj81TNnd5Z+Gt92M= cloud.google.com/go/apigeeconnect v1.9.0/go.mod h1:mYJekCKZHc2ia5yZX5lwtexTn9CzsOfb6+sh/2hi42Q= +cloud.google.com/go/apigeeconnect v1.12.0/go.mod h1:mYJekCKZHc2ia5yZX5lwtexTn9CzsOfb6+sh/2hi42Q= cloud.google.com/go/apigeeregistry v0.12.0 h1:7D4E+Gf1wV5GAo9VsmWnwGX2MFprjjDYMe9ItT4xmg8= cloud.google.com/go/apigeeregistry v0.12.0/go.mod h1:o+j6eA8hYhTWX5gEqMMBVDWY+/QQFrYe/YJBsO19pn0= +cloud.google.com/go/apigeeregistry v1.0.0/go.mod h1:o+j6eA8hYhTWX5gEqMMBVDWY+/QQFrYe/YJBsO19pn0= cloud.google.com/go/appengine v1.11.0 h1:BGf1Ig76ZB5m11sJGLkcMrx9LZ8YkYWQVk9vD+kV8KA= cloud.google.com/go/appengine v1.11.0/go.mod h1:JMjrVFg+YgfksZCWbtA3TgbKbPfZZtapB9cGL/5WVnM= +cloud.google.com/go/appengine v1.14.0/go.mod h1:JMjrVFg+YgfksZCWbtA3TgbKbPfZZtapB9cGL/5WVnM= cloud.google.com/go/area120 v0.12.0 h1:huHekUFvDRlDLM1asSGoKvGp639Hki6WYu0t84gqKTU= cloud.google.com/go/area120 v0.12.0/go.mod h1:jD1fw9W4xxIZMY68g7PpbCPleoeGddFs5jPcdhfg3+Y= +cloud.google.com/go/area120 v0.15.0/go.mod h1:jD1fw9W4xxIZMY68g7PpbCPleoeGddFs5jPcdhfg3+Y= cloud.google.com/go/artifactregistry v1.22.0 h1:yL1iX0kxord8xVi1C+wvWA8zy07x8XyPIv34/uOnYq8= cloud.google.com/go/artifactregistry v1.22.0/go.mod h1:aMmdtqKVmbuxCCb/NGDJYZHsK6AtqlcyvD05ACzs1n8= +cloud.google.com/go/artifactregistry v1.25.0/go.mod h1:aMmdtqKVmbuxCCb/NGDJYZHsK6AtqlcyvD05ACzs1n8= cloud.google.com/go/asset v1.24.0 h1:guLe35IT8wGzMj1HSPCjuF+KYoJRf7uy6AKhXhE4unY= cloud.google.com/go/asset v1.24.0/go.mod h1:+HaDReZQAh/0syAf0uTMeUrMfXikr+KKyDtCdvf7j4M= +cloud.google.com/go/asset v1.27.0/go.mod h1:+HaDReZQAh/0syAf0uTMeUrMfXikr+KKyDtCdvf7j4M= cloud.google.com/go/assuredworkloads v1.15.0 h1:K21uUQ+KzM8QakqBE378lbwkk3VdlqTK31vfUx7Tq/8= cloud.google.com/go/assuredworkloads v1.15.0/go.mod h1:zBnVYn0E+sDW/mhEmcg1R8+8tguXrtBgmfGY0q34kss= +cloud.google.com/go/assuredworkloads v1.18.0/go.mod h1:zBnVYn0E+sDW/mhEmcg1R8+8tguXrtBgmfGY0q34kss= +cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= cloud.google.com/go/automl v1.17.0 h1:9X05fKqAcLbsJ3hFohvcgwmaPnhrUbHt6ph28CWcow8= cloud.google.com/go/automl v1.17.0/go.mod h1:OkHxjbVDblDafhwuP8yEkz1xcUJhgcbhbsieCW7GaiI= +cloud.google.com/go/automl v1.20.0/go.mod h1:OkHxjbVDblDafhwuP8yEkz1xcUJhgcbhbsieCW7GaiI= cloud.google.com/go/baremetalsolution v1.6.0 h1:JYf8RUjYvfLQ9FGCA/PhkmENbhLZkW8yAXTeB6hvots= cloud.google.com/go/baremetalsolution v1.6.0/go.mod h1:o+stutiS8t+HmjNIG92Gkn8H9+5/q27d6lQp7e9GWdg= +cloud.google.com/go/baremetalsolution v1.9.0/go.mod h1:o+stutiS8t+HmjNIG92Gkn8H9+5/q27d6lQp7e9GWdg= cloud.google.com/go/batch v1.16.0 h1:iop/7lKI28WHJ2mEFhkt0HNpX3MXy7r++zqL0BRy1+k= cloud.google.com/go/batch v1.16.0/go.mod h1:dpWfhLmLQZqsTBAFYjZA3pS04fCY5ttTenZcWmSeILw= +cloud.google.com/go/batch v1.19.0/go.mod h1:dpWfhLmLQZqsTBAFYjZA3pS04fCY5ttTenZcWmSeILw= cloud.google.com/go/beyondcorp v1.4.0 h1:UfJgOIKgAoFCtSrYFj/DwRmQxEsC1qvP3RjNgdXYNE8= cloud.google.com/go/beyondcorp v1.4.0/go.mod h1:vujdO0wfsBV2y1egrJxGtwKZr5P5V6bIHKWp1phWHBY= +cloud.google.com/go/beyondcorp v1.7.0/go.mod h1:vujdO0wfsBV2y1egrJxGtwKZr5P5V6bIHKWp1phWHBY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.76.0 h1:wnfVSXN6GEMlsAoHWdhzTC8NMsptOx2hsqPiI+lTs3I= cloud.google.com/go/bigquery v1.76.0/go.mod h1:J4wuqka/1hEpdJxH2oBrUR0vjTD+r7drGkpcA3yqERM= +cloud.google.com/go/bigquery v1.78.0/go.mod h1:NreOOkdlH/8ji6liI8wpSRvys9C6NSqfNx/M1VKQjBc= cloud.google.com/go/bigtable v1.46.0 h1:Bd6vITx01s6gsdEPvjIGJL/oMMdKvraGI34UiixR2gk= cloud.google.com/go/bigtable v1.46.0/go.mod h1:GUM6PdkG3rrDse9kugqvX5+ktwo3ldfLtLi1VFn5Wj4= +cloud.google.com/go/bigtable v1.50.0/go.mod h1:RTannV5mvoJM8KscLTfRYMPo84u9/j+C3PSyYJGf5Ic= cloud.google.com/go/billing v1.22.0 h1:yU4GwnMlr67TAV7apg+Ca2i7Lu1rJnWGgmLS6d+qTyM= cloud.google.com/go/billing v1.22.0/go.mod h1:nQVTVqWyw8YK0QScVV1pXr9KrvjvM7cabIQr6tLJMys= +cloud.google.com/go/billing v1.26.0/go.mod h1:axqDO1uHegh7u5qngkTfqN1djAeLGsWAFAblERgmgEk= cloud.google.com/go/binaryauthorization v1.12.0 h1:dWY+Vq1Y/aIz01okcLU5S4herPtkU/8r9KGsrwzqe7s= cloud.google.com/go/binaryauthorization v1.12.0/go.mod h1:+0CndCJPtcHuVCNok+qQskWvbP5Sp5m6eGL8Vpu5mss= +cloud.google.com/go/binaryauthorization v1.15.0/go.mod h1:+0CndCJPtcHuVCNok+qQskWvbP5Sp5m6eGL8Vpu5mss= cloud.google.com/go/certificatemanager v1.11.0 h1:PATHWGLy/kNDiUlGlM5FCoA+aZVKcELbXai40TsFGEU= cloud.google.com/go/certificatemanager v1.11.0/go.mod h1:QOA8qRoM6/Ik03+srLnBykenGTy0fk78dnPcx5ZWOW8= +cloud.google.com/go/certificatemanager v1.14.0/go.mod h1:QOA8qRoM6/Ik03+srLnBykenGTy0fk78dnPcx5ZWOW8= cloud.google.com/go/channel v1.23.0 h1:mWw74pQ14Kl5c4Akkh/7vnkv9VXJIXMs3O7Qs8TDTuU= cloud.google.com/go/channel v1.23.0/go.mod h1:04T5Wjq+mHlvEUNzExydnBW1vO64q3Q2Wsblp/dpBxY= +cloud.google.com/go/channel v1.26.0/go.mod h1:04T5Wjq+mHlvEUNzExydnBW1vO64q3Q2Wsblp/dpBxY= cloud.google.com/go/cloudbuild v1.27.0 h1:CNvJxVXqDj3cVSXQ2OnVNEa5w7Oo+PZvp41e6A99QAQ= cloud.google.com/go/cloudbuild v1.27.0/go.mod h1:rg52xEmndQQPiC9NV/8sCaVtKxHMU9D9MeU+oE9VGKA= +cloud.google.com/go/cloudbuild v1.31.0/go.mod h1:QeHawskCCsONQoWJAUeV/qOq4Jablq5n0x8hamsQi7Y= cloud.google.com/go/clouddms v1.10.0 h1:fbXBTIzk58x9KTHEGTBw9v7R2PkdiZjz+iu5h+iJQb0= cloud.google.com/go/clouddms v1.10.0/go.mod h1:aMgrOZ+/EKF/PL+h1sDbS+7fAIYV5rTwD+G/apCeHQk= +cloud.google.com/go/clouddms v1.13.0/go.mod h1:aMgrOZ+/EKF/PL+h1sDbS+7fAIYV5rTwD+G/apCeHQk= cloud.google.com/go/cloudtasks v1.15.0 h1:hVjZUqEbqrTl9GPLcGl1jEyd48YXOn8F4Y5HA7OYYEA= cloud.google.com/go/cloudtasks v1.15.0/go.mod h1:3KeCxwtGEyaySL7CR3lMmEa2I4mq1ynXdgmfNiO4RYE= +cloud.google.com/go/cloudtasks v1.18.0/go.mod h1:3KeCxwtGEyaySL7CR3lMmEa2I4mq1ynXdgmfNiO4RYE= cloud.google.com/go/compute v1.58.0 h1:WQamD8d9Vxu+m5IJBxgUzqsbHxXwx/T7Ol4V5GyVCZE= cloud.google.com/go/compute v1.58.0/go.mod h1:5/1KiFDIdFPcx/Fw6pFQpcQaBGc6MGtgSfczeLFBj7o= cloud.google.com/go/contactcenterinsights v1.19.0 h1:3BGjbFCfSnRLkIhhJ4eRinKONZXRDohDxC6A1nZ0EWU= cloud.google.com/go/contactcenterinsights v1.19.0/go.mod h1:2Crd36H59Lwkt4gWrLgmnbnF59IIZIa3XYt1gtNqJkQ= +cloud.google.com/go/contactcenterinsights v1.22.0/go.mod h1:2Crd36H59Lwkt4gWrLgmnbnF59IIZIa3XYt1gtNqJkQ= cloud.google.com/go/container v1.48.0 h1:OIz8pl1oDPYq0+X1eWS3WLQXBWa2UKQtieq1MTP6nl4= cloud.google.com/go/container v1.48.0/go.mod h1:9pBrzW8z1Ytu/c8a3AD8QISILiRsNqYV7lJ1xInrdhU= +cloud.google.com/go/container v1.53.0/go.mod h1:SBOylKhlKYCBFs/8kz2yqRdUW5ctVNHs82JKOTjrB9s= cloud.google.com/go/containeranalysis v0.16.0 h1:v+Q3ZBOf5LdmQ9uiAcxqwTZfGCBpQ6ZniG0iZK398js= cloud.google.com/go/containeranalysis v0.16.0/go.mod h1:4bj+b/9SrnbGoxOVRaPwr1DlKxeRPHFWDZE3We/l8ng= +cloud.google.com/go/containeranalysis v0.19.0/go.mod h1:Zq0XHzUIa0oTa7H6aSR8HWqeJnoRI9syUcYJzfozjZQ= cloud.google.com/go/datacatalog v1.28.0 h1:hkwiX19v+qpZtVu8wpSVbb50P90Td3LmbGZYG6Dcivk= cloud.google.com/go/datacatalog v1.28.0/go.mod h1:MP8V3kNuESnwMk4mB6zdWmw/4KQ5xZ8dyUNVsggqN5I= +cloud.google.com/go/datacatalog v1.32.0/go.mod h1:DE272tynQUwheJeQAyVfV+nO8yrdkuDyOgH2LtOrkWM= cloud.google.com/go/dataflow v0.13.0 h1:ImEtopgIS7kQBQh+tnj/zJDS0YIwcJSIcfWRkTf1hEA= cloud.google.com/go/dataflow v0.13.0/go.mod h1:BWhSrIGmsMfuYj3J+nJ2Tw7tplRR6r28kvRiqCD3WlQ= +cloud.google.com/go/dataflow v0.16.0/go.mod h1:BWhSrIGmsMfuYj3J+nJ2Tw7tplRR6r28kvRiqCD3WlQ= cloud.google.com/go/dataform v0.16.0 h1:G4hlmGO8BFxM2NJ+9KDUOT2eIw+JE4cR24AjJ3RvIiI= cloud.google.com/go/dataform v0.16.0/go.mod h1:i1a0zkS751kvrY1IIPpUQZ77H5doxx7cs0AP3hnXTMk= +cloud.google.com/go/dataform v1.1.0/go.mod h1:ITyvb6cr2uQ5l4IEdHPrKmCVn6naTVoFbs0OqJJPtis= cloud.google.com/go/datafusion v1.10.0 h1:T+8c2aBThPJY33/yH0G9eH/PWqjB7GcouTl/4V9sD9k= cloud.google.com/go/datafusion v1.10.0/go.mod h1:MQdANs3I/4gitzY+mTBx27rrQyMiUg8uc2Z4TPLWWfc= +cloud.google.com/go/datafusion v1.13.0/go.mod h1:MQdANs3I/4gitzY+mTBx27rrQyMiUg8uc2Z4TPLWWfc= cloud.google.com/go/datalabeling v0.11.0 h1:CqtTYmgkEuSzHSJu6In30Rw3CmH12Vj30LQ8ZXf/pDM= cloud.google.com/go/datalabeling v0.11.0/go.mod h1:DYjvP4RhQ0332YgO22APYlBjCebb+SCaS0e2KApDq/Q= +cloud.google.com/go/datalabeling v0.14.0/go.mod h1:DYjvP4RhQ0332YgO22APYlBjCebb+SCaS0e2KApDq/Q= cloud.google.com/go/dataplex v1.31.0 h1:F3RgMT5RMW0nAtn9XuVdw83HcbPaSwzVVhgYQnoe7Zo= cloud.google.com/go/dataplex v1.31.0/go.mod h1:sOazL+Bs/PTxiMHQ5yBboBvEW9qPrpGogx3+RAgfIt8= +cloud.google.com/go/dataplex v1.35.0/go.mod h1:B7AFwXU1u3sp7FVQ3IFYnQguGTycJS2mF1voE0lLe1o= cloud.google.com/go/dataproc/v2 v2.18.0 h1:pcCfmceUwcFZsdw1vIx6x4tFfdaqLBZCdTFZdkse+ZQ= cloud.google.com/go/dataproc/v2 v2.18.0/go.mod h1:oARVSa38kAHvSuG+cozsrY2sE6UajGuvOOf9vS+ADHI= +cloud.google.com/go/dataproc/v2 v2.24.0/go.mod h1:sjjMzfmK3Ne/gTyA/H+uV0yIDix8+sXAJO44OgAo1q8= cloud.google.com/go/dataqna v0.11.0 h1:4NEQ+z6Qw9nl6FhNSQMiExiFKXyYeIhDsYp8mGQ8C84= cloud.google.com/go/dataqna v0.11.0/go.mod h1:XiVVFTOEJLBSvm3ILbyjXngGQYpjb/66MSksqz/56fs= +cloud.google.com/go/dataqna v0.13.0/go.mod h1:XiVVFTOEJLBSvm3ILbyjXngGQYpjb/66MSksqz/56fs= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.22.0 h1:FOyx2Ag6ibD2wFkz9S8EiNrmBugia8pQOfpyJxi2yqA= cloud.google.com/go/datastore v1.22.0/go.mod h1:aopSX+Whx0lHspWWBj+AjWt68/zjYsPfDe3LjWtqZg8= +cloud.google.com/go/datastore v1.24.0/go.mod h1:cEkLhU6Ti/gauQ7DFrUrG8bQjiMIxi++b5ePiThi5So= cloud.google.com/go/datastream v1.17.0 h1:VA0bz7qFVDDoUrTMm77q6cdhkrpeAqdKXvSoLh+Qq1E= cloud.google.com/go/datastream v1.17.0/go.mod h1:uoWTtfP20W8MXuV2DPcl5zqnVsxQ9QEmmBHX858oYTQ= +cloud.google.com/go/datastream v1.20.0/go.mod h1:uoWTtfP20W8MXuV2DPcl5zqnVsxQ9QEmmBHX858oYTQ= cloud.google.com/go/deploy v1.29.0 h1:fnhmOmf4XktCwEAQPJsBeggImukEApR4DJU9jzLyrjI= cloud.google.com/go/deploy v1.29.0/go.mod h1:lUG7maG/NkoTXmQ8G1mtcVymnbizfDJh6ER7vljVa/U= +cloud.google.com/go/deploy v1.32.0/go.mod h1:lUG7maG/NkoTXmQ8G1mtcVymnbizfDJh6ER7vljVa/U= cloud.google.com/go/dialogflow v1.79.0 h1:k3GOUD4Viu3IR/SW65y4NaFyFdsnAOvxoI+j3DzMq8U= cloud.google.com/go/dialogflow v1.79.0/go.mod h1:UtuiGOq9gAlTz9u4Vt+q1syMrx9ANQzTk+lC3WDdSOw= +cloud.google.com/go/dialogflow v1.83.0/go.mod h1:Rr0/YdrUAVQ+CPgt1yq8VpJmVSChfhz+uodykrBP1d0= cloud.google.com/go/dlp v1.31.0 h1:O0hRNTcMhUNrvHmIAUpk3GY6sOS5TUWmLHJdtH7YOog= cloud.google.com/go/dlp v1.31.0/go.mod h1:+haQd/n0QTv5BK7wZnCk2qctd5sfKL50jjh9E6N0d/Q= +cloud.google.com/go/dlp v1.36.0/go.mod h1:UW92dBhxvqkSKLct+Ril7Y9B4CanS5VLuDwlTGVA9VQ= cloud.google.com/go/documentai v1.45.0 h1:IY+gbb7B/J9XbPMTmmM8M4klPXjk4ZRFtpKbT5wuGMc= cloud.google.com/go/documentai v1.45.0/go.mod h1:mGjfbNf0cqCHKgxMZZV7frbfoF9T2hKkU1h88QyOy3c= +cloud.google.com/go/documentai v1.48.0/go.mod h1:mGjfbNf0cqCHKgxMZZV7frbfoF9T2hKkU1h88QyOy3c= cloud.google.com/go/domains v0.12.0 h1:3nohkVMXkqjaBAZIc9l7VfhwxtG/JYe36csp0mBVb/c= cloud.google.com/go/domains v0.12.0/go.mod h1:BjoSVNc+LVwoHMnE2fxTQNzGLSWWb6f3a8VAN6+VjVk= +cloud.google.com/go/domains v0.15.0/go.mod h1:BjoSVNc+LVwoHMnE2fxTQNzGLSWWb6f3a8VAN6+VjVk= cloud.google.com/go/edgecontainer v1.6.0 h1:1Zfa7DBtxc5u5vqurw9XEJ/UhghHXLH+aHLCSpyXTgs= cloud.google.com/go/edgecontainer v1.6.0/go.mod h1:mZmgXuMGTGI6RUUTXsOZa+F2rFF21v0JPnuX7LQEqBE= +cloud.google.com/go/edgecontainer v1.9.0/go.mod h1:mZmgXuMGTGI6RUUTXsOZa+F2rFF21v0JPnuX7LQEqBE= cloud.google.com/go/errorreporting v0.6.0 h1:52yZlQc72U1ygLfLUrulWshCxUjRVXsjv9agJQ5Lk2w= cloud.google.com/go/errorreporting v0.6.0/go.mod h1:POGEbtuDfvr8gjl9Je4fZGyAHOa5oF1FmlV9TeB3hvw= +cloud.google.com/go/errorreporting v0.9.0/go.mod h1:V7ojx7z76JITDZNGyDNkIIa9nNEkQzF6Yj+VHl2YF84= cloud.google.com/go/essentialcontacts v1.9.0 h1:ZqIG4XB1IaOJac+IgH6vriKBoJ9KmLLYToz6ohNRcFI= cloud.google.com/go/essentialcontacts v1.9.0/go.mod h1:W8fTL17jP6vmsPHQaCT5rOjWGohEssuqDUroxnjST0A= +cloud.google.com/go/essentialcontacts v1.12.0/go.mod h1:W8fTL17jP6vmsPHQaCT5rOjWGohEssuqDUroxnjST0A= cloud.google.com/go/eventarc v1.20.0 h1:5oJY8mvuM2a+zJ5V4M9FlyF27VOjuOEderzpPBWTyyQ= cloud.google.com/go/eventarc v1.20.0/go.mod h1:tIJL0hoWtZXVa5MjcAep/4xB+AXz4AbqQV14ogX5VwU= +cloud.google.com/go/eventarc v1.24.0/go.mod h1:pFJA4y1jNwTT1oq7BlV04G3oRr0PjQB4lQjJxmpFZdE= cloud.google.com/go/filestore v1.12.0 h1:9gyvG3IbbP2TW2Vx5lnxfjk/54bmWuw0aUZ0J3tDOHY= cloud.google.com/go/filestore v1.12.0/go.mod h1:oD+PvCWu4HqfEdNv65yk2XaLIiP7h4AuAH9Ua5YBRTM= +cloud.google.com/go/filestore v1.15.0/go.mod h1:oD+PvCWu4HqfEdNv65yk2XaLIiP7h4AuAH9Ua5YBRTM= cloud.google.com/go/firestore v1.21.0 h1:BhopUsx7kh6NFx77ccRsHhrtkbJUmDAxNY3uapWdjcM= cloud.google.com/go/firestore v1.21.0/go.mod h1:1xH6HNcnkf/gGyR8udd6pFO4Z7GWJSwLKQMx/u6UrP4= +cloud.google.com/go/firestore v1.23.0/go.mod h1:2EWfSUj+iqHckyH7uSjLI+lDgSAWJs9jL/uP45PnuQQ= cloud.google.com/go/functions v1.21.0 h1:16ViG3LpWYNgiEf1Ke6HD3x6kklvAjM22BxASxklXmo= cloud.google.com/go/functions v1.21.0/go.mod h1:t40GeqBAQNuqKlHCxmV/pxhyYJnImLcvRa3GBv4tAy0= +cloud.google.com/go/functions v1.24.0/go.mod h1:t40GeqBAQNuqKlHCxmV/pxhyYJnImLcvRa3GBv4tAy0= cloud.google.com/go/gkebackup v1.10.0 h1:FSFo87mslXBuzi0HoQd0uIsujFZAYpLwGIHlwNwY3E0= cloud.google.com/go/gkebackup v1.10.0/go.mod h1:D2MDbHW4V/uKCmS9TnT8hNKX2tPkE/pWp9nSm0TQ9hY= +cloud.google.com/go/gkebackup v1.13.0/go.mod h1:D2MDbHW4V/uKCmS9TnT8hNKX2tPkE/pWp9nSm0TQ9hY= cloud.google.com/go/gkeconnect v0.14.0 h1:1/OdUqvqpOUe22Id39LWMvIU7SEkGNIM3d/8jypvgJ8= cloud.google.com/go/gkeconnect v0.14.0/go.mod h1:5iWSBQzMIRLwUHUWVhxxcNK45ZPE8ntyBgE0MkavlqQ= +cloud.google.com/go/gkeconnect v1.0.0/go.mod h1:5iWSBQzMIRLwUHUWVhxxcNK45ZPE8ntyBgE0MkavlqQ= cloud.google.com/go/gkehub v0.18.0 h1:GW/XtnNhHG5lqghiRKD6Vi4Z5Uo2tb8bgA3WgBt1ZMY= cloud.google.com/go/gkehub v0.18.0/go.mod h1:xKePlMrI8LpKErzKMWdH/yQv+GDV60ypCNfTTdT+BN0= +cloud.google.com/go/gkehub v0.21.0/go.mod h1:xKePlMrI8LpKErzKMWdH/yQv+GDV60ypCNfTTdT+BN0= cloud.google.com/go/gkemulticloud v1.8.0 h1:Kad9qd7D4wRv/IYbLF2Np2N1wsgPO1GWpgjrVs/4wUA= cloud.google.com/go/gkemulticloud v1.8.0/go.mod h1:OtfHtgqOgDrXfcdFw8eUkCUI154Q51vvdqZYZV4c4qM= +cloud.google.com/go/gkemulticloud v1.11.0/go.mod h1:OtfHtgqOgDrXfcdFw8eUkCUI154Q51vvdqZYZV4c4qM= cloud.google.com/go/gsuiteaddons v1.9.0 h1:Mikqc/0DDiH1sr0lJXEAusEUPPdjtUvkqNasOkwkTuQ= cloud.google.com/go/gsuiteaddons v1.9.0/go.mod h1:rm/XT7wmwOFGn7jmWtVV65QmZCakzTbHLSojIC4Hskg= +cloud.google.com/go/gsuiteaddons v1.12.0/go.mod h1:rm/XT7wmwOFGn7jmWtVV65QmZCakzTbHLSojIC4Hskg= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= cloud.google.com/go/iap v1.14.0 h1:/VpfHrG9hHSkbNdI8MpgS0zlJpERdmJfuR4JmhD8jQA= cloud.google.com/go/iap v1.14.0/go.mod h1:b+r+yjrss2WmAEzNrQQjlEdD5E9B8c47mOF7XnqT+z0= +cloud.google.com/go/iap v1.17.0/go.mod h1:b+r+yjrss2WmAEzNrQQjlEdD5E9B8c47mOF7XnqT+z0= cloud.google.com/go/ids v1.7.0 h1:0mCN95Cemgur3U6n6TuX3phJ3HsGysG1JzykcOG1dbI= cloud.google.com/go/ids v1.7.0/go.mod h1:uCSFrXfCnRUKBl5PdE/ZqBNp1+vKSKPWpdYGa61WjpQ= +cloud.google.com/go/ids v1.10.0/go.mod h1:uCSFrXfCnRUKBl5PdE/ZqBNp1+vKSKPWpdYGa61WjpQ= cloud.google.com/go/iot v1.10.0 h1:lZtlw2YAXOOb8JKjC3jybu3/lnQwIZ8lQhgJhJ30hvE= cloud.google.com/go/iot v1.10.0/go.mod h1:62W4n2fe/Ct66NWJEfCB5suZ3XsL5Atx+MxFjScr+9s= +cloud.google.com/go/iot v1.13.0/go.mod h1:62W4n2fe/Ct66NWJEfCB5suZ3XsL5Atx+MxFjScr+9s= +cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58= cloud.google.com/go/kms v1.28.0 h1:TeBvmmVF3EtrXXC8hiBo0uV8ntX8QBbZeEQFnVpE70k= cloud.google.com/go/kms v1.28.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U= +cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U= cloud.google.com/go/language v1.16.0 h1:F/r+Fw6ZEkuw3kLz/hnkqtAv+U4PyciLzxpp9SsWKh8= cloud.google.com/go/language v1.16.0/go.mod h1:xSeiVB4UiA9wYmFy2GWjf1Mb1K3uR1Yi/80qoqTxH04= +cloud.google.com/go/language v1.18.0/go.mod h1:xSeiVB4UiA9wYmFy2GWjf1Mb1K3uR1Yi/80qoqTxH04= cloud.google.com/go/lifesciences v0.12.0 h1:du9NOdooXpqvQLEeUedIc5v6yVvH6qRr7xxz+9iI5II= cloud.google.com/go/lifesciences v0.12.0/go.mod h1:FwS+QkqPdVWl4SmKUCFozFvsTVWTLH13HCKcwR/MR9U= +cloud.google.com/go/lifesciences v0.15.0/go.mod h1:FwS+QkqPdVWl4SmKUCFozFvsTVWTLH13HCKcwR/MR9U= +cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= +cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= cloud.google.com/go/managedidentities v1.9.0 h1:304LxYj+TrAvjNFsnN+1I02+/GIGdC+7cHrnwEVqv3I= cloud.google.com/go/managedidentities v1.9.0/go.mod h1:rm72jf/v//0NG73VQNZM1JlV2E95uhJymmSXlgi6hMA= +cloud.google.com/go/managedidentities v1.12.0/go.mod h1:rm72jf/v//0NG73VQNZM1JlV2E95uhJymmSXlgi6hMA= cloud.google.com/go/maps v1.32.0 h1:TnqBkMtgPFl1SHKtlF7iArQHtYJL52V+OErIplesmXM= cloud.google.com/go/maps v1.32.0/go.mod h1:HH1V8tduMn+b9oRMCdl3vok98uvHco/wElZXyJQ/9kU= +cloud.google.com/go/maps v1.36.0/go.mod h1:Ly0sd/0G1MgKuWpGc2vCBjNZ+fc8iRHzcBWJqrw7Xao= cloud.google.com/go/mediatranslation v0.11.0 h1:w1yErnkEUAxt+1SDeXUi3ntGdJCYdh5nGMdRuh7m/3s= cloud.google.com/go/mediatranslation v0.11.0/go.mod h1:kjZrowuigFr+Bf1HM1TCtp1a3E3kfG1ovPK5VEuaNAQ= +cloud.google.com/go/mediatranslation v0.13.0/go.mod h1:kjZrowuigFr+Bf1HM1TCtp1a3E3kfG1ovPK5VEuaNAQ= cloud.google.com/go/memcache v1.13.0 h1:aeiNy8t+G5LebFlsNziEOkxNFAE9vl192KQY0A/ZNaE= cloud.google.com/go/memcache v1.13.0/go.mod h1:y/rXhJiieCF742K958dY29fSfM+Y3wh2thRmWspU2Dg= +cloud.google.com/go/memcache v1.16.0/go.mod h1:y/rXhJiieCF742K958dY29fSfM+Y3wh2thRmWspU2Dg= cloud.google.com/go/metastore v1.16.0 h1:msqpQlbkzJ99PvdrmxX75FQQiELjOshGrxprjKiGKug= cloud.google.com/go/metastore v1.16.0/go.mod h1:JGTjGdQ627m2ptDo86XsIKqzzZCk+GG41VEFD7ENsqs= +cloud.google.com/go/metastore v1.19.0/go.mod h1:JGTjGdQ627m2ptDo86XsIKqzzZCk+GG41VEFD7ENsqs= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= cloud.google.com/go/networkconnectivity v1.23.0 h1:KWR8jIWQ7cu8fp/oeedp4Rbd1G8yOlz66uY0Bo5stJM= cloud.google.com/go/networkconnectivity v1.23.0/go.mod h1:Uhzfk7NbiY6RNqV9XFvPWRji58+MkTYsTRfQ3EPtrGg= +cloud.google.com/go/networkconnectivity v1.26.0/go.mod h1:Uhzfk7NbiY6RNqV9XFvPWRji58+MkTYsTRfQ3EPtrGg= cloud.google.com/go/networkmanagement v1.25.0 h1:PBGV/4TV9SX5asGZAF7QiJq1sI1XfVk2cLynBNQhihs= cloud.google.com/go/networkmanagement v1.25.0/go.mod h1:2YogSU3sD7LvtmWntUAuGARbFQmy3A0En3LrJr69jkU= +cloud.google.com/go/networkmanagement v1.29.0/go.mod h1:lk9xX5YTlDyEc6zTp5ARD6MxfDwOJ6qw+MP6yU2Mluw= cloud.google.com/go/networksecurity v0.13.0 h1:TEOg4586EFhLypoPjE8LKdpOzwpOb0Jf/ZDs2JfRal4= cloud.google.com/go/networksecurity v0.13.0/go.mod h1:LMn10eRVf4K85PMF33yRoKAra7VhCOetxFcLDMh9A74= +cloud.google.com/go/networksecurity v0.18.0/go.mod h1:mcXDEKYoT2E3oKO6nh9vpz25DfmT8FYOC90Ua+Z/D8E= cloud.google.com/go/notebooks v1.14.0 h1:MBNcFjcuuQRC0a5qEy6G6RNhZR3GqTp4ax3XgwY9wzc= cloud.google.com/go/notebooks v1.14.0/go.mod h1:NScGIhfQCqLRIlVaUVbm595F6dhqiTl5XS1KaKgitKM= +cloud.google.com/go/notebooks v1.17.0/go.mod h1:NScGIhfQCqLRIlVaUVbm595F6dhqiTl5XS1KaKgitKM= cloud.google.com/go/optimization v1.9.0 h1:tHK6kYfZmzS3yvu40/vspI0C+NvMYD0hu7P/ZMb6cFg= cloud.google.com/go/optimization v1.9.0/go.mod h1:qCWskZMcynh0GBsUrCP6oPwwnUhbwg5UcXvVM9hzOD8= +cloud.google.com/go/optimization v1.11.0/go.mod h1:qCWskZMcynh0GBsUrCP6oPwwnUhbwg5UcXvVM9hzOD8= cloud.google.com/go/orchestration v1.13.0 h1:Zc7B3dhsb5HNbPc/62BRopMaPS34EQESkS2lwYeOOKg= cloud.google.com/go/orchestration v1.13.0/go.mod h1:H7MFVP8Z/dtml39nf43sWYPL/2o7J4tdSZAlJrBuqnQ= +cloud.google.com/go/orchestration v1.16.0/go.mod h1:H7MFVP8Z/dtml39nf43sWYPL/2o7J4tdSZAlJrBuqnQ= cloud.google.com/go/orgpolicy v1.17.0 h1:zpP13I+e3yOISCdGVf0ibXODieQzyr3wFVp1nM/o5Zo= cloud.google.com/go/orgpolicy v1.17.0/go.mod h1:9LHqEGx5P5dhansdKTNIEXpM+QbebAIOs66+HUID4aQ= +cloud.google.com/go/orgpolicy v1.20.0/go.mod h1:9LHqEGx5P5dhansdKTNIEXpM+QbebAIOs66+HUID4aQ= cloud.google.com/go/osconfig v1.18.0 h1:p9CgUYAOUXduHBOP7IlxujnqxJd+R//RvAa5itKokrk= cloud.google.com/go/osconfig v1.18.0/go.mod h1:BofnHqjjvu6lZQv/hqo2+rLCUiY4O6A9UYwwvVrSBjk= +cloud.google.com/go/osconfig v1.21.0/go.mod h1:BofnHqjjvu6lZQv/hqo2+rLCUiY4O6A9UYwwvVrSBjk= cloud.google.com/go/oslogin v1.16.0 h1:s/aH9HVBJgLWwqplshyEr6ppIug9MMboE8yTE1bvpVw= cloud.google.com/go/oslogin v1.16.0/go.mod h1:3Oa36T3781Mv+yCSVYlfasi7auHjfPFqvNOd1q92umc= +cloud.google.com/go/oslogin v1.18.0/go.mod h1:3Oa36T3781Mv+yCSVYlfasi7auHjfPFqvNOd1q92umc= cloud.google.com/go/phishingprotection v0.11.0 h1:OK26yBWBr1Ldd2E0UPSVIZfnjb88lCB0knlC9mJzxhM= cloud.google.com/go/phishingprotection v0.11.0/go.mod h1:2gyYqwNjePPEocXDkDve3EuJPaRqN/E7fp28K3arR0k= +cloud.google.com/go/phishingprotection v0.13.0/go.mod h1:2gyYqwNjePPEocXDkDve3EuJPaRqN/E7fp28K3arR0k= cloud.google.com/go/policytroubleshooter v1.13.0 h1:qct52TKyLYT51Xf5Atwv/elgNjJPvHI/gbkGEJz71eo= cloud.google.com/go/policytroubleshooter v1.13.0/go.mod h1:yNuROjN6h+2/TE2JOvBBJMjYIjC6j0UYHq8f2kVHlA4= +cloud.google.com/go/policytroubleshooter v1.15.0/go.mod h1:yNuROjN6h+2/TE2JOvBBJMjYIjC6j0UYHq8f2kVHlA4= cloud.google.com/go/privatecatalog v0.12.0 h1:nJi3YDgERB/Zs5Pmzs8G1fGDIxPMPnKGSOwVJK0yJeg= cloud.google.com/go/privatecatalog v0.12.0/go.mod h1:av2b5Rv+oG5ORxUqGlCAYO9s4pXjgc6q2qO9nkTcqT8= +cloud.google.com/go/privatecatalog v0.15.0/go.mod h1:av2b5Rv+oG5ORxUqGlCAYO9s4pXjgc6q2qO9nkTcqT8= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk= cloud.google.com/go/pubsub v1.50.2 h1:54Up97HnThdP4H8jjWJSSQ/mnYG2EKon7ZSNETRq0tM= cloud.google.com/go/pubsub v1.50.2/go.mod h1:jyCWeZdGFqd4mitSsBERnJcpqaHBsxQoPkNvjj4sp0w= +cloud.google.com/go/pubsub v1.50.4/go.mod h1:CBCG3lNP243mGNB8kTILs/Pd/gTV9a00kM1KjOtdxEk= +cloud.google.com/go/pubsub/v2 v2.4.0/go.mod h1:2lS/XQKq5qtOMs6kHBK+WX1ytUC36kLl2ig3zqsGUx8= cloud.google.com/go/pubsub/v2 v2.5.1 h1:+TwXJr78P9RrMV3S8lKHIhJo2E99jI7ta65e+ujJjts= cloud.google.com/go/pubsub/v2 v2.5.1/go.mod h1:Pd+qeabMX+576vQJhTN7TelE4k6kJh15dLU/ptOQ/UA= +cloud.google.com/go/pubsub/v2 v2.6.0/go.mod h1:4anqvV/w8Pcgu2tO0qr2XgsF3GXHowzryfQ5gOnVmWY= cloud.google.com/go/pubsublite v1.8.2 h1:jLQozsEVr+c6tOU13vDugtnaBSUy/PD5zK6mhm+uF1Y= cloud.google.com/go/pubsublite v1.8.2/go.mod h1:4r8GSa9NznExjuLPEJlF1VjOPOpgf3IT6k8x/YgaOPI= +cloud.google.com/go/pubsublite v1.9.0/go.mod h1:a2QASlHAcTR3sFs8Tz8TTZcyymyaTr67V3DFG3FkteI= cloud.google.com/go/recaptchaenterprise/v2 v2.23.0 h1:Xg76iWTekGXtmlDKVbujV6NVGPKm6Gn7hzr1Berd5ho= cloud.google.com/go/recaptchaenterprise/v2 v2.23.0/go.mod h1:+ntF70/j7qBa6G/pwmYA0mkBcDeTCXV6WDqUL7GObfs= +cloud.google.com/go/recaptchaenterprise/v2 v2.26.0/go.mod h1:+ntF70/j7qBa6G/pwmYA0mkBcDeTCXV6WDqUL7GObfs= cloud.google.com/go/recommendationengine v0.11.0 h1:1kifT41qy/r/XNITOgL2wOGA2EY0QV3gTgfXqvSjYRw= cloud.google.com/go/recommendationengine v0.11.0/go.mod h1:UP9cN46tDpZ/N57eDYIWeIRHjMOchtiIyjWjV0Dvr3k= +cloud.google.com/go/recommendationengine v0.14.0/go.mod h1:UP9cN46tDpZ/N57eDYIWeIRHjMOchtiIyjWjV0Dvr3k= cloud.google.com/go/recommender v1.15.0 h1:I82Iu70NcyHnTswn4IlJVnzLsp5zC9mCrMBDY0SP6bA= cloud.google.com/go/recommender v1.15.0/go.mod h1:INRBLfBQJCrgPqjBVFht4OjaFq/WhB/c5V1sqBOdX8g= +cloud.google.com/go/recommender v1.19.0/go.mod h1:LRh+1HJjLx2kDE3S65AIlG/lvwA0llEFWYPD/QtgoaU= cloud.google.com/go/redis v1.20.0 h1:pQYJxWxYLDx0r/W75ju5+vP5b5ePpZqpm4NWx04OYTQ= cloud.google.com/go/redis v1.20.0/go.mod h1:EUlUT24BAL6LsE1f/N9Bg3LhRCfH+LzwLGbst3KuZRw= +cloud.google.com/go/redis v1.23.0/go.mod h1:EUlUT24BAL6LsE1f/N9Bg3LhRCfH+LzwLGbst3KuZRw= cloud.google.com/go/resourcemanager v1.12.0 h1:sknKs2H0QW9uYPhnR7lD73HQ6v+qb4XXlxoCq0rbj9g= cloud.google.com/go/resourcemanager v1.12.0/go.mod h1:ve0VNxPoDU6XxDuEMCjkineb0YzXQXx3mOWwnNckGDE= +cloud.google.com/go/resourcemanager v1.15.0/go.mod h1:ve0VNxPoDU6XxDuEMCjkineb0YzXQXx3mOWwnNckGDE= cloud.google.com/go/resourcesettings v1.8.3 h1:13HOFU7v4cEvIHXSAQbinF4wp2Baybbq7q9FMctg1Ek= cloud.google.com/go/resourcesettings v1.8.3/go.mod h1:BzgfXFHIWOOmHe6ZV9+r3OWfpHJgnqXy8jqwx4zTMLw= cloud.google.com/go/retail v1.28.0 h1:TZdueueRw5KO0sec7m1TQIWHHIFvf2GhqOiWtCn4rPY= cloud.google.com/go/retail v1.28.0/go.mod h1:sfq/cT+gfSLuURf/mdVAw5n0pav3hxSP1rT8RfL7Qxk= +cloud.google.com/go/retail v1.31.0/go.mod h1:sfq/cT+gfSLuURf/mdVAw5n0pav3hxSP1rT8RfL7Qxk= cloud.google.com/go/run v1.18.0 h1:D4u23qVmL5H4nq0wjf8hvnkPqF/8Y3pN8QpvUG/LNys= cloud.google.com/go/run v1.18.0/go.mod h1:Z5wHbyFirI8XU48EPs5XJf/qmVm1SXZEhuS8EvZOuQU= +cloud.google.com/go/run v1.21.0/go.mod h1:Z5wHbyFirI8XU48EPs5XJf/qmVm1SXZEhuS8EvZOuQU= cloud.google.com/go/scheduler v1.13.0 h1:1yxpZj9nJZ5+ZAeLaB9AfvfZ3QN/uUktV5n4jst7Meo= cloud.google.com/go/scheduler v1.13.0/go.mod h1:0hsZg0MZJADyke1lutI0FHAYJR8Dtm8oIivXkmpACkA= +cloud.google.com/go/scheduler v1.16.0/go.mod h1:0hsZg0MZJADyke1lutI0FHAYJR8Dtm8oIivXkmpACkA= +cloud.google.com/go/secretmanager v1.16.0/go.mod h1://C/e4I8D26SDTz1f3TQcddhcmiC3rMEl0S1Cakvs3Q= cloud.google.com/go/secretmanager v1.18.0 h1:VA/ynUUapUF3+xrm0R1dMx8i21p2jfRAWFpokYPncKU= cloud.google.com/go/secretmanager v1.18.0/go.mod h1:9OmSuOeiiUicANglrbdKWSnT3gYkRcXuUQDk7dDW0zU= +cloud.google.com/go/secretmanager v1.20.0/go.mod h1:9OmSuOeiiUicANglrbdKWSnT3gYkRcXuUQDk7dDW0zU= cloud.google.com/go/security v1.21.0 h1:kNdALc/zRzp7JAoVangZS4/72PkjBBigol0DyVw21gc= cloud.google.com/go/security v1.21.0/go.mod h1:XaB3p0SE7v2bBitsLBb1hM6R8/oI/k/IujpXFJalFK0= +cloud.google.com/go/security v1.25.0/go.mod h1:xKPO7XBfUtgjfzPJeznEhI0gp/ZRJt/ZbWtuMYMeUDk= cloud.google.com/go/securitycenter v1.41.0 h1:UnBuTkbp9VS/yGzgNYi6K+m3MLJdGkt3pPOXRnARGIE= cloud.google.com/go/securitycenter v1.41.0/go.mod h1:7BMMbSTAddVfiE+HrC8tKS6SuRkyK7FRPlkpAZBRV3U= +cloud.google.com/go/securitycenter v1.44.0/go.mod h1:7BMMbSTAddVfiE+HrC8tKS6SuRkyK7FRPlkpAZBRV3U= cloud.google.com/go/servicedirectory v1.14.0 h1:cZGu+3RdGcCITK0bYqJ6Bz2sAyIJKjOT75gCGOCsmIc= cloud.google.com/go/servicedirectory v1.14.0/go.mod h1:CtgjXS1idj3s9Q6tB68021Rzk8Q6decV6+ldXC1BoBk= +cloud.google.com/go/servicedirectory v1.17.0/go.mod h1:CtgjXS1idj3s9Q6tB68021Rzk8Q6decV6+ldXC1BoBk= cloud.google.com/go/shell v1.10.0 h1:EH/c5tP+vIwzWKSW0U41C+39Pi5cEt529rrh1x1Eml4= cloud.google.com/go/shell v1.10.0/go.mod h1:TivWrVriy6xQ0wBjNJJridJgODZz8zXUEW2u48kynzY= +cloud.google.com/go/shell v1.12.0/go.mod h1:TivWrVriy6xQ0wBjNJJridJgODZz8zXUEW2u48kynzY= cloud.google.com/go/spanner v1.89.0 h1:r3h5Z5RA8JRPf3HCvA6ujNhREIMhPY+MrDL9mkY8jS0= cloud.google.com/go/spanner v1.89.0/go.mod h1:okNuxnp1wdPaVoM5M28Al2irKZLkHhZ2Z+DW6/ZJWGw= +cloud.google.com/go/spanner v1.92.0/go.mod h1:rCDPfWXNX0h+t484r+crCEaaMKbJfoWkHRDKU3H3+oY= cloud.google.com/go/speech v1.32.0 h1:GajYWX7B+BnHuXBxUBBaSk95uiY5AHxOo8TC63dYQz8= cloud.google.com/go/speech v1.32.0/go.mod h1:shnf33sZbGnQQZyek1fdLOR5rRKV6D3jsNqpqyijvj8= +cloud.google.com/go/speech v1.35.0/go.mod h1:shnf33sZbGnQQZyek1fdLOR5rRKV6D3jsNqpqyijvj8= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= cloud.google.com/go/storagetransfer v1.15.0 h1:ScTtw+rEawshRNfxusOrf0DqFCz/r2IcqnBooB2mBhw= cloud.google.com/go/storagetransfer v1.15.0/go.mod h1:AbGutEym/KNasoiDpSj/CYbigp5yhgosSgwlhGvQNs4= +cloud.google.com/go/storagetransfer v1.18.0/go.mod h1:AbGutEym/KNasoiDpSj/CYbigp5yhgosSgwlhGvQNs4= cloud.google.com/go/talent v1.10.0 h1:/BDAuMr5LwZFvIoeYGBGHQiM0bs5cX+cMHM2BK9kVtw= cloud.google.com/go/talent v1.10.0/go.mod h1:GSwli9V25WQdzeuJDJWH9TlQmA8lPFn7yKsxowdxW9Y= +cloud.google.com/go/talent v1.13.0/go.mod h1:GSwli9V25WQdzeuJDJWH9TlQmA8lPFn7yKsxowdxW9Y= cloud.google.com/go/texttospeech v1.18.0 h1:0pI/aUIR7/hpf6T1IhRb9Hbr1nrKddnPeFL1v9cQbH0= cloud.google.com/go/texttospeech v1.18.0/go.mod h1:p/UVJILAo/S5vsJaWZVdDRzNzA7wXIA+hTACvpMeOBk= +cloud.google.com/go/texttospeech v1.21.0/go.mod h1:p/UVJILAo/S5vsJaWZVdDRzNzA7wXIA+hTACvpMeOBk= cloud.google.com/go/tpu v1.10.0 h1:wYKfCRb1jG4JadajlM6Wr0dCN+3PiidLOYPuJn/4Yo8= cloud.google.com/go/tpu v1.10.0/go.mod h1:F5gT5BL22Dhsr05JLHdMjAjj+wcTn3Xtuu4jvq9yFug= +cloud.google.com/go/tpu v1.13.0/go.mod h1:F5gT5BL22Dhsr05JLHdMjAjj+wcTn3Xtuu4jvq9yFug= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= cloud.google.com/go/translate v1.14.0 h1:d0MD1Zj1N62knqL5uapNuYorjPAww86tOPrpnAYYdWM= cloud.google.com/go/translate v1.14.0/go.mod h1:XnU3rbMmUBZFThzPWJOSNy8V1N9ONnjdH/vWyPppcXU= +cloud.google.com/go/translate v1.17.0/go.mod h1:3mErnHTQBu9yeLiL35K0HBBuaM6Vk2fD/vyWFz790VU= cloud.google.com/go/video v1.29.0 h1:hNj9gsZAg3p47TeE+hlwxIQyaX20zoQVg2oLTd535Z8= cloud.google.com/go/video v1.29.0/go.mod h1:KxDL728ZzH+FJwtEb9XkiLTETW5bI37hTWbJiRYeXkk= +cloud.google.com/go/video v1.32.0/go.mod h1:KxDL728ZzH+FJwtEb9XkiLTETW5bI37hTWbJiRYeXkk= cloud.google.com/go/videointelligence v1.14.0 h1:Eh9I5RuROBy4zIG5QmPENhwBKK74hgOBpeSUVUKtjJ8= cloud.google.com/go/videointelligence v1.14.0/go.mod h1:mmX1JpIWzwozaigrdRNjikZc3aFLNHFKh+OFwAdfiW4= +cloud.google.com/go/videointelligence v1.16.0/go.mod h1:mmX1JpIWzwozaigrdRNjikZc3aFLNHFKh+OFwAdfiW4= cloud.google.com/go/vision/v2 v2.11.0 h1:6MdlpZsPOUpvdx54XsahTxWUMb81NZC3cDgU6izCJsw= cloud.google.com/go/vision/v2 v2.11.0/go.mod h1:ODlLCajJOq4t8thoi1uVvbnfIfix73HsYWhZuIveagQ= +cloud.google.com/go/vision/v2 v2.14.0/go.mod h1:ODlLCajJOq4t8thoi1uVvbnfIfix73HsYWhZuIveagQ= cloud.google.com/go/vmmigration v1.12.0 h1:2/OSRMTfyFx16G7YGTGjURFhobLJXPsx68wbTi3No6Y= cloud.google.com/go/vmmigration v1.12.0/go.mod h1:MP6mQ21ru1usBeCbl805Ioz0Fy+yf3qK2kUkhZ69QQY= +cloud.google.com/go/vmmigration v1.15.0/go.mod h1:MP6mQ21ru1usBeCbl805Ioz0Fy+yf3qK2kUkhZ69QQY= cloud.google.com/go/vmwareengine v1.5.0 h1:BlQwexZuVDgHKtNc51BxO6MM45UpNgg4eWeVEpRp3EU= cloud.google.com/go/vmwareengine v1.5.0/go.mod h1:e66l90IZhm1yQfYZv+YCWjSNSklQZCRmuEvKL8n3Ua0= +cloud.google.com/go/vmwareengine v1.8.0/go.mod h1:e66l90IZhm1yQfYZv+YCWjSNSklQZCRmuEvKL8n3Ua0= cloud.google.com/go/vpcaccess v1.10.0 h1:gY/MOPAWnw9QnPT1VSwdKtJZ9I276rP/+ZwrGDN7G0A= cloud.google.com/go/vpcaccess v1.10.0/go.mod h1:4Uus6E/9FYUtIrwBE1wJ1RosKwb02H6kEd9puJ02TL8= +cloud.google.com/go/vpcaccess v1.13.0/go.mod h1:4Uus6E/9FYUtIrwBE1wJ1RosKwb02H6kEd9puJ02TL8= cloud.google.com/go/webrisk v1.13.0 h1:ANgImW/EBfqPEQtYckW5+vWTgfBb2VvG6YImZCk6NUk= cloud.google.com/go/webrisk v1.13.0/go.mod h1:VIQw8smiaMOlget/xOk6niTkNJTiQc5skEmCuAksxJc= +cloud.google.com/go/webrisk v1.16.0/go.mod h1:VIQw8smiaMOlget/xOk6niTkNJTiQc5skEmCuAksxJc= cloud.google.com/go/websecurityscanner v1.9.0 h1:0QUAHT973x5ZJWQT9oFgUgGRmdcvtbANXmCx54SdqnQ= cloud.google.com/go/websecurityscanner v1.9.0/go.mod h1:cZSc9HqoFdccL1mqZtPIInOd4R8PBGwI20wdnrz6AO8= +cloud.google.com/go/websecurityscanner v1.12.0/go.mod h1:cZSc9HqoFdccL1mqZtPIInOd4R8PBGwI20wdnrz6AO8= cloud.google.com/go/workflows v1.16.0 h1:Ml3EHP/Bmnh8HwOlrJcCXHl3iiRPFCgbp7A1Cg6TGvw= cloud.google.com/go/workflows v1.16.0/go.mod h1:TWsrDGgsJy7xAJ07byzHhKKehEWItJG3BivEHVhGH5g= +cloud.google.com/go/workflows v1.19.0/go.mod h1:TWsrDGgsJy7xAJ07byzHhKKehEWItJG3BivEHVhGH5g= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/Azure/azure-amqp-common-go/v3 v3.2.3 h1:uDF62mbd9bypXWi19V1bN5NZEO84JqgmI5G73ibAmrk= github.com/Azure/azure-amqp-common-go/v3 v3.2.3/go.mod h1:7rPmbSfszeovxGfc5fSAXE4ehlXQZHpMja2OtxC2Tas= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.10.0 h1:kE5kpeiSqu4jcCQ/sWuyggMXJ/pT6oQ99+8hwPmyeJ0= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.10.0/go.mod h1:IAN3Z0DMtehoxoQQnfqg1891z1P7GNoDryKtFcAyMBI= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0/go.mod h1:Y2b/1clN4zsAoUd/pgNAQHjLDnTis/6ROkUfyob6psM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= github.com/Azure/go-amqp v1.5.0 h1:GRiQK1VhrNFbyx5VlmI6BsA1FCp27W5rb9kxOZScnTo= github.com/Azure/go-amqp v1.5.0/go.mod h1:vZAogwdrkbyK3Mla8m/CxSc/aKdnTZ4IbPxl51Y5WZE= +github.com/Azure/go-amqp v1.5.1/go.mod h1:vZAogwdrkbyK3Mla8m/CxSc/aKdnTZ4IbPxl51Y5WZE= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 h1:sR+/8Yb4slttB4vD+b9btVEnWgL3Q00OBTzVT8B9C0c= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v6 v6.2.0 h1:EpcZ6SR9n28BUGtNJSvlBqf90IpjeFr36Tizxhn/oME= github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4= github.com/GoogleCloudPlatform/cloudsql-proxy v1.37.10 h1:0jDrC5r/G+L/p715lTXEYRQ6sET0lzPxwTQlMTy9XfQ= github.com/GoogleCloudPlatform/cloudsql-proxy v1.37.10/go.mod h1:gij9WLu9mdiAFCM2EB+fwnbrVvc7cLr/klV1eEcFwbQ= +github.com/GoogleCloudPlatform/cloudsql-proxy v1.37.13/go.mod h1:6IMmxkLFo8kvCZbG0egv30L6YSgeZwxjH0WVRMGAacQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0 h1:5eCqTd9rTwMlE62z0xFdzPJ+3pji75hJrwq1jrCjo5w= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0/go.mod h1:4BcvJy7WxY8X2eX49z2VO1ByhO+CcQK8lKPCH/QlZvo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.31.0/go.mod h1:VLoD5cAsRQXsAFXpOZrrTGzbuMsntlspIZno4xor5Zg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.58.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0 h1:mQdVn6c25/S2MHfJTWGSK3NwGoI/w9Ad7tzyLWbjAQI= github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0/go.mod h1:8W5IW/jylevlBQKSWkh5ZMP2oy7yT9Pnfug6Y6W/9D8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.55.0/go.mod h1:8W5IW/jylevlBQKSWkh5ZMP2oy7yT9Pnfug6Y6W/9D8= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3 h1:Qbeh12Vq6BxURXT1qZBRHsDxeURB8ztcL6f3EXSGeHk= github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= @@ -254,29 +448,62 @@ github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 h1:KkH3I3sJuOLP github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06/go.mod h1:7erjKLwalezA0k99cWs5L11HWOAPNjdUZ6RxH1BXbbM= github.com/XSAM/otelsql v0.40.0 h1:8jaiQ6KcoEXF46fBmPEqb+pp29w2xjWfuXjZXTXBjaA= github.com/XSAM/otelsql v0.40.0/go.mod h1:/7F+1XKt3/sTlYtwKtkHQ5Gzoom+EerXmD1VdnTqfB4= +github.com/XSAM/otelsql v0.41.0/go.mod h1:NMQT0PiKoFILp9QgjQz+D5mvW+9mT0suR7OejqrtMaM= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= +github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= +github.com/aws/aws-sdk-go-v2/config v1.32.20/go.mod h1:PuwEpciweIXGULWeOeSTXtSbH4CW9mWdWrhdCKQI1sM= +github.com/aws/aws-sdk-go-v2/credentials v1.19.19/go.mod h1:7y63L1kGzeoDlJaQ3Z578KrnmfBut96JjvJUzGwR+YE= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.20.26 h1:khdgzmb6QKweEAnjBhg/Ikcn0VguyOyg0gMSVyK8ddI= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.20.26/go.mod h1:P5lKM3+laQ9v0KAOLhxOkClj4UbBwXJ2QcQc2sKSOYo= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.20.35/go.mod h1:ypTMB9nZhpqfMeRVesGj4dEknIg0YS+aXGtLMidw/Ek= github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.8.26 h1:tHmqvPzeKaCIlD4kdQVP+7v5GdHHtu9olhvweho5zr8= github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.8.26/go.mod h1:qEScmjwld3lw08e6CIWbPIgfcCKBdw2htqqBtSOSINQ= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.8.35/go.mod h1:SomvXQRUKYBML53k4LqIgszKJKz8TdUwi/Zwig7JhfU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25/go.mod h1:9FDWUothyr5RCRAHc45XOiVCzUR8n/IhCYX+uVqw6vk= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.14 h1:gKXU53GYsPuYgkdTdMHh6vNdcbIgoxFQLQGjg+iRG+k= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.14/go.mod h1:jyoemRAktfCyZR9bTb5gT3kn/Vj2KwYDm0Pev5TsmEQ= +github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.20/go.mod h1:wtCkeFPPKHdxFPrZGkdT5tKR4boa3GvW54sYdGNWPHg= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25/go.mod h1:G6kntsA2GorAxDPbap6xgB2F+amSLUF8GJTi7PUoX44= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25/go.mod h1:cKf+D+NMDK1LndD7BowHbBZPgR9V0/5HubH0PFWvA+c= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26/go.mod h1:dY4MRzXEizrD4hqtpKvWVGPX7QleSGGVY+EBolo1RmM= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.53.2 h1:+/HEQj1fQGr17AQ0fAKpefDHw2hxQ3f0q96hY39J8Ao= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.53.2/go.mod h1:bz4cZH7uK5fLxQbj7hL4MFDL+pjReC9en/nM2Wfwxsk= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.56.2/go.mod h1:dLREOeW66eVaaGIOi2ZlLHDgkR3nuJ02rd00j0YSlBE= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.32.6 h1:m8Odxvyy7nirivpiI0VLwqd3lUkVRgeKPQgdJ9YhvcQ= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.32.6/go.mod h1:r2DJVcbGPv7oJGoPICCQJ+4ci5oSGjdXtdscnJIQBfk= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.32.13/go.mod h1:D5up2/CMSP4sF8ESBWla6gJvIMySJi8dYYAaED4oTCc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10/go.mod h1:a57l7Hwh+FWI+we50g5NPJHYUKeJKfXbc4w8SyXu8Ig= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.14 h1:3exo28cClRTVnxdj/LULxkESZSSv74RUIjZ7tfHXfWQ= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.14/go.mod h1:yLon9pByjyB6JZq5IAmwnjE3ObIhD0QibfRWH7tUhLU= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.20/go.mod h1:ihZMtPTKoX/ugQRHbui6zNdSgVYN1KY2Dgwb2d3hXlc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25/go.mod h1:0yAbjPfd64gG7mj85RW+fMEYdfBgCRZw8g/oWcL1pjc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM= +github.com/aws/aws-sdk-go-v2/service/kms v1.50.3/go.mod h1:/iSgiUor15ZuxFGQSTf3lA2FmKxFsQoc2tADOarQBSw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.40.2 h1:p0tPbc1uXSAYs9ACiVB9WxlV6AY5TBVNadXdvGrtOHA= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.40.2/go.mod h1:c6Vg0BRiU7v0MVhHupw90RyL120QBwAMLbDCzptGeMk= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4/go.mod h1:cxiXDhEzIq7Xx1BtmC4lGBK3SwAZ79+EUWiKawYHo14= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.1/go.mod h1:vUtyoSj0OPji3kjIVSc/GlKuWEiL33f/WFxl6dmpy/A= github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4= github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks= +github.com/aws/aws-sdk-go-v2/service/sns v1.39.14/go.mod h1:NKVY7DER6VXHkt2I/ycmHakALNboi3Rqwt4eEf/1Cnk= github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc= github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I= +github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24/go.mod h1:Ql9ziDutk8ERAN9HMaYANCW3lop451ppebkxEJMLCTM= github.com/aws/aws-sdk-go-v2/service/ssm v1.67.4 h1:pOwUUY5FzKUsxtxGR6qsczZP7MuZMVlMbAOPQOcmJlo= github.com/aws/aws-sdk-go-v2/service/ssm v1.67.4/go.mod h1:+nlWvcgDPQ56mChEBzTC0puAMck+4onOFaHg5cE+Lgg= +github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3/go.mod h1:rcRkKbUJ2437WuXdq9fbj+MjTudYWzY9Ct8kiBbN8a8= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.19/go.mod h1:3gt5WJArFooNmyLONS+h/R4J+o86II8du38IgCwj9dE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2/go.mod h1:hU6fqB3OJA6/ePheD47LQnxvjYk6br6PtQxs+Q9ojvk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.3/go.mod h1:ULe4HCzfKPiR6R3HEurE3b1upEkuk8AkMrOKtaOxKO8= +github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= @@ -288,26 +515,50 @@ github.com/calypr/syfon/apigen v0.1.4/go.mod h1:2oqNkseovHqbU8kYu4nU+w5VrwhzAj6K github.com/calypr/syfon/client v0.1.5/go.mod h1:LA+07PI2ztRlLND0lxOSjgkBm62CT6+02Cc/iBjVPaY= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0Hw= github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -320,53 +571,144 @@ github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1 github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gofiber/schema v1.8.0/go.mod h1:lmbXPQ8hvzXSLkdS2DS7pb4kpunC2Roh7Sj3HMjGfzA= +github.com/gofiber/utils/v2 v2.1.1/go.mod h1:DdOgEVwQTi8cou/AKWPqhXOR4fHGRVhA/rEWL3IXG7Q= +github.com/gofiber/utils/v2 v2.1.2/go.mod h1:DdOgEVwQTi8cou/AKWPqhXOR4fHGRVhA/rEWL3IXG7Q= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 h1:EcQR3gusLHN46TAD+G+EbaaqJArt5vHhNpXAa12PQf4= github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/gomarkdown/markdown v0.0.0-20240328165702-4d01890c35c0/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td8= github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d h1:c93kUJDtVAXFEhsCh5jSxyOJmFHuzcihnslQiX8Urwo= github.com/kataras/blocks v0.0.7 h1:cF3RDY/vxnSRezc7vLFlQFTYXG/yAr1o7WImJuZbzC4= github.com/kataras/blocks v0.0.7/go.mod h1:UJIU97CluDo0f+zEjbnbkeMRlvYORtmc1304EeyXf4I= +github.com/kataras/blocks v0.0.8/go.mod h1:9Jm5zx6BB+06NwA+OhTbHW1xkMOYxahnqTN5DveZ2Yg= github.com/kataras/golog v0.1.9 h1:vLvSDpP7kihFGKFAvBSofYo7qZNULYSHOH2D7rPTKJk= github.com/kataras/golog v0.1.9/go.mod h1:jlpk/bOaYCyqDqH18pgDHdaJab72yBE6i0O3s30hpWY= +github.com/kataras/golog v0.1.11/go.mod h1:mAkt1vbPowFUuUGvexyQ5NFW6djEgGyxQBIARJ0AH4A= github.com/kataras/iris/v12 v12.2.6-0.20230908161203-24ba4e8933b9 h1:Vx8kDVhO2qepK8w44lBtp+RzN3ld743i+LYPzODJSpQ= github.com/kataras/iris/v12 v12.2.6-0.20230908161203-24ba4e8933b9/go.mod h1:ldkoR3iXABBeqlTibQ3MYaviA1oSlPvim6f55biwBh4= +github.com/kataras/iris/v12 v12.2.11/go.mod h1:uMAeX8OqG9vqdhyrIPv8Lajo/wXTtAF43wchP9WHt2w= github.com/kataras/pio v0.0.12 h1:o52SfVYauS3J5X08fNjlGS5arXHjW/ItLkyLcKjoH6w= github.com/kataras/pio v0.0.12/go.mod h1:ODK/8XBhhQ5WqrAhKy+9lTPS7sBf6O3KcLhc9klfRcY= +github.com/kataras/pio v0.0.13/go.mod h1:k3HNuSw+eJ8Pm2lA4lRhg3DiCjVgHlP8hmXApSej3oM= github.com/kataras/sitemap v0.0.6 h1:w71CRMMKYMJh6LR2wTgnk5hSgjVNB9KL60n5e2KHvLY= github.com/kataras/sitemap v0.0.6/go.mod h1:dW4dOCNs896OR1HmG+dMLdT7JjDk7mYBzoIRwuj5jA4= github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA= github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46 h1:veS9QfglfvqAw2e+eeNT/SbGySq8ajECXJ9e4fPoLhY= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labstack/echo/v4 v4.15.1 h1:S9keusg26gZpjMmPqB5hOEvNKnmd1lNmcHrbbH2lnFs= @@ -375,13 +717,16 @@ github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0 github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.12.0/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/lyft/protoc-gen-star/v2 v2.0.4 h1:JDlNKttNIRd68AAIychs0AqEpO8/I/WYi01OQ7Raw6Q= github.com/lyft/protoc-gen-star/v2 v2.0.4/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= +github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/moby/sys/mount v0.3.4 h1:yn5jq4STPztkkzSKpZkLcmjue+bZJ0u2AuQY1iNI1Ww= github.com/moby/sys/mount v0.3.4/go.mod h1:KcQJMbQdJHPlq5lcYT+/CjatWM4PuxKe+XLSVS4J6Os= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= @@ -396,10 +741,18 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU= github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -407,12 +760,16 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6Ng github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= +github.com/shamaton/msgpack/v3 v3.1.2/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc= +github.com/shamaton/msgpack/v3 v3.2.0/go.mod h1:sgBYvEiyz8JR1NC3yGRoPVME9xXovpnh3l/plW1nfRo= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.1/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad h1:fiWzISvDn0Csy5H0iwgAuJGQTUpVfEMJJd4nRFXogbc= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -420,38 +777,70 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tdewolff/minify/v2 v2.12.9 h1:dvn5MtmuQ/DFMwqf5j8QhEVpPX6fi3WGImhv8RUB4zA= github.com/tdewolff/minify/v2 v2.12.9/go.mod h1:qOqdlDfL+7v0/fyymB+OP497nIxJYSvX4MQWA8OoiXU= +github.com/tdewolff/minify/v2 v2.20.19/go.mod h1:ulkFoeAVWMLEyjuDz1ZIWOA31g5aWOawCFRp9R/MudM= github.com/tdewolff/parse/v2 v2.6.8 h1:mhNZXYCx//xG7Yq2e/kVLNZw4YfYmeHbhx+Zc0OvFMA= github.com/tdewolff/parse/v2 v2.6.8/go.mod h1:XHDhaU6IBgsryfdnpzUXBlT6leW/l25yrFBTEb4eIyM= +github.com/tdewolff/parse/v2 v2.7.12/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA= github.com/tdewolff/test v1.0.9/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= +github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA= github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/detectors/aws/ec2 v1.38.0 h1:gSqtaXUzONG+J3PutTnKpLVLpKLsX+FztJwxSFQ15PU= go.opentelemetry.io/contrib/detectors/aws/ec2 v1.38.0/go.mod h1:AqLDNPbKVFwdXy2/Xu2EYElVHO7ghhbEhKCCWymjpMI= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/contrib/propagators/aws v1.38.0 h1:eRZ7asSbLc5dH7+TBzL6hFKb1dabz0IV51uUUwYRZts= go.opentelemetry.io/contrib/propagators/aws v1.38.0/go.mod h1:wXqc9NTGcXapBExHBDVLEZlByu6quiQL8w7Tjgv8TCg= +go.opentelemetry.io/contrib/propagators/aws v1.42.0/go.mod h1:Jzw9hZHtxdpCN7x8S17UH59X/EiFivp6VXLs9bdM1OQ= +go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q= +go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -463,58 +852,444 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.272.0/go.mod h1:wKjowi5LNJc5qarNvDCvNQBn3rVK8nSy6jg2SwRwzIA= +google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= +google.golang.org/api v0.280.0/go.mod h1:oGKmPZRDoD3vdkf6MA7F4VNkR1rxCiuaPSkhsf3EolU= +google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:RRHjglSYABVCWpQ7USCpdfhcd9t4PkajvVwyynZizTc= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:1dCETSCY2YKZNXQE3h4fun3TYwF5p8jejRKZgfWAgAY= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800/go.mod h1:FPk7EXUKMtImne7AmknoYjT4QXqKIzzRbeQIXzLk6fQ= google.golang.org/genproto/googleapis/bytestream v0.0.0-20260319201613-d00831a3d3e7 h1:6jJ8xNMxB3mmH3HsxEMPjJU+WeFOiwBJ+cLm60OvSZs= google.golang.org/genproto/googleapis/bytestream v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:6TABGosqSqU2l1+fJ3jdvOYPPVryeKybxYF0cCZkTBE= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20260630182238-925bb5da69e7/go.mod h1:6TABGosqSqU2l1+fJ3jdvOYPPVryeKybxYF0cCZkTBE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401001100-f93e5f3e9f0f/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds= google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From a8a633fc359aeb310b09f16eb52647f846e77562 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 15 Jul 2026 11:03:26 -0700 Subject: [PATCH 24/24] tidy --- go.mod | 7 -- go.sum | 208 +++++---------------------------------------------------- 2 files changed, 18 insertions(+), 197 deletions(-) diff --git a/go.mod b/go.mod index 46075780..ae3e3c3c 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,6 @@ require ( github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect @@ -86,7 +85,6 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v1.0.0 // indirect - github.com/go-openapi/swag/jsonname v0.27.0 // indirect github.com/gofiber/schema v1.8.2 // indirect github.com/gofiber/utils/v2 v2.2.0 // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -98,11 +96,9 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/yamux v0.1.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.19.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect - github.com/mailru/easyjson v0.9.2 // indirect github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.24 // indirect @@ -115,14 +111,12 @@ require ( github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oapi-codegen/runtime v1.5.0 // indirect github.com/oasdiff/yaml v0.1.1 // indirect github.com/oasdiff/yaml3 v0.0.14 // indirect github.com/oklog/run v1.2.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -138,7 +132,6 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.72.0 // indirect github.com/vbauerster/mpb/v8 v8.12.1 // indirect - github.com/woodsbury/decimal128 v1.4.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect diff --git a/go.sum b/go.sum index efd7da46..f94ec34a 100644 --- a/go.sum +++ b/go.sum @@ -1,46 +1,29 @@ -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= -cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ= cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/compute v1.64.0 h1:7MmuzeAxlG5MOG5PQD2NLtyYR6bWjkvGljRu7pByoRU= -cloud.google.com/go/compute v1.64.0/go.mod h1:eHhcRZ6vf70fQCS3VEsiWSh+nQ+tLvSMb7mwLQskgN0= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.9.0 h1:89wyjxT6DL4b5rk/Nk8eBC9DHqf+JiMstrn5IEYxFw4= -cloud.google.com/go/iam v1.9.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= -cloud.google.com/go/logging v1.15.0 h1:6ooUEBNT6jdWh2b36+iuPn6b/R9qN/tHCbvGS5255gg= -cloud.google.com/go/logging v1.15.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k= -cloud.google.com/go/longrunning v0.10.0 h1:4OWvp1BjCvoeSZTog3sRFDu6j4IrI9TI4/Y9N+8h25g= -cloud.google.com/go/longrunning v0.10.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= +cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= -cloud.google.com/go/monitoring v1.27.0 h1:BhYwMqao+e5Nn7JtWMM9m6zRtKtVUK6kJWMizXChkLU= -cloud.google.com/go/monitoring v1.27.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= -cloud.google.com/go/storage v1.62.1 h1:Os0G3XbUbjZumkpDUf2Y0rLoXJTCF1kU2kWUujKYXD8= -cloud.google.com/go/storage v1.62.1/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= cloud.google.com/go/storage v1.63.1 h1:CYXILV9G4CH0C18IQ9+V0h4XiqD2LhKnMLO0o7uJWNs= cloud.google.com/go/storage v1.63.1/go.mod h1:lWyAtwvDZHdL3k68WVKbESP6bmWaV23ZJJ/JEVw/ZaQ= -cloud.google.com/go/trace v1.13.0 h1:RfqsqPOiSCG8ql50UZt5F65KrVa1zbY9mJrO7xvZfbE= -cloud.google.com/go/trace v1.13.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= @@ -49,30 +32,20 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xu github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1 h1:edShSHV3DV90+kt+CMaEXEzR9QF7wFrPJxVGz2blMIU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 h1:yzIYdwuro811Z27D3T80Wkd3rqZzb0K43nner7Eh1yE= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 h1:O2sXMyJh8b7devAGdE+163xtRurt0RVpB6DIzX5vGfg= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0/go.mod h1:hEpiGU18xf70qb3jbTcIggWAiEfX/cOIVc2OTe4OegA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0 h1:ZYGajzJNcirVZpT1rltgf9iM+j9zZ4v8V9DrF+xKRJ8= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0/go.mod h1:PDQyYBOzGtQgvshQI//UiXyzuMHCz0ndyu+4W8X82vM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0 h1:ZIT85vKP7LBS84XJ0WdJ3dPOX3iz4j3c0+lpajGQMyo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0/go.mod h1:rqP9UEhOXv9WhQ7Gjz+G5y/pf8+BJZW5/Ts0AhE0PwE= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.58.0 h1:IBF8BbhKJkMsON/eY+LMu3aF3XMiotCb9KvkUmEkOJo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 h1:0YP0+/ixwu+Uqeu/FGiBZNQ19huiUxxiPXIc9WsLKuQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0/go.mod h1:6ZZMQhZKDvUvkJw2rc+oDP90tMMzuU/J+5HG1ZmPOmE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.58.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 h1:SBZzZCiPmDrUV7NSCWY54OnKikO/oTydPCvyEyYaDDE= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -82,90 +55,48 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= -github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek= github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= -github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= -github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= github.com/aws/aws-sdk-go-v2/config v1.32.30 h1:XwsEzpTJfQYJbFicz/QMLwAZdyeNVVoOEkbF7R3gPJk= github.com/aws/aws-sdk-go-v2/config v1.32.30/go.mod h1:Ud32SuMc+/9BGxfpSVld7HrE2o05JwKmXY4M3jOQNZU= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/credentials v1.19.29 h1:WHZGssHH887cO0ox07SIQZsFx3MKD4ps6w0xUEmnKYQ= github.com/aws/aws-sdk-go-v2/credentials v1.19.29/go.mod h1:Mhl0xR6zjguiuj00XRx2wMx22sAltk7oya39sT7fdg8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I= -github.com/aws/aws-sdk-go-v2/service/kms v1.50.4 h1:PgD1y0ZagPokGIZPmejCBUySBzOFDN+leZxCOfb1OEQ= -github.com/aws/aws-sdk-go-v2/service/kms v1.50.4/go.mod h1:FfXDb5nXrsoGgxsBFxwxr3vdHXheC2tV+6lmuLghhjQ= github.com/aws/aws-sdk-go-v2/service/kms v1.54.1 h1:aeJAJyvWS3gQ679pJbz8ZdOh3MViD1zvEdoZMVEawbg= github.com/aws/aws-sdk-go-v2/service/kms v1.54.1/go.mod h1:0RXNc6Yf3AvSMldGD6Lcch96Ojlw2TtGnHsqfD/L4u8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1 h1:LkBKxAOE5WXjlFuFZqPG1rREnl6I6QCMElcXFDEidos= github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= github.com/aws/aws-sdk-go-v2/service/signin v1.4.1 h1:V7ZZ300WPXGjvkyore5DGe0ljVPOxCXie/thWdtSBXE= github.com/aws/aws-sdk-go-v2/service/signin v1.4.1/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= github.com/aws/aws-sdk-go-v2/service/sso v1.32.1 h1:gYFYh4iLLcAOJRLNPY2aD2g9DIhKn4eof8UkIrr1rTk= github.com/aws/aws-sdk-go-v2/service/sso v1.32.1/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1 h1:arjT9Cm3/WYbGmD5TUZHk4UQn4Lle1fUNZs5FC6CtF0= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= github.com/aws/aws-sdk-go-v2/service/sts v1.44.1 h1:RvfHDg+xvAeZ+5741vUEjpOVtYSIm93W2zhx10Xtydw= github.com/aws/aws-sdk-go-v2/service/sts v1.44.1/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q= -github.com/aws/smithy-go v1.24.3 h1:XgOAaUgx+HhVBoP4v8n6HCQoTRDhoMghKqw4LNHsDNg= -github.com/aws/smithy-go v1.24.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= @@ -202,6 +133,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -217,19 +150,12 @@ github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJP github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78= -github.com/getkin/kin-openapi v0.135.0 h1:751SjYfbiwqukYuVjwYEIKNfrSwS5YpA7DZnKSwQgtg= -github.com/getkin/kin-openapi v0.135.0/go.mod h1:6dd5FJl6RdX4usBtFBaQhk9q62Yb2J0Mk5IhUO/QqFI= +github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/getkin/kin-openapi v0.142.0 h1:izj0vBdFprMhitfzaX8sTqztsEQyvwhssBoB6n8NO7w= github.com/getkin/kin-openapi v0.142.0/go.mod h1:3BH9M9XDe/y9M5DSvEocVYAYq1w0qrhJHjC/vZi0AaY= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= @@ -241,29 +167,14 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= -github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= -github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= -github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= -github.com/go-openapi/swag/jsonname v0.27.0 h1:4QVB//CKOdE8IOiBg19JNY2wfDS48MhesIquYBy2rUE= -github.com/go-openapi/swag/jsonname v0.27.0/go.mod h1:I1YsyvvhBuZsFXSW6I7ODfdyq13p7hDil//1T9/pFFk= -github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= -github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gofiber/fiber/v3 v3.1.0 h1:1p4I820pIa+FGxfwWuQZ5rAyX0WlGZbGT6Hnuxt6hKY= -github.com/gofiber/fiber/v3 v3.1.0/go.mod h1:n2nYQovvL9z3Too/FGOfgtERjW3GQcAUqgfoezGBZdU= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/gofiber/fiber/v3 v3.4.0 h1:F0aND4vwZF7dR7cbvSwFQQEpBU902XHKWxrLsFBkVqw= github.com/gofiber/fiber/v3 v3.4.0/go.mod h1:nAhJfdxUIJJph2tPWPmqWf8QDIN2iiqQiQf3lENZpdk= -github.com/gofiber/schema v1.7.0 h1:yNM+FNRZjyYEli9Ey0AXRBrAY9jTnb+kmGs3lJGPvKg= -github.com/gofiber/schema v1.7.0/go.mod h1:A/X5Ffyru4p9eBdp99qu+nzviHzQiZ7odLT+TwxWhbk= github.com/gofiber/schema v1.8.2 h1:wq+LO2xEGlsqma/8Akp9PUebQ6vcsYmF0xYQ4F2ijvU= github.com/gofiber/schema v1.8.2/go.mod h1:iyAMJztdyky7Pk2U7bwUP3EHjzMqUNjZ/hglE0nYO/g= -github.com/gofiber/utils/v2 v2.0.2 h1:ShRRssz0F3AhTlAQcuEj54OEDtWF7+HJDwEi/aa6QLI= -github.com/gofiber/utils/v2 v2.0.2/go.mod h1:+9Ub4NqQ+IaJoTliq5LfdmOJAA/Hzwf4pXOxOa3RrJ0= github.com/gofiber/utils/v2 v2.2.0 h1:YSSmCzQponq/f9uSOg2HtXC5qK1Dmor0o6DqaQVz8GE= github.com/gofiber/utils/v2 v2.2.0/go.mod h1:Ieopk6sQh7rbhQ12aBNCJtJuG0gxAg0nz63sFCrrOmE= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= @@ -281,12 +192,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= -github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= -github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= -github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl/wMbiI= -github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4= github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= @@ -295,8 +202,6 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-plugin v1.7.0 h1:YghfQH/0QmPNc/AZMTFE3ac8fipZyZECHdDPshfk+mA= -github.com/hashicorp/go-plugin v1.7.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= github.com/hashicorp/go-plugin v1.8.0 h1:ie8S6RRY8RvB2usYZv+AAZ/wBvx2AU5p5QeP5j/FORs= github.com/hashicorp/go-plugin v1.8.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= @@ -307,12 +212,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -327,26 +228,16 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.9.2 h1:dX8U45hQsZpxd80nLvDGihsQ/OxlvTkVUXH2r/8cb2M= -github.com/mailru/easyjson v0.9.2/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= -github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= -github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= -github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= -github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/mattn/go-sqlite3 v1.14.48 h1:7XHIgl0a8HwOaiK4E47ozLkST78rR9+OtNGx27D/TFs= github.com/mattn/go-sqlite3 v1.14.48/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -367,30 +258,20 @@ github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/oapi-codegen/runtime v1.4.0 h1:KLOSFOp7UzkbS7Cs1ms6NBEKYr0WmH2wZG0KKbd2er4= -github.com/oapi-codegen/runtime v1.4.0/go.mod h1:5sw5fxCDmnOzKNYmkVNF8d34kyUeejJEY8HNT2WaPec= +github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= +github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= github.com/oapi-codegen/runtime v1.5.0 h1:aiil4QnH+eiWYSO60eaYZ4aur7sJH3rz6BvT5EBFnxc= github.com/oapi-codegen/runtime v1.5.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= -github.com/oasdiff/yaml v0.0.9 h1:zQOvd2UKoozsSsAknnWoDJlSK4lC0mpmjfDsfqNwX48= -github.com/oasdiff/yaml v0.0.9/go.mod h1:8lvhgJG4xiKPj3HN5lDow4jZHPlx1i7dIwzkdAo6oAM= github.com/oasdiff/yaml v0.1.1 h1:6nHx+pn9gBRM6YpBlFZFQGCCd1nuvqOBtTD3KKTgGxY= github.com/oasdiff/yaml v0.1.1/go.mod h1:EYJNoyktvWMJ0Hmhx+6qTaqMOsalUaRGT8Sj1hNcegU= -github.com/oasdiff/yaml3 v0.0.9 h1:rWPrKccrdUm8J0F3sGuU+fuh9+1K/RdJlWF7O/9yw2g= -github.com/oasdiff/yaml3 v0.0.9/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= github.com/oasdiff/yaml3 v0.0.14 h1:aLJee3hxBK2H5wdXd9iPcIXb93Nty1Ge0pT171eHtkw= github.com/oasdiff/yaml3 v0.0.14/go.mod h1:csto2xfDjYccdUn/yw/bPjj/cYTdp6HtFA0J4TWG+gg= -github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= -github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/run v1.2.0 h1:O8x3yXwah4A73hJdlrwo/2X6J62gE5qTMusH0dvz60E= github.com/oklog/run v1.2.0/go.mod h1:mgDbKRSwPhJfesJ4PntqFUbKQRZ50NgmZTSPlFA0YFk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= -github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -407,9 +288,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/shamaton/msgpack/v3 v3.1.0 h1:jsk0vEAqVvvS9+fTZ5/EcQ9tz860c9pWxJ4Iwecz8gU= -github.com/shamaton/msgpack/v3 v3.1.0/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc= github.com/shamaton/msgpack/v3 v3.2.0 h1:1q2Ms+MWmuRju+PuDMSFDB7p7621npeX4zprJN5Zck8= +github.com/shamaton/msgpack/v3 v3.2.0/go.mod h1:sgBYvEiyz8JR1NC3yGRoPVME9xXovpnh3l/plW1nfRo= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= @@ -419,8 +299,6 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= -github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= @@ -439,28 +317,18 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= -github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= -github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI= -github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw= github.com/valyala/fasthttp v1.72.0 h1:R7kYdoWhn1ye1fVpP+cDHDJwYm3NkwLliwgzJ/Abg7M= github.com/valyala/fasthttp v1.72.0/go.mod h1:zsbLTYqcpIktdQytlVBwIjY9La5d6bs990nBxWg8efk= -github.com/vbauerster/mpb/v8 v8.12.0 h1:+gneY3ifzc88tKDzOtfG8k8gfngCx615S2ZmFM4liWg= -github.com/vbauerster/mpb/v8 v8.12.0/go.mod h1:V02YIuMVo301Y1VE9VtZlD8s84OMsk+EKN6mwvf/588= github.com/vbauerster/mpb/v8 v8.12.1 h1:pyj3yQ2ZGQJgUXm4h17QpR+eERaNz5OQ1ftPSEE/sMM= github.com/vbauerster/mpb/v8 v8.12.1/go.mod h1:XLXRfStkw/6i5k0aQltijDHT1Z93fD1DVwmIdcFUp6k= -github.com/woodsbury/decimal128 v1.4.0 h1:xJATj7lLu4f2oObouMt2tgGiElE5gO6mSWUjQsBgUlc= -github.com/woodsbury/decimal128 v1.4.0/go.mod h1:BP46FUrVjVhdTbKT+XuQh2xfQaGki9LMIRJSFuh6THU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -469,58 +337,35 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= -go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0/go.mod h1:D7J12YRapIekYyPWgGPlA/23pRmpSEZC5xJC/TTLI9U= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0 h1:hqxVTu/GtBF+vJ8d1fzW7fRxZFvgoDjWcxwwCaFDYpU= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY= -gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI= gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -531,15 +376,10 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -549,24 +389,14 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.276.0 h1:nVArUtfLEihtW+b0DdcqRGK1xoEm2+ltAihyztq7MKY= -google.golang.org/api v0.276.0/go.mod h1:Fnag/EWUPIcJXuIkP1pjoTgS5vdxlk3eeemL7Do6bvw= google.golang.org/api v0.288.0 h1:glhO/J88obKP5I269W3hB73dvBKrjU56ZfmNlNXpgTU= google.golang.org/api v0.288.0/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= -google.golang.org/genproto v0.0.0-20260414002931-afd174a4e478 h1:aLsVTW0lZ8+IY5u/ERjZSCvAmhuR7slKzyha3YikDNA= -google.golang.org/genproto v0.0.0-20260414002931-afd174a4e478/go.mod h1:YJAzKjfHIUHb9T+bfu8L7mthAp7VVXQBUs1PLdBWS7M= google.golang.org/genproto v0.0.0-20260713224248-f5fc221cf8c4 h1:PWVFocb6Y93aqxY7m9ArHu5zQiLkIrXBLr+ajQxRHH0= google.golang.org/genproto v0.0.0-20260713224248-f5fc221cf8c4/go.mod h1:v47/Kzm6KzB/2uGxuDjGww+7Dk7ny/VFIaIw4WxEtu0= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= google.golang.org/genproto/googleapis/api v0.0.0-20260713224248-f5fc221cf8c4 h1:lI0NbdWVmT6lOJJNDd7vyeTdfxP/7ouCLSJUKNNXa0k= google.golang.org/genproto/googleapis/api v0.0.0-20260713224248-f5fc221cf8c4/go.mod h1:WRrQ7/7N19PypuT0fxLOL5Lq0waoiRri4FbtHDEKrGE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/genproto/googleapis/rpc v0.0.0-20260713224248-f5fc221cf8c4 h1:7RtFDizMtT9eZzHzKxifoMGfcDBBy+LYZlgfg24ZmOM= google.golang.org/genproto/googleapis/rpc v0.0.0-20260713224248-f5fc221cf8c4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= @@ -574,8 +404,6 @@ google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= -gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=