diff --git a/Dockerfile b/Dockerfile index 8692dbcf..d831c8ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.7 -FROM golang:1.26.4-alpine AS builder +FROM golang:1.26.5-alpine AS builder RUN apk add --no-cache build-base git WORKDIR /src diff --git a/apigen/client/internalapi/internal.gen.go b/apigen/client/internalapi/internal.gen.go index 11f0ae9c..9600b4ae 100644 --- a/apigen/client/internalapi/internal.gen.go +++ b/apigen/client/internalapi/internal.gen.go @@ -32,8 +32,7 @@ type BulkDocumentsRequest0 = []string // BulkDocumentsRequest1 defines model for . type BulkDocumentsRequest1 struct { - Dids *[]string `json:"dids,omitempty"` - Ids *[]string `json:"ids,omitempty"` + Ids *[]string `json:"ids,omitempty"` } // BulkHashesRequest defines model for BulkHashesRequest. @@ -41,6 +40,19 @@ type BulkHashesRequest struct { Hashes []string `json:"hashes"` } +// BulkMissingSHA256Request defines model for BulkMissingSHA256Request. +type BulkMissingSHA256Request struct { + Organization string `json:"organization"` + Project string `json:"project"` + Sha256 []string `json:"sha256"` +} + +// BulkMissingSHA256Response defines model for BulkMissingSHA256Response. +type BulkMissingSHA256Response struct { + Checked int32 `json:"checked"` + MissingSha256 []string `json:"missing_sha256"` +} + // BulkSHA256ValidityRequest defines model for BulkSHA256ValidityRequest. type BulkSHA256ValidityRequest struct { Hashes *[]string `json:"hashes,omitempty"` @@ -60,12 +72,6 @@ type DeleteByQueryResponse struct { // HashInfo Hash map, e.g. {"sha256":"..."} type HashInfo map[string]string -// IndexDirectory defines model for IndexDirectory. -type IndexDirectory struct { - Name string `json:"name"` - Path string `json:"path"` -} - // InternalMultipartCompleteRequest defines model for InternalMultipartCompleteRequest. type InternalMultipartCompleteRequest struct { Bucket *string `json:"bucket,omitempty"` @@ -82,8 +88,8 @@ type InternalMultipartInitOutput struct { // InternalMultipartInitRequest defines model for InternalMultipartInitRequest. type InternalMultipartInitRequest struct { - FileName *string `json:"file_name,omitempty"` Guid *string `json:"guid,omitempty"` + Key *string `json:"key,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` } @@ -107,6 +113,64 @@ type InternalMultipartUploadRequest struct { UploadId string `json:"uploadId"` } +// InternalProjectBucketInventoryItem defines model for InternalProjectBucketInventoryItem. +type InternalProjectBucketInventoryItem struct { + Bucket *string `json:"bucket,omitempty"` + Etag *string `json:"etag,omitempty"` + Key *string `json:"key,omitempty"` + LastModified *string `json:"last_modified,omitempty"` + MetaSha256 *string `json:"meta_sha256,omitempty"` + ObjectUrl *string `json:"object_url,omitempty"` + Path *string `json:"path,omitempty"` + Provider *string `json:"provider,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` +} + +// InternalProjectBucketInventoryRequest defines model for InternalProjectBucketInventoryRequest. +type InternalProjectBucketInventoryRequest struct { + Organization *string `json:"organization,omitempty"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project *string `json:"project,omitempty"` +} + +// InternalProjectBucketInventoryResponse defines model for InternalProjectBucketInventoryResponse. +type InternalProjectBucketInventoryResponse struct { + Items *[]InternalProjectBucketInventoryItem `json:"items,omitempty"` + Summary *InternalProjectBucketInventorySummary `json:"summary,omitempty"` +} + +// InternalProjectBucketInventorySummary defines model for InternalProjectBucketInventorySummary. +type InternalProjectBucketInventorySummary struct { + Bucket *string `json:"bucket,omitempty"` + ComputedAt *string `json:"computed_at,omitempty"` + Exists *bool `json:"exists,omitempty"` + Mode *string `json:"mode,omitempty"` + ObjectCount *int `json:"object_count,omitempty"` + ObjectUrl *string `json:"object_url,omitempty"` + Prefix *string `json:"prefix,omitempty"` + Provider *string `json:"provider,omitempty"` + TotalBytes *int64 `json:"total_bytes,omitempty"` +} + +// InternalProjectScopeItem defines model for InternalProjectScopeItem. +type InternalProjectScopeItem struct { + Bucket *string `json:"bucket,omitempty"` + Organization *string `json:"organization,omitempty"` + Path *string `json:"path,omitempty"` + ProjectId *string `json:"project_id,omitempty"` +} + +// InternalProjectScopesRequest defines model for InternalProjectScopesRequest. +type InternalProjectScopesRequest struct { + Organization string `json:"organization"` + Project string `json:"project"` +} + +// InternalProjectScopesResponse defines model for InternalProjectScopesResponse. +type InternalProjectScopesResponse struct { + Items *[]InternalProjectScopeItem `json:"items,omitempty"` +} + // InternalRecord defines model for InternalRecord. type InternalRecord struct { AccessMethods *[]externalRef0.AccessMethod `json:"access_methods,omitempty"` @@ -114,11 +178,11 @@ type InternalRecord struct { CreatedTime *string `json:"created_time,omitempty"` Description *string `json:"description,omitempty"` Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` Name *string `json:"name,omitempty"` + NameAliases *[]string `json:"name_aliases,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` Size *int64 `json:"size,omitempty"` @@ -135,11 +199,11 @@ type InternalRecordResponse struct { CreatedTime *string `json:"created_time,omitempty"` Description *string `json:"description,omitempty"` Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` Name *string `json:"name,omitempty"` + NameAliases *[]string `json:"name_aliases,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` Rev *string `json:"rev,omitempty"` @@ -173,7 +237,7 @@ type InternalUploadBlankRequest struct { type InternalUploadBulkItem struct { ExpiresIn *int32 `json:"expires_in,omitempty"` FileId string `json:"file_id"` - FileName *string `json:"file_name,omitempty"` + Key *string `json:"key,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` } @@ -190,18 +254,17 @@ type InternalUploadBulkRequest struct { // InternalUploadBulkResult defines model for InternalUploadBulkResult. type InternalUploadBulkResult struct { - Bucket *string `json:"bucket,omitempty"` - Error *string `json:"error,omitempty"` - FileId string `json:"file_id"` - FileName *string `json:"file_name,omitempty"` - Status int32 `json:"status"` - Url *string `json:"url,omitempty"` + Bucket *string `json:"bucket,omitempty"` + Error *string `json:"error,omitempty"` + FileId string `json:"file_id"` + Key *string `json:"key,omitempty"` + Status int32 `json:"status"` + Url *string `json:"url,omitempty"` } // ListRecordsResponse defines model for ListRecordsResponse. type ListRecordsResponse struct { - Directories *[]IndexDirectory `json:"directories,omitempty"` - Records *[]InternalRecord `json:"records,omitempty"` + Records *[]InternalRecord `json:"records,omitempty"` } // InternalDownloadParams defines parameters for InternalDownload. @@ -216,11 +279,17 @@ type InternalDownloadPartParams struct { End int64 `form:"end" json:"end"` } +// InternalInspectProjectScopesParams defines parameters for InternalInspectProjectScopes. +type InternalInspectProjectScopesParams struct { + Organization string `form:"organization" json:"organization"` + Project string `form:"project" json:"project"` +} + // InternalUploadURLParams defines parameters for InternalUploadURL. type InternalUploadURLParams struct { Organization *string `form:"organization,omitempty" json:"organization,omitempty"` Project *string `form:"project,omitempty" json:"project,omitempty"` - FileName *string `form:"file_name,omitempty" json:"file_name,omitempty"` + Key *string `form:"key,omitempty" json:"key,omitempty"` ExpiresIn *int32 `form:"expires_in,omitempty" json:"expires_in,omitempty"` } @@ -246,6 +315,12 @@ type InternalListParams struct { Page *int `form:"page,omitempty" json:"page,omitempty"` } +// InternalInspectProjectBucketInventoryJSONRequestBody defines body for InternalInspectProjectBucketInventory for application/json ContentType. +type InternalInspectProjectBucketInventoryJSONRequestBody = InternalProjectBucketInventoryRequest + +// InternalInspectProjectScopesPostJSONRequestBody defines body for InternalInspectProjectScopesPost for application/json ContentType. +type InternalInspectProjectScopesPostJSONRequestBody = InternalProjectScopesRequest + // InternalMultipartCompleteJSONRequestBody defines body for InternalMultipartComplete for application/json ContentType. type InternalMultipartCompleteJSONRequestBody = InternalMultipartCompleteRequest @@ -276,6 +351,9 @@ type InternalBulkDocumentsJSONRequestBody = BulkDocumentsRequest // InternalBulkHashesJSONRequestBody defines body for InternalBulkHashes for application/json ContentType. type InternalBulkHashesJSONRequestBody = BulkHashesRequest +// InternalBulkMissingSHA256JSONRequestBody defines body for InternalBulkMissingSHA256 for application/json ContentType. +type InternalBulkMissingSHA256JSONRequestBody = BulkMissingSHA256Request + // InternalBulkSHA256ValidityJSONRequestBody defines body for InternalBulkSHA256Validity for application/json ContentType. type InternalBulkSHA256ValidityJSONRequestBody = BulkSHA256ValidityRequest @@ -426,6 +504,19 @@ type ClientInterface interface { // InternalDownloadPart request InternalDownloadPart(ctx context.Context, fileId string, params *InternalDownloadPartParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalInspectProjectBucketInventoryWithBody request with any body + InternalInspectProjectBucketInventoryWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + InternalInspectProjectBucketInventory(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + + // InternalInspectProjectScopes request + InternalInspectProjectScopes(ctx context.Context, params *InternalInspectProjectScopesParams, reqEditors ...RequestEditorFn) (*http.Response, error) + + // InternalInspectProjectScopesPostWithBody request with any body + InternalInspectProjectScopesPostWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + InternalInspectProjectScopesPost(ctx context.Context, body InternalInspectProjectScopesPostJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalMultipartCompleteWithBody request with any body InternalMultipartCompleteWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -485,6 +576,11 @@ type ClientInterface interface { InternalBulkHashes(ctx context.Context, body InternalBulkHashesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalBulkMissingSHA256WithBody request with any body + InternalBulkMissingSHA256WithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + InternalBulkMissingSHA256(ctx context.Context, body InternalBulkMissingSHA256JSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // InternalBulkSHA256ValidityWithBody request with any body InternalBulkSHA256ValidityWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -531,6 +627,66 @@ func (c *Client) InternalDownloadPart(ctx context.Context, fileId string, params return c.Client.Do(req) } +func (c *Client) InternalInspectProjectBucketInventoryWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectBucketInventoryRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalInspectProjectBucketInventory(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectBucketInventoryRequest(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalInspectProjectScopes(ctx context.Context, params *InternalInspectProjectScopesParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectScopesRequest(c.Server, params) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalInspectProjectScopesPostWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectScopesPostRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalInspectProjectScopesPost(ctx context.Context, body InternalInspectProjectScopesPostJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalInspectProjectScopesPostRequest(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) InternalMultipartCompleteWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewInternalMultipartCompleteRequestWithBody(c.Server, contentType, body) if err != nil { @@ -807,6 +963,30 @@ func (c *Client) InternalBulkHashes(ctx context.Context, body InternalBulkHashes return c.Client.Do(req) } +func (c *Client) InternalBulkMissingSHA256WithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalBulkMissingSHA256RequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) InternalBulkMissingSHA256(ctx context.Context, body InternalBulkMissingSHA256JSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewInternalBulkMissingSHA256Request(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) InternalBulkSHA256ValidityWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewInternalBulkSHA256ValidityRequestWithBody(c.Server, contentType, body) if err != nil { @@ -1039,6 +1219,143 @@ func NewInternalDownloadPartRequest(server string, fileId string, params *Intern return req, nil } +// NewInternalInspectProjectBucketInventoryRequest calls the generic InternalInspectProjectBucketInventory builder with application/json body +func NewInternalInspectProjectBucketInventoryRequest(server string, body InternalInspectProjectBucketInventoryJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewInternalInspectProjectBucketInventoryRequestWithBody(server, "application/json", bodyReader) +} + +// NewInternalInspectProjectBucketInventoryRequestWithBody generates requests for InternalInspectProjectBucketInventory with any type of body +func NewInternalInspectProjectBucketInventoryRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/data/inspect/project-bucket/inventory") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +// NewInternalInspectProjectScopesRequest generates requests for InternalInspectProjectScopes +func NewInternalInspectProjectScopesRequest(server string, params *InternalInspectProjectScopesParams) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/data/inspect/project-scopes") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "organization", runtime.ParamLocationQuery, params.Organization); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "project", runtime.ParamLocationQuery, params.Project); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewInternalInspectProjectScopesPostRequest calls the generic InternalInspectProjectScopesPost builder with application/json body +func NewInternalInspectProjectScopesPostRequest(server string, body InternalInspectProjectScopesPostJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewInternalInspectProjectScopesPostRequestWithBody(server, "application/json", bodyReader) +} + +// NewInternalInspectProjectScopesPostRequestWithBody generates requests for InternalInspectProjectScopesPost with any type of body +func NewInternalInspectProjectScopesPostRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/data/inspect/project-scopes") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewInternalMultipartCompleteRequest calls the generic InternalMultipartComplete builder with application/json body func NewInternalMultipartCompleteRequest(server string, body InternalMultipartCompleteJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -1300,9 +1617,9 @@ func NewInternalUploadURLRequest(server string, fileId string, params *InternalU } - if params.FileName != nil { + if params.Key != nil { - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "file_name", runtime.ParamLocationQuery, *params.FileName); err != nil { + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "key", runtime.ParamLocationQuery, *params.Key); err != nil { return nil, err } else if parsed, err := url.ParseQuery(queryFrag); err != nil { return nil, err @@ -1833,6 +2150,46 @@ func NewInternalBulkHashesRequestWithBody(server string, contentType string, bod return req, nil } +// NewInternalBulkMissingSHA256Request calls the generic InternalBulkMissingSHA256 builder with application/json body +func NewInternalBulkMissingSHA256Request(server string, body InternalBulkMissingSHA256JSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewInternalBulkMissingSHA256RequestWithBody(server, "application/json", bodyReader) +} + +// NewInternalBulkMissingSHA256RequestWithBody generates requests for InternalBulkMissingSHA256 with any type of body +func NewInternalBulkMissingSHA256RequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/index/bulk/sha256/missing") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewInternalBulkSHA256ValidityRequest calls the generic InternalBulkSHA256Validity builder with application/json body func NewInternalBulkSHA256ValidityRequest(server string, body InternalBulkSHA256ValidityJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -2084,6 +2441,19 @@ type ClientWithResponsesInterface interface { // InternalDownloadPartWithResponse request InternalDownloadPartWithResponse(ctx context.Context, fileId string, params *InternalDownloadPartParams, reqEditors ...RequestEditorFn) (*InternalDownloadPartResponse, error) + // InternalInspectProjectBucketInventoryWithBodyWithResponse request with any body + InternalInspectProjectBucketInventoryWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalInspectProjectBucketInventoryResponse, error) + + InternalInspectProjectBucketInventoryWithResponse(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalInspectProjectBucketInventoryResponse, error) + + // InternalInspectProjectScopesWithResponse request + InternalInspectProjectScopesWithResponse(ctx context.Context, params *InternalInspectProjectScopesParams, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesResponse, error) + + // InternalInspectProjectScopesPostWithBodyWithResponse request with any body + InternalInspectProjectScopesPostWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesPostResponse, error) + + InternalInspectProjectScopesPostWithResponse(ctx context.Context, body InternalInspectProjectScopesPostJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesPostResponse, error) + // InternalMultipartCompleteWithBodyWithResponse request with any body InternalMultipartCompleteWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalMultipartCompleteResponse, error) @@ -2143,6 +2513,11 @@ type ClientWithResponsesInterface interface { InternalBulkHashesWithResponse(ctx context.Context, body InternalBulkHashesJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalBulkHashesResponse, error) + // InternalBulkMissingSHA256WithBodyWithResponse request with any body + InternalBulkMissingSHA256WithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalBulkMissingSHA256Response, error) + + InternalBulkMissingSHA256WithResponse(ctx context.Context, body InternalBulkMissingSHA256JSONRequestBody, reqEditors ...RequestEditorFn) (*InternalBulkMissingSHA256Response, error) + // InternalBulkSHA256ValidityWithBodyWithResponse request with any body InternalBulkSHA256ValidityWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalBulkSHA256ValidityResponse, error) @@ -2209,6 +2584,72 @@ func (r InternalDownloadPartResponse) StatusCode() int { return 0 } +type InternalInspectProjectBucketInventoryResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *InternalProjectBucketInventoryResponse +} + +// Status returns HTTPResponse.Status +func (r InternalInspectProjectBucketInventoryResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalInspectProjectBucketInventoryResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type InternalInspectProjectScopesResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *InternalProjectScopesResponse +} + +// Status returns HTTPResponse.Status +func (r InternalInspectProjectScopesResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalInspectProjectScopesResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type InternalInspectProjectScopesPostResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *InternalProjectScopesResponse +} + +// Status returns HTTPResponse.Status +func (r InternalInspectProjectScopesPostResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalInspectProjectScopesPostResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type InternalMultipartCompleteResponse struct { Body []byte HTTPResponse *http.Response @@ -2495,6 +2936,28 @@ func (r InternalBulkHashesResponse) StatusCode() int { return 0 } +type InternalBulkMissingSHA256Response struct { + Body []byte + HTTPResponse *http.Response + JSON200 *BulkMissingSHA256Response +} + +// Status returns HTTPResponse.Status +func (r InternalBulkMissingSHA256Response) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r InternalBulkMissingSHA256Response) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type InternalBulkSHA256ValidityResponse struct { Body []byte HTTPResponse *http.Response @@ -2622,6 +3085,49 @@ func (c *ClientWithResponses) InternalDownloadPartWithResponse(ctx context.Conte return ParseInternalDownloadPartResponse(rsp) } +// InternalInspectProjectBucketInventoryWithBodyWithResponse request with arbitrary body returning *InternalInspectProjectBucketInventoryResponse +func (c *ClientWithResponses) InternalInspectProjectBucketInventoryWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalInspectProjectBucketInventoryResponse, error) { + rsp, err := c.InternalInspectProjectBucketInventoryWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectBucketInventoryResponse(rsp) +} + +func (c *ClientWithResponses) InternalInspectProjectBucketInventoryWithResponse(ctx context.Context, body InternalInspectProjectBucketInventoryJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalInspectProjectBucketInventoryResponse, error) { + rsp, err := c.InternalInspectProjectBucketInventory(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectBucketInventoryResponse(rsp) +} + +// InternalInspectProjectScopesWithResponse request returning *InternalInspectProjectScopesResponse +func (c *ClientWithResponses) InternalInspectProjectScopesWithResponse(ctx context.Context, params *InternalInspectProjectScopesParams, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesResponse, error) { + rsp, err := c.InternalInspectProjectScopes(ctx, params, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectScopesResponse(rsp) +} + +// InternalInspectProjectScopesPostWithBodyWithResponse request with arbitrary body returning *InternalInspectProjectScopesPostResponse +func (c *ClientWithResponses) InternalInspectProjectScopesPostWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesPostResponse, error) { + rsp, err := c.InternalInspectProjectScopesPostWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectScopesPostResponse(rsp) +} + +func (c *ClientWithResponses) InternalInspectProjectScopesPostWithResponse(ctx context.Context, body InternalInspectProjectScopesPostJSONRequestBody, reqEditors ...RequestEditorFn) (*InternalInspectProjectScopesPostResponse, error) { + rsp, err := c.InternalInspectProjectScopesPost(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalInspectProjectScopesPostResponse(rsp) +} + // InternalMultipartCompleteWithBodyWithResponse request with arbitrary body returning *InternalMultipartCompleteResponse func (c *ClientWithResponses) InternalMultipartCompleteWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalMultipartCompleteResponse, error) { rsp, err := c.InternalMultipartCompleteWithBody(ctx, contentType, body, reqEditors...) @@ -2819,6 +3325,23 @@ func (c *ClientWithResponses) InternalBulkHashesWithResponse(ctx context.Context return ParseInternalBulkHashesResponse(rsp) } +// InternalBulkMissingSHA256WithBodyWithResponse request with arbitrary body returning *InternalBulkMissingSHA256Response +func (c *ClientWithResponses) InternalBulkMissingSHA256WithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalBulkMissingSHA256Response, error) { + rsp, err := c.InternalBulkMissingSHA256WithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalBulkMissingSHA256Response(rsp) +} + +func (c *ClientWithResponses) InternalBulkMissingSHA256WithResponse(ctx context.Context, body InternalBulkMissingSHA256JSONRequestBody, reqEditors ...RequestEditorFn) (*InternalBulkMissingSHA256Response, error) { + rsp, err := c.InternalBulkMissingSHA256(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseInternalBulkMissingSHA256Response(rsp) +} + // InternalBulkSHA256ValidityWithBodyWithResponse request with arbitrary body returning *InternalBulkSHA256ValidityResponse func (c *ClientWithResponses) InternalBulkSHA256ValidityWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*InternalBulkSHA256ValidityResponse, error) { rsp, err := c.InternalBulkSHA256ValidityWithBody(ctx, contentType, body, reqEditors...) @@ -2940,6 +3463,84 @@ func ParseInternalDownloadPartResponse(rsp *http.Response) (*InternalDownloadPar return response, nil } +// ParseInternalInspectProjectBucketInventoryResponse parses an HTTP response from a InternalInspectProjectBucketInventoryWithResponse call +func ParseInternalInspectProjectBucketInventoryResponse(rsp *http.Response) (*InternalInspectProjectBucketInventoryResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalInspectProjectBucketInventoryResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest InternalProjectBucketInventoryResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseInternalInspectProjectScopesResponse parses an HTTP response from a InternalInspectProjectScopesWithResponse call +func ParseInternalInspectProjectScopesResponse(rsp *http.Response) (*InternalInspectProjectScopesResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalInspectProjectScopesResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest InternalProjectScopesResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseInternalInspectProjectScopesPostResponse parses an HTTP response from a InternalInspectProjectScopesPostWithResponse call +func ParseInternalInspectProjectScopesPostResponse(rsp *http.Response) (*InternalInspectProjectScopesPostResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalInspectProjectScopesPostResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest InternalProjectScopesResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseInternalMultipartCompleteResponse parses an HTTP response from a InternalMultipartCompleteWithResponse call func ParseInternalMultipartCompleteResponse(rsp *http.Response) (*InternalMultipartCompleteResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -3275,6 +3876,32 @@ func ParseInternalBulkHashesResponse(rsp *http.Response) (*InternalBulkHashesRes return response, nil } +// ParseInternalBulkMissingSHA256Response parses an HTTP response from a InternalBulkMissingSHA256WithResponse call +func ParseInternalBulkMissingSHA256Response(rsp *http.Response) (*InternalBulkMissingSHA256Response, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &InternalBulkMissingSHA256Response{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest BulkMissingSHA256Response + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseInternalBulkSHA256ValidityResponse parses an HTTP response from a InternalBulkSHA256ValidityWithResponse call func ParseInternalBulkSHA256ValidityResponse(rsp *http.Response) (*InternalBulkSHA256ValidityResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/client/metricsapi/metrics.gen.go b/apigen/client/metricsapi/metrics.gen.go index 863b0183..5a117d21 100644 --- a/apigen/client/metricsapi/metrics.gen.go +++ b/apigen/client/metricsapi/metrics.gen.go @@ -59,12 +59,20 @@ type FileUsage struct { UploadCount *int64 `json:"upload_count,omitempty"` } +// FileUsageBulkRequest defines model for FileUsageBulkRequest. +type FileUsageBulkRequest struct { + InactiveDays *int `json:"inactive_days,omitempty"` + ObjectIds []string `json:"object_ids"` +} + // FileUsageSummary defines model for FileUsageSummary. type FileUsageSummary struct { - InactiveFileCount *int64 `json:"inactive_file_count,omitempty"` - TotalDownloads *int64 `json:"total_downloads,omitempty"` - TotalFiles *int64 `json:"total_files,omitempty"` - TotalUploads *int64 `json:"total_uploads,omitempty"` + InactiveFileCount *int64 `json:"inactive_file_count,omitempty"` + RecordCount *int64 `json:"record_count,omitempty"` + RecordLatestUpdatedTime *time.Time `json:"record_latest_updated_time"` + TotalDownloads *int64 `json:"total_downloads,omitempty"` + TotalFiles *int64 `json:"total_files,omitempty"` + TotalUploads *int64 `json:"total_uploads,omitempty"` } // MetricsListResponse defines model for MetricsListResponse. @@ -221,6 +229,15 @@ type ListMetricsFilesParams struct { Project *Project `form:"project,omitempty" json:"project,omitempty"` } +// BulkMetricsFilesParams defines parameters for BulkMetricsFiles. +type BulkMetricsFilesParams struct { + // Organization Organization/program scope filter. + Organization *Organization `form:"organization,omitempty" json:"organization,omitempty"` + + // Project Project scope filter. Requires organization when set. + Project *Project `form:"project,omitempty" json:"project,omitempty"` +} + // GetMetricsFileParams defines parameters for GetMetricsFile. type GetMetricsFileParams struct { // Organization Organization/program scope filter. @@ -298,6 +315,9 @@ type GetTransferSummaryParams struct { AllowStale *AllowStale `form:"allow_stale,omitempty" json:"allow_stale,omitempty"` } +// BulkMetricsFilesJSONRequestBody defines body for BulkMetricsFiles for application/json ContentType. +type BulkMetricsFilesJSONRequestBody = FileUsageBulkRequest + // RecordProviderTransferEventsJSONRequestBody defines body for RecordProviderTransferEvents for application/json ContentType. type RecordProviderTransferEventsJSONRequestBody = ProviderTransferEventsRequest @@ -377,6 +397,11 @@ type ClientInterface interface { // ListMetricsFiles request ListMetricsFiles(ctx context.Context, params *ListMetricsFilesParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // BulkMetricsFilesWithBody request with any body + BulkMetricsFilesWithBody(ctx context.Context, params *BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + BulkMetricsFiles(ctx context.Context, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetMetricsFile request GetMetricsFile(ctx context.Context, objectId string, params *GetMetricsFileParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -407,6 +432,30 @@ func (c *Client) ListMetricsFiles(ctx context.Context, params *ListMetricsFilesP return c.Client.Do(req) } +func (c *Client) BulkMetricsFilesWithBody(ctx context.Context, params *BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewBulkMetricsFilesRequestWithBody(c.Server, params, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) BulkMetricsFiles(ctx context.Context, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewBulkMetricsFilesRequest(c.Server, params, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) GetMetricsFile(ctx context.Context, objectId string, params *GetMetricsFileParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetMetricsFileRequest(c.Server, objectId, params) if err != nil { @@ -592,6 +641,84 @@ func NewListMetricsFilesRequest(server string, params *ListMetricsFilesParams) ( return req, nil } +// NewBulkMetricsFilesRequest calls the generic BulkMetricsFiles builder with application/json body +func NewBulkMetricsFilesRequest(server string, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewBulkMetricsFilesRequestWithBody(server, params, "application/json", bodyReader) +} + +// NewBulkMetricsFilesRequestWithBody generates requests for BulkMetricsFiles with any type of body +func NewBulkMetricsFilesRequestWithBody(server string, params *BulkMetricsFilesParams, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/index/v1/metrics/files/bulk") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if params.Organization != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "organization", runtime.ParamLocationQuery, *params.Organization); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.Project != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "project", runtime.ParamLocationQuery, *params.Project); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewGetMetricsFileRequest generates requests for GetMetricsFile func NewGetMetricsFileRequest(server string, objectId string, params *GetMetricsFileParams) (*http.Request, error) { var err error @@ -1303,6 +1430,11 @@ type ClientWithResponsesInterface interface { // ListMetricsFilesWithResponse request ListMetricsFilesWithResponse(ctx context.Context, params *ListMetricsFilesParams, reqEditors ...RequestEditorFn) (*ListMetricsFilesResponse, error) + // BulkMetricsFilesWithBodyWithResponse request with any body + BulkMetricsFilesWithBodyWithResponse(ctx context.Context, params *BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*BulkMetricsFilesResponse, error) + + BulkMetricsFilesWithResponse(ctx context.Context, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody, reqEditors ...RequestEditorFn) (*BulkMetricsFilesResponse, error) + // GetMetricsFileWithResponse request GetMetricsFileWithResponse(ctx context.Context, objectId string, params *GetMetricsFileParams, reqEditors ...RequestEditorFn) (*GetMetricsFileResponse, error) @@ -1343,6 +1475,28 @@ func (r ListMetricsFilesResponse) StatusCode() int { return 0 } +type BulkMetricsFilesResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *MetricsListResponse +} + +// Status returns HTTPResponse.Status +func (r BulkMetricsFilesResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r BulkMetricsFilesResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type GetMetricsFileResponse struct { Body []byte HTTPResponse *http.Response @@ -1462,6 +1616,23 @@ func (c *ClientWithResponses) ListMetricsFilesWithResponse(ctx context.Context, return ParseListMetricsFilesResponse(rsp) } +// BulkMetricsFilesWithBodyWithResponse request with arbitrary body returning *BulkMetricsFilesResponse +func (c *ClientWithResponses) BulkMetricsFilesWithBodyWithResponse(ctx context.Context, params *BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*BulkMetricsFilesResponse, error) { + rsp, err := c.BulkMetricsFilesWithBody(ctx, params, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseBulkMetricsFilesResponse(rsp) +} + +func (c *ClientWithResponses) BulkMetricsFilesWithResponse(ctx context.Context, params *BulkMetricsFilesParams, body BulkMetricsFilesJSONRequestBody, reqEditors ...RequestEditorFn) (*BulkMetricsFilesResponse, error) { + rsp, err := c.BulkMetricsFiles(ctx, params, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseBulkMetricsFilesResponse(rsp) +} + // GetMetricsFileWithResponse request returning *GetMetricsFileResponse func (c *ClientWithResponses) GetMetricsFileWithResponse(ctx context.Context, objectId string, params *GetMetricsFileParams, reqEditors ...RequestEditorFn) (*GetMetricsFileResponse, error) { rsp, err := c.GetMetricsFile(ctx, objectId, params, reqEditors...) @@ -1541,6 +1712,32 @@ func ParseListMetricsFilesResponse(rsp *http.Response) (*ListMetricsFilesRespons return response, nil } +// ParseBulkMetricsFilesResponse parses an HTTP response from a BulkMetricsFilesWithResponse call +func ParseBulkMetricsFilesResponse(rsp *http.Response) (*BulkMetricsFilesResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &BulkMetricsFilesResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest MetricsListResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseGetMetricsFileResponse parses an HTTP response from a GetMetricsFileWithResponse call func ParseGetMetricsFileResponse(rsp *http.Response) (*GetMetricsFileResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/apigen/openapi/internal.openapi.yaml b/apigen/openapi/internal.openapi.yaml index f0e53d17..c4509d38 100644 --- a/apigen/openapi/internal.openapi.yaml +++ b/apigen/openapi/internal.openapi.yaml @@ -342,6 +342,34 @@ paths: description: Content too large '500': description: Internal server error + /index/bulk/sha256/missing: + post: + tags: + - internal-index + operationId: internalBulkMissingSHA256 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/BulkMissingSHA256Request' + responses: + '200': + description: SHA-256 values that are not registered in the requested project + content: + application/json: + schema: + $ref: '#/components/schemas/BulkMissingSHA256Response' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '413': + description: Content too large + '500': + description: Internal server error /index/bulk/documents: post: tags: @@ -368,6 +396,95 @@ paths: description: Content too large '500': description: Internal server error + /data/inspect/project-scopes: + get: + tags: + - internal-data + operationId: internalInspectProjectScopes + description: List bucket scopes visible for a project. + parameters: + - in: query + name: organization + required: true + schema: + type: string + - in: query + name: project + required: true + schema: + type: string + responses: + '200': + description: Project bucket scopes + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectScopesResponse' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error + post: + tags: + - internal-data + operationId: internalInspectProjectScopesPost + description: Compatibility request-body form of project scope listing. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectScopesRequest' + responses: + '200': + description: Project bucket scopes + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectScopesResponse' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error + /data/inspect/project-bucket/inventory: + post: + tags: + - internal-data + operationId: internalInspectProjectBucketInventory + description: Recursively list objects under the configured project bucket scope for audit validation. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectBucketInventoryRequest' + responses: + '200': + description: Project bucket inventory + content: + application/json: + schema: + $ref: '#/components/schemas/InternalProjectBucketInventoryResponse' + '400': + description: Validation error + '401': + description: Authentication required + '403': + description: Forbidden + '404': + description: Bucket scope not found + '409': + description: Bucket inventory unavailable + '500': + description: Internal server error /data/upload: post: tags: @@ -417,7 +534,7 @@ paths: schema: type: string - in: query - name: file_name + name: key required: false schema: type: string @@ -657,6 +774,10 @@ components: minLength: 1 name: type: string + name_aliases: + type: array + items: + type: string hashes: $ref: '#/components/schemas/HashInfo' size: @@ -670,8 +791,6 @@ components: type: array items: $ref: './openapi.yaml#/components/schemas/AccessMethod' - file_name: - type: string organization: type: string project: @@ -702,24 +821,10 @@ components: ListRecordsResponse: type: object properties: - directories: - type: array - items: - $ref: '#/components/schemas/IndexDirectory' records: type: array items: $ref: '#/components/schemas/InternalRecord' - IndexDirectory: - type: object - required: - - name - - path - properties: - name: - type: string - path: - type: string BulkCreateRequest: type: object required: @@ -749,6 +854,34 @@ components: type: array items: type: string + BulkMissingSHA256Request: + type: object + required: + - organization + - project + - sha256 + properties: + organization: + type: string + project: + type: string + sha256: + type: array + items: + type: string + BulkMissingSHA256Response: + type: object + required: + - checked + - missing_sha256 + properties: + checked: + type: integer + format: int32 + missing_sha256: + type: array + items: + type: string BulkDocumentsRequest: oneOf: - type: array @@ -760,10 +893,6 @@ components: type: array items: type: string - dids: - type: array - items: - type: string DeleteByQueryResponse: type: object properties: @@ -774,6 +903,96 @@ components: properties: url: type: string + InternalProjectBucketInventoryRequest: + type: object + properties: + organization: + type: string + project: + type: string + path_prefix: + type: string + InternalProjectBucketInventoryResponse: + type: object + properties: + summary: + $ref: '#/components/schemas/InternalProjectBucketInventorySummary' + items: + type: array + items: + $ref: '#/components/schemas/InternalProjectBucketInventoryItem' + InternalProjectBucketInventorySummary: + type: object + properties: + provider: + type: string + bucket: + type: string + prefix: + type: string + object_url: + type: string + exists: + type: boolean + object_count: + type: integer + total_bytes: + type: integer + format: int64 + computed_at: + type: string + mode: + type: string + InternalProjectBucketInventoryItem: + type: object + properties: + object_url: + type: string + provider: + type: string + bucket: + type: string + key: + type: string + path: + type: string + size_bytes: + type: integer + format: int64 + meta_sha256: + type: string + etag: + type: string + last_modified: + type: string + InternalProjectScopesRequest: + type: object + required: + - organization + - project + properties: + organization: + type: string + project: + type: string + InternalProjectScopesResponse: + type: object + properties: + items: + type: array + items: + $ref: '#/components/schemas/InternalProjectScopeItem' + InternalProjectScopeItem: + type: object + properties: + bucket: + type: string + organization: + type: string + project_id: + type: string + path: + type: string InternalUploadBlankRequest: type: object properties: @@ -808,7 +1027,7 @@ components: properties: file_id: type: string - file_name: + key: type: string organization: type: string @@ -834,7 +1053,7 @@ components: type: string bucket: type: string - file_name: + key: type: string url: type: string @@ -848,7 +1067,7 @@ components: properties: guid: type: string - file_name: + key: type: string organization: type: string diff --git a/apigen/openapi/metrics.openapi.yaml b/apigen/openapi/metrics.openapi.yaml index 5d1c502e..4661a72c 100644 --- a/apigen/openapi/metrics.openapi.yaml +++ b/apigen/openapi/metrics.openapi.yaml @@ -49,6 +49,35 @@ paths: description: Forbidden '500': description: Internal server error + /index/v1/metrics/files/bulk: + post: + tags: + - metrics + operationId: bulkMetricsFiles + parameters: + - $ref: '#/components/parameters/Organization' + - $ref: '#/components/parameters/Project' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FileUsageBulkRequest' + responses: + '200': + description: File usage metrics for requested object ids + content: + application/json: + schema: + $ref: '#/components/schemas/MetricsListResponse' + '400': + description: Invalid request body + '401': + description: Authentication required + '403': + description: Forbidden + '500': + description: Internal server error /index/v1/metrics/files/{object_id}: get: tags: @@ -321,6 +350,26 @@ components: inactive_file_count: type: integer format: int64 + record_count: + type: integer + format: int64 + record_latest_updated_time: + type: string + format: date-time + nullable: true + FileUsageBulkRequest: + type: object + required: + - object_ids + properties: + object_ids: + type: array + minItems: 1 + items: + type: string + inactive_days: + type: integer + minimum: 0 MetricsListResponse: type: object properties: diff --git a/apigen/server/internalapi/internal.gen.go b/apigen/server/internalapi/internal.gen.go index 5dac272a..776b88fc 100644 --- a/apigen/server/internalapi/internal.gen.go +++ b/apigen/server/internalapi/internal.gen.go @@ -30,8 +30,7 @@ type BulkDocumentsRequest0 = []string // BulkDocumentsRequest1 defines model for . type BulkDocumentsRequest1 struct { - Dids *[]string `json:"dids,omitempty"` - Ids *[]string `json:"ids,omitempty"` + Ids *[]string `json:"ids,omitempty"` } // BulkHashesRequest defines model for BulkHashesRequest. @@ -39,6 +38,19 @@ type BulkHashesRequest struct { Hashes []string `json:"hashes"` } +// BulkMissingSHA256Request defines model for BulkMissingSHA256Request. +type BulkMissingSHA256Request struct { + Organization string `json:"organization"` + Project string `json:"project"` + Sha256 []string `json:"sha256"` +} + +// BulkMissingSHA256Response defines model for BulkMissingSHA256Response. +type BulkMissingSHA256Response struct { + Checked int32 `json:"checked"` + MissingSha256 []string `json:"missing_sha256"` +} + // BulkSHA256ValidityRequest defines model for BulkSHA256ValidityRequest. type BulkSHA256ValidityRequest struct { Hashes *[]string `json:"hashes,omitempty"` @@ -58,12 +70,6 @@ type DeleteByQueryResponse struct { // HashInfo Hash map, e.g. {"sha256":"..."} type HashInfo map[string]string -// IndexDirectory defines model for IndexDirectory. -type IndexDirectory struct { - Name string `json:"name"` - Path string `json:"path"` -} - // InternalMultipartCompleteRequest defines model for InternalMultipartCompleteRequest. type InternalMultipartCompleteRequest struct { Bucket *string `json:"bucket,omitempty"` @@ -80,8 +86,8 @@ type InternalMultipartInitOutput struct { // InternalMultipartInitRequest defines model for InternalMultipartInitRequest. type InternalMultipartInitRequest struct { - FileName *string `json:"file_name,omitempty"` Guid *string `json:"guid,omitempty"` + Key *string `json:"key,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` } @@ -105,6 +111,64 @@ type InternalMultipartUploadRequest struct { UploadId string `json:"uploadId"` } +// InternalProjectBucketInventoryItem defines model for InternalProjectBucketInventoryItem. +type InternalProjectBucketInventoryItem struct { + Bucket *string `json:"bucket,omitempty"` + Etag *string `json:"etag,omitempty"` + Key *string `json:"key,omitempty"` + LastModified *string `json:"last_modified,omitempty"` + MetaSha256 *string `json:"meta_sha256,omitempty"` + ObjectUrl *string `json:"object_url,omitempty"` + Path *string `json:"path,omitempty"` + Provider *string `json:"provider,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` +} + +// InternalProjectBucketInventoryRequest defines model for InternalProjectBucketInventoryRequest. +type InternalProjectBucketInventoryRequest struct { + Organization *string `json:"organization,omitempty"` + PathPrefix *string `json:"path_prefix,omitempty"` + Project *string `json:"project,omitempty"` +} + +// InternalProjectBucketInventoryResponse defines model for InternalProjectBucketInventoryResponse. +type InternalProjectBucketInventoryResponse struct { + Items *[]InternalProjectBucketInventoryItem `json:"items,omitempty"` + Summary *InternalProjectBucketInventorySummary `json:"summary,omitempty"` +} + +// InternalProjectBucketInventorySummary defines model for InternalProjectBucketInventorySummary. +type InternalProjectBucketInventorySummary struct { + Bucket *string `json:"bucket,omitempty"` + ComputedAt *string `json:"computed_at,omitempty"` + Exists *bool `json:"exists,omitempty"` + Mode *string `json:"mode,omitempty"` + ObjectCount *int `json:"object_count,omitempty"` + ObjectUrl *string `json:"object_url,omitempty"` + Prefix *string `json:"prefix,omitempty"` + Provider *string `json:"provider,omitempty"` + TotalBytes *int64 `json:"total_bytes,omitempty"` +} + +// InternalProjectScopeItem defines model for InternalProjectScopeItem. +type InternalProjectScopeItem struct { + Bucket *string `json:"bucket,omitempty"` + Organization *string `json:"organization,omitempty"` + Path *string `json:"path,omitempty"` + ProjectId *string `json:"project_id,omitempty"` +} + +// InternalProjectScopesRequest defines model for InternalProjectScopesRequest. +type InternalProjectScopesRequest struct { + Organization string `json:"organization"` + Project string `json:"project"` +} + +// InternalProjectScopesResponse defines model for InternalProjectScopesResponse. +type InternalProjectScopesResponse struct { + Items *[]InternalProjectScopeItem `json:"items,omitempty"` +} + // InternalRecord defines model for InternalRecord. type InternalRecord struct { AccessMethods *[]externalRef0.AccessMethod `json:"access_methods,omitempty"` @@ -112,11 +176,11 @@ type InternalRecord struct { CreatedTime *string `json:"created_time,omitempty"` Description *string `json:"description,omitempty"` Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` Name *string `json:"name,omitempty"` + NameAliases *[]string `json:"name_aliases,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` Size *int64 `json:"size,omitempty"` @@ -133,11 +197,11 @@ type InternalRecordResponse struct { CreatedTime *string `json:"created_time,omitempty"` Description *string `json:"description,omitempty"` Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` // Hashes Hash map, e.g. {"sha256":"..."} Hashes *HashInfo `json:"hashes,omitempty"` Name *string `json:"name,omitempty"` + NameAliases *[]string `json:"name_aliases,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` Rev *string `json:"rev,omitempty"` @@ -171,7 +235,7 @@ type InternalUploadBlankRequest struct { type InternalUploadBulkItem struct { ExpiresIn *int32 `json:"expires_in,omitempty"` FileId string `json:"file_id"` - FileName *string `json:"file_name,omitempty"` + Key *string `json:"key,omitempty"` Organization *string `json:"organization,omitempty"` Project *string `json:"project,omitempty"` } @@ -188,18 +252,17 @@ type InternalUploadBulkRequest struct { // InternalUploadBulkResult defines model for InternalUploadBulkResult. type InternalUploadBulkResult struct { - Bucket *string `json:"bucket,omitempty"` - Error *string `json:"error,omitempty"` - FileId string `json:"file_id"` - FileName *string `json:"file_name,omitempty"` - Status int32 `json:"status"` - Url *string `json:"url,omitempty"` + Bucket *string `json:"bucket,omitempty"` + Error *string `json:"error,omitempty"` + FileId string `json:"file_id"` + Key *string `json:"key,omitempty"` + Status int32 `json:"status"` + Url *string `json:"url,omitempty"` } // ListRecordsResponse defines model for ListRecordsResponse. type ListRecordsResponse struct { - Directories *[]IndexDirectory `json:"directories,omitempty"` - Records *[]InternalRecord `json:"records,omitempty"` + Records *[]InternalRecord `json:"records,omitempty"` } // InternalDownloadParams defines parameters for InternalDownload. @@ -214,11 +277,17 @@ type InternalDownloadPartParams struct { End int64 `form:"end" json:"end"` } +// InternalInspectProjectScopesParams defines parameters for InternalInspectProjectScopes. +type InternalInspectProjectScopesParams struct { + Organization string `form:"organization" json:"organization"` + Project string `form:"project" json:"project"` +} + // InternalUploadURLParams defines parameters for InternalUploadURL. type InternalUploadURLParams struct { Organization *string `form:"organization,omitempty" json:"organization,omitempty"` Project *string `form:"project,omitempty" json:"project,omitempty"` - FileName *string `form:"file_name,omitempty" json:"file_name,omitempty"` + Key *string `form:"key,omitempty" json:"key,omitempty"` ExpiresIn *int32 `form:"expires_in,omitempty" json:"expires_in,omitempty"` } @@ -244,6 +313,12 @@ type InternalListParams struct { Page *int `form:"page,omitempty" json:"page,omitempty"` } +// InternalInspectProjectBucketInventoryJSONRequestBody defines body for InternalInspectProjectBucketInventory for application/json ContentType. +type InternalInspectProjectBucketInventoryJSONRequestBody = InternalProjectBucketInventoryRequest + +// InternalInspectProjectScopesPostJSONRequestBody defines body for InternalInspectProjectScopesPost for application/json ContentType. +type InternalInspectProjectScopesPostJSONRequestBody = InternalProjectScopesRequest + // InternalMultipartCompleteJSONRequestBody defines body for InternalMultipartComplete for application/json ContentType. type InternalMultipartCompleteJSONRequestBody = InternalMultipartCompleteRequest @@ -274,6 +349,9 @@ type InternalBulkDocumentsJSONRequestBody = BulkDocumentsRequest // InternalBulkHashesJSONRequestBody defines body for InternalBulkHashes for application/json ContentType. type InternalBulkHashesJSONRequestBody = BulkHashesRequest +// InternalBulkMissingSHA256JSONRequestBody defines body for InternalBulkMissingSHA256 for application/json ContentType. +type InternalBulkMissingSHA256JSONRequestBody = BulkMissingSHA256Request + // InternalBulkSHA256ValidityJSONRequestBody defines body for InternalBulkSHA256Validity for application/json ContentType. type InternalBulkSHA256ValidityJSONRequestBody = BulkSHA256ValidityRequest @@ -354,6 +432,15 @@ type ServerInterface interface { // (GET /data/download/{file_id}/part) InternalDownloadPart(c fiber.Ctx, fileId string, params InternalDownloadPartParams) error + // (POST /data/inspect/project-bucket/inventory) + InternalInspectProjectBucketInventory(c fiber.Ctx) error + + // (GET /data/inspect/project-scopes) + InternalInspectProjectScopes(c fiber.Ctx, params InternalInspectProjectScopesParams) error + + // (POST /data/inspect/project-scopes) + InternalInspectProjectScopesPost(c fiber.Ctx) error + // (POST /data/multipart/complete) InternalMultipartComplete(c fiber.Ctx) error @@ -393,6 +480,9 @@ type ServerInterface interface { // (POST /index/bulk/hashes) InternalBulkHashes(c fiber.Ctx) error + // (POST /index/bulk/sha256/missing) + InternalBulkMissingSHA256(c fiber.Ctx) error + // (POST /index/bulk/sha256/validity) InternalBulkSHA256Validity(c fiber.Ctx) error @@ -498,6 +588,53 @@ func (siw *ServerInterfaceWrapper) InternalDownloadPart(c fiber.Ctx) error { return siw.Handler.InternalDownloadPart(c, fileId, params) } +// InternalInspectProjectBucketInventory operation middleware +func (siw *ServerInterfaceWrapper) InternalInspectProjectBucketInventory(c fiber.Ctx) error { + + return siw.Handler.InternalInspectProjectBucketInventory(c) +} + +// InternalInspectProjectScopes operation middleware +func (siw *ServerInterfaceWrapper) InternalInspectProjectScopes(c fiber.Ctx) error { + var err error + var params InternalInspectProjectScopesParams + + // ------------- Required query parameter "organization" ------------- + if paramValue := c.Query("organization"); paramValue != "" { + + var value string + err = runtime.BindStyledParameterWithOptions("form", "organization", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter organization: %w", err).Error()) + } + params.Organization = value + + } else { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument organization is required, but not found").Error()) + } + // ------------- Required query parameter "project" ------------- + if paramValue := c.Query("project"); paramValue != "" { + + var value string + err = runtime.BindStyledParameterWithOptions("form", "project", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: true}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter project: %w", err).Error()) + } + params.Project = value + + } else { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Query argument project is required, but not found").Error()) + } + + return siw.Handler.InternalInspectProjectScopes(c, params) +} + +// InternalInspectProjectScopesPost operation middleware +func (siw *ServerInterfaceWrapper) InternalInspectProjectScopesPost(c fiber.Ctx) error { + + return siw.Handler.InternalInspectProjectScopesPost(c) +} + // InternalMultipartComplete operation middleware func (siw *ServerInterfaceWrapper) InternalMultipartComplete(c fiber.Ctx) error { @@ -563,15 +700,15 @@ func (siw *ServerInterfaceWrapper) InternalUploadURL(c fiber.Ctx) error { params.Project = &value } - // ------------- Optional query parameter "file_name" ------------- - if paramValue := c.Query("file_name"); paramValue != "" { + // ------------- Optional query parameter "key" ------------- + if paramValue := c.Query("key"); paramValue != "" { var value string - err = runtime.BindStyledParameterWithOptions("form", "file_name", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + err = runtime.BindStyledParameterWithOptions("form", "key", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter file_name: %w", err).Error()) + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter key: %w", err).Error()) } - params.FileName = &value + params.Key = &value } // ------------- Optional query parameter "expires_in" ------------- @@ -791,6 +928,12 @@ func (siw *ServerInterfaceWrapper) InternalBulkHashes(c fiber.Ctx) error { return siw.Handler.InternalBulkHashes(c) } +// InternalBulkMissingSHA256 operation middleware +func (siw *ServerInterfaceWrapper) InternalBulkMissingSHA256(c fiber.Ctx) error { + + return siw.Handler.InternalBulkMissingSHA256(c) +} + // InternalBulkSHA256Validity operation middleware func (siw *ServerInterfaceWrapper) InternalBulkSHA256Validity(c fiber.Ctx) error { @@ -882,6 +1025,12 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Get(options.BaseURL+"/data/download/:file_id/part", wrapper.InternalDownloadPart) + router.Post(options.BaseURL+"/data/inspect/project-bucket/inventory", wrapper.InternalInspectProjectBucketInventory) + + router.Get(options.BaseURL+"/data/inspect/project-scopes", wrapper.InternalInspectProjectScopes) + + router.Post(options.BaseURL+"/data/inspect/project-scopes", wrapper.InternalInspectProjectScopesPost) + router.Post(options.BaseURL+"/data/multipart/complete", wrapper.InternalMultipartComplete) router.Post(options.BaseURL+"/data/multipart/init", wrapper.InternalMultipartInit) @@ -908,6 +1057,8 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Post(options.BaseURL+"/index/bulk/hashes", wrapper.InternalBulkHashes) + router.Post(options.BaseURL+"/index/bulk/sha256/missing", wrapper.InternalBulkMissingSHA256) + router.Post(options.BaseURL+"/index/bulk/sha256/validity", wrapper.InternalBulkSHA256Validity) router.Delete(options.BaseURL+"/index/:id", wrapper.InternalDelete) @@ -1044,6 +1195,169 @@ func (response InternalDownloadPart500Response) VisitInternalDownloadPartRespons return nil } +type InternalInspectProjectBucketInventoryRequestObject struct { + Body *InternalInspectProjectBucketInventoryJSONRequestBody +} + +type InternalInspectProjectBucketInventoryResponseObject interface { + VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error +} + +type InternalInspectProjectBucketInventory200JSONResponse InternalProjectBucketInventoryResponse + +func (response InternalInspectProjectBucketInventory200JSONResponse) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalInspectProjectBucketInventory400Response struct { +} + +func (response InternalInspectProjectBucketInventory400Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalInspectProjectBucketInventory401Response struct { +} + +func (response InternalInspectProjectBucketInventory401Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalInspectProjectBucketInventory403Response struct { +} + +func (response InternalInspectProjectBucketInventory403Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalInspectProjectBucketInventory404Response struct { +} + +func (response InternalInspectProjectBucketInventory404Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(404) + return nil +} + +type InternalInspectProjectBucketInventory409Response struct { +} + +func (response InternalInspectProjectBucketInventory409Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(409) + return nil +} + +type InternalInspectProjectBucketInventory500Response struct { +} + +func (response InternalInspectProjectBucketInventory500Response) VisitInternalInspectProjectBucketInventoryResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + +type InternalInspectProjectScopesRequestObject struct { + Params InternalInspectProjectScopesParams +} + +type InternalInspectProjectScopesResponseObject interface { + VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error +} + +type InternalInspectProjectScopes200JSONResponse InternalProjectScopesResponse + +func (response InternalInspectProjectScopes200JSONResponse) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalInspectProjectScopes400Response struct { +} + +func (response InternalInspectProjectScopes400Response) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalInspectProjectScopes401Response struct { +} + +func (response InternalInspectProjectScopes401Response) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalInspectProjectScopes403Response struct { +} + +func (response InternalInspectProjectScopes403Response) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalInspectProjectScopes500Response struct { +} + +func (response InternalInspectProjectScopes500Response) VisitInternalInspectProjectScopesResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + +type InternalInspectProjectScopesPostRequestObject struct { + Body *InternalInspectProjectScopesPostJSONRequestBody +} + +type InternalInspectProjectScopesPostResponseObject interface { + VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error +} + +type InternalInspectProjectScopesPost200JSONResponse InternalProjectScopesResponse + +func (response InternalInspectProjectScopesPost200JSONResponse) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalInspectProjectScopesPost400Response struct { +} + +func (response InternalInspectProjectScopesPost400Response) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalInspectProjectScopesPost401Response struct { +} + +func (response InternalInspectProjectScopesPost401Response) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalInspectProjectScopesPost403Response struct { +} + +func (response InternalInspectProjectScopesPost403Response) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalInspectProjectScopesPost500Response struct { +} + +func (response InternalInspectProjectScopesPost500Response) VisitInternalInspectProjectScopesPostResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + type InternalMultipartCompleteRequestObject struct { Body *InternalMultipartCompleteJSONRequestBody } @@ -1706,6 +2020,63 @@ func (response InternalBulkHashes500Response) VisitInternalBulkHashesResponse(ct return nil } +type InternalBulkMissingSHA256RequestObject struct { + Body *InternalBulkMissingSHA256JSONRequestBody +} + +type InternalBulkMissingSHA256ResponseObject interface { + VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error +} + +type InternalBulkMissingSHA256200JSONResponse BulkMissingSHA256Response + +func (response InternalBulkMissingSHA256200JSONResponse) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type InternalBulkMissingSHA256400Response struct { +} + +func (response InternalBulkMissingSHA256400Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type InternalBulkMissingSHA256401Response struct { +} + +func (response InternalBulkMissingSHA256401Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type InternalBulkMissingSHA256403Response struct { +} + +func (response InternalBulkMissingSHA256403Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type InternalBulkMissingSHA256413Response struct { +} + +func (response InternalBulkMissingSHA256413Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(413) + return nil +} + +type InternalBulkMissingSHA256500Response struct { +} + +func (response InternalBulkMissingSHA256500Response) VisitInternalBulkMissingSHA256Response(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + type InternalBulkSHA256ValidityRequestObject struct { Body *InternalBulkSHA256ValidityJSONRequestBody } @@ -1905,6 +2276,15 @@ type StrictServerInterface interface { // (GET /data/download/{file_id}/part) InternalDownloadPart(ctx context.Context, request InternalDownloadPartRequestObject) (InternalDownloadPartResponseObject, error) + // (POST /data/inspect/project-bucket/inventory) + InternalInspectProjectBucketInventory(ctx context.Context, request InternalInspectProjectBucketInventoryRequestObject) (InternalInspectProjectBucketInventoryResponseObject, error) + + // (GET /data/inspect/project-scopes) + InternalInspectProjectScopes(ctx context.Context, request InternalInspectProjectScopesRequestObject) (InternalInspectProjectScopesResponseObject, error) + + // (POST /data/inspect/project-scopes) + InternalInspectProjectScopesPost(ctx context.Context, request InternalInspectProjectScopesPostRequestObject) (InternalInspectProjectScopesPostResponseObject, error) + // (POST /data/multipart/complete) InternalMultipartComplete(ctx context.Context, request InternalMultipartCompleteRequestObject) (InternalMultipartCompleteResponseObject, error) @@ -1944,6 +2324,9 @@ type StrictServerInterface interface { // (POST /index/bulk/hashes) InternalBulkHashes(ctx context.Context, request InternalBulkHashesRequestObject) (InternalBulkHashesResponseObject, error) + // (POST /index/bulk/sha256/missing) + InternalBulkMissingSHA256(ctx context.Context, request InternalBulkMissingSHA256RequestObject) (InternalBulkMissingSHA256ResponseObject, error) + // (POST /index/bulk/sha256/validity) InternalBulkSHA256Validity(ctx context.Context, request InternalBulkSHA256ValidityRequestObject) (InternalBulkSHA256ValidityResponseObject, error) @@ -2029,6 +2412,95 @@ func (sh *strictHandler) InternalDownloadPart(ctx fiber.Ctx, fileId string, para return nil } +// InternalInspectProjectBucketInventory operation middleware +func (sh *strictHandler) InternalInspectProjectBucketInventory(ctx fiber.Ctx) error { + var request InternalInspectProjectBucketInventoryRequestObject + + var body InternalInspectProjectBucketInventoryJSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalInspectProjectBucketInventory(ctx.Context(), request.(InternalInspectProjectBucketInventoryRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalInspectProjectBucketInventory") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalInspectProjectBucketInventoryResponseObject); ok { + if err := validResponse.VisitInternalInspectProjectBucketInventoryResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + +// InternalInspectProjectScopes operation middleware +func (sh *strictHandler) InternalInspectProjectScopes(ctx fiber.Ctx, params InternalInspectProjectScopesParams) error { + var request InternalInspectProjectScopesRequestObject + + request.Params = params + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalInspectProjectScopes(ctx.Context(), request.(InternalInspectProjectScopesRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalInspectProjectScopes") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalInspectProjectScopesResponseObject); ok { + if err := validResponse.VisitInternalInspectProjectScopesResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + +// InternalInspectProjectScopesPost operation middleware +func (sh *strictHandler) InternalInspectProjectScopesPost(ctx fiber.Ctx) error { + var request InternalInspectProjectScopesPostRequestObject + + var body InternalInspectProjectScopesPostJSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalInspectProjectScopesPost(ctx.Context(), request.(InternalInspectProjectScopesPostRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalInspectProjectScopesPost") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalInspectProjectScopesPostResponseObject); ok { + if err := validResponse.VisitInternalInspectProjectScopesPostResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + // InternalMultipartComplete operation middleware func (sh *strictHandler) InternalMultipartComplete(ctx fiber.Ctx) error { var request InternalMultipartCompleteRequestObject @@ -2424,6 +2896,37 @@ func (sh *strictHandler) InternalBulkHashes(ctx fiber.Ctx) error { return nil } +// InternalBulkMissingSHA256 operation middleware +func (sh *strictHandler) InternalBulkMissingSHA256(ctx fiber.Ctx) error { + var request InternalBulkMissingSHA256RequestObject + + var body InternalBulkMissingSHA256JSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.InternalBulkMissingSHA256(ctx.Context(), request.(InternalBulkMissingSHA256RequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "InternalBulkMissingSHA256") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(InternalBulkMissingSHA256ResponseObject); ok { + if err := validResponse.VisitInternalBulkMissingSHA256Response(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + // InternalBulkSHA256Validity operation middleware func (sh *strictHandler) InternalBulkSHA256Validity(ctx fiber.Ctx) error { var request InternalBulkSHA256ValidityRequestObject diff --git a/apigen/server/metricsapi/metrics.gen.go b/apigen/server/metricsapi/metrics.gen.go index ab20b076..f8db9536 100644 --- a/apigen/server/metricsapi/metrics.gen.go +++ b/apigen/server/metricsapi/metrics.gen.go @@ -54,12 +54,20 @@ type FileUsage struct { UploadCount *int64 `json:"upload_count,omitempty"` } +// FileUsageBulkRequest defines model for FileUsageBulkRequest. +type FileUsageBulkRequest struct { + InactiveDays *int `json:"inactive_days,omitempty"` + ObjectIds []string `json:"object_ids"` +} + // FileUsageSummary defines model for FileUsageSummary. type FileUsageSummary struct { - InactiveFileCount *int64 `json:"inactive_file_count,omitempty"` - TotalDownloads *int64 `json:"total_downloads,omitempty"` - TotalFiles *int64 `json:"total_files,omitempty"` - TotalUploads *int64 `json:"total_uploads,omitempty"` + InactiveFileCount *int64 `json:"inactive_file_count,omitempty"` + RecordCount *int64 `json:"record_count,omitempty"` + RecordLatestUpdatedTime *time.Time `json:"record_latest_updated_time"` + TotalDownloads *int64 `json:"total_downloads,omitempty"` + TotalFiles *int64 `json:"total_files,omitempty"` + TotalUploads *int64 `json:"total_uploads,omitempty"` } // MetricsListResponse defines model for MetricsListResponse. @@ -216,6 +224,15 @@ type ListMetricsFilesParams struct { Project *Project `form:"project,omitempty" json:"project,omitempty"` } +// BulkMetricsFilesParams defines parameters for BulkMetricsFiles. +type BulkMetricsFilesParams struct { + // Organization Organization/program scope filter. + Organization *Organization `form:"organization,omitempty" json:"organization,omitempty"` + + // Project Project scope filter. Requires organization when set. + Project *Project `form:"project,omitempty" json:"project,omitempty"` +} + // GetMetricsFileParams defines parameters for GetMetricsFile. type GetMetricsFileParams struct { // Organization Organization/program scope filter. @@ -293,6 +310,9 @@ type GetTransferSummaryParams struct { AllowStale *AllowStale `form:"allow_stale,omitempty" json:"allow_stale,omitempty"` } +// BulkMetricsFilesJSONRequestBody defines body for BulkMetricsFiles for application/json ContentType. +type BulkMetricsFilesJSONRequestBody = FileUsageBulkRequest + // RecordProviderTransferEventsJSONRequestBody defines body for RecordProviderTransferEvents for application/json ContentType. type RecordProviderTransferEventsJSONRequestBody = ProviderTransferEventsRequest @@ -302,6 +322,9 @@ type ServerInterface interface { // (GET /index/v1/metrics/files) ListMetricsFiles(c fiber.Ctx, params ListMetricsFilesParams) error + // (POST /index/v1/metrics/files/bulk) + BulkMetricsFiles(c fiber.Ctx, params BulkMetricsFilesParams) error + // (GET /index/v1/metrics/files/{object_id}) GetMetricsFile(c fiber.Ctx, objectId string, params GetMetricsFileParams) error // Record provider-observed transfer events @@ -389,6 +412,37 @@ func (siw *ServerInterfaceWrapper) ListMetricsFiles(c fiber.Ctx) error { return siw.Handler.ListMetricsFiles(c, params) } +// BulkMetricsFiles operation middleware +func (siw *ServerInterfaceWrapper) BulkMetricsFiles(c fiber.Ctx) error { + var err error + var params BulkMetricsFilesParams + + // ------------- Optional query parameter "organization" ------------- + if paramValue := c.Query("organization"); paramValue != "" { + + var value Organization + err = runtime.BindStyledParameterWithOptions("form", "organization", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter organization: %w", err).Error()) + } + params.Organization = &value + + } + // ------------- Optional query parameter "project" ------------- + if paramValue := c.Query("project"); paramValue != "" { + + var value Project + err = runtime.BindStyledParameterWithOptions("form", "project", paramValue, &value, runtime.BindStyledParameterOptions{Explode: true, Required: false}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter project: %w", err).Error()) + } + params.Project = &value + + } + + return siw.Handler.BulkMetricsFiles(c, params) +} + // GetMetricsFile operation middleware func (siw *ServerInterfaceWrapper) GetMetricsFile(c fiber.Ctx) error { var err error @@ -795,6 +849,8 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Get(options.BaseURL+"/index/v1/metrics/files", wrapper.ListMetricsFiles) + router.Post(options.BaseURL+"/index/v1/metrics/files/bulk", wrapper.BulkMetricsFiles) + router.Get(options.BaseURL+"/index/v1/metrics/files/:object_id", wrapper.GetMetricsFile) router.Post(options.BaseURL+"/index/v1/metrics/provider-transfer-events", wrapper.RecordProviderTransferEvents) @@ -856,6 +912,56 @@ func (response ListMetricsFiles500Response) VisitListMetricsFilesResponse(ctx fi return nil } +type BulkMetricsFilesRequestObject struct { + Params BulkMetricsFilesParams + Body *BulkMetricsFilesJSONRequestBody +} + +type BulkMetricsFilesResponseObject interface { + VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error +} + +type BulkMetricsFiles200JSONResponse MetricsListResponse + +func (response BulkMetricsFiles200JSONResponse) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Response().Header.Set("Content-Type", "application/json") + ctx.Status(200) + + return ctx.JSON(&response) +} + +type BulkMetricsFiles400Response struct { +} + +func (response BulkMetricsFiles400Response) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Status(400) + return nil +} + +type BulkMetricsFiles401Response struct { +} + +func (response BulkMetricsFiles401Response) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Status(401) + return nil +} + +type BulkMetricsFiles403Response struct { +} + +func (response BulkMetricsFiles403Response) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Status(403) + return nil +} + +type BulkMetricsFiles500Response struct { +} + +func (response BulkMetricsFiles500Response) VisitBulkMetricsFilesResponse(ctx fiber.Ctx) error { + ctx.Status(500) + return nil +} + type GetMetricsFileRequestObject struct { ObjectId string `json:"object_id"` Params GetMetricsFileParams @@ -1117,6 +1223,9 @@ type StrictServerInterface interface { // (GET /index/v1/metrics/files) ListMetricsFiles(ctx context.Context, request ListMetricsFilesRequestObject) (ListMetricsFilesResponseObject, error) + // (POST /index/v1/metrics/files/bulk) + BulkMetricsFiles(ctx context.Context, request BulkMetricsFilesRequestObject) (BulkMetricsFilesResponseObject, error) + // (GET /index/v1/metrics/files/{object_id}) GetMetricsFile(ctx context.Context, request GetMetricsFileRequestObject) (GetMetricsFileResponseObject, error) // Record provider-observed transfer events @@ -1173,6 +1282,39 @@ func (sh *strictHandler) ListMetricsFiles(ctx fiber.Ctx, params ListMetricsFiles return nil } +// BulkMetricsFiles operation middleware +func (sh *strictHandler) BulkMetricsFiles(ctx fiber.Ctx, params BulkMetricsFilesParams) error { + var request BulkMetricsFilesRequestObject + + request.Params = params + + var body BulkMetricsFilesJSONRequestBody + if err := ctx.Bind().Body(&body); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + request.Body = &body + + handler := func(ctx fiber.Ctx, request interface{}) (interface{}, error) { + return sh.ssi.BulkMetricsFiles(ctx.Context(), request.(BulkMetricsFilesRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "BulkMetricsFiles") + } + + response, err := handler(ctx, request) + + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else if validResponse, ok := response.(BulkMetricsFilesResponseObject); ok { + if err := validResponse.VisitBulkMetricsFilesResponse(ctx); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} + // GetMetricsFile operation middleware func (sh *strictHandler) GetMetricsFile(ctx fiber.Ctx, objectId string, params GetMetricsFileParams) error { var request GetMetricsFileRequestObject diff --git a/client/bucket/location.go b/client/bucket/location.go deleted file mode 100644 index dfda1d95..00000000 --- a/client/bucket/location.go +++ /dev/null @@ -1,24 +0,0 @@ -package bucket - -import ( - "fmt" - "net/url" - "strings" -) - -// ParseStorageLocation extracts the bucket and key from a raw storage URL (e.g., s3://bucket/key). -func ParseStorageLocation(rawURL string) (bucketName, key string, err error) { - if rawURL == "" { - return "", "", fmt.Errorf("empty storage URL") - } - parsed, err := url.Parse(rawURL) - if err != nil { - return "", "", fmt.Errorf("failed to parse storage URL %q: %w", rawURL, err) - } - bucketName = strings.TrimSpace(parsed.Host) - key = strings.Trim(strings.TrimSpace(parsed.Path), "/") - if bucketName == "" || key == "" { - return "", "", fmt.Errorf("invalid storage location %q: missing bucket or key", rawURL) - } - return bucketName, key, nil -} diff --git a/client/bucket/location_test.go b/client/bucket/location_test.go deleted file mode 100644 index 9182fbb6..00000000 --- a/client/bucket/location_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package bucket - -import ( - "context" - "testing" - - bucketapi "github.com/calypr/syfon/apigen/client/bucketapi" -) - -func TestParseStorageLocation(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - rawURL string - wantBucket string - wantKey string - wantErr bool - }{ - {name: "valid s3 url", rawURL: "s3://bucket/path/to/file.txt", wantBucket: "bucket", wantKey: "path/to/file.txt"}, - {name: "trim key slashes", rawURL: "s3://bucket//nested/file.txt/", wantBucket: "bucket", wantKey: "nested/file.txt"}, - {name: "empty", rawURL: "", wantErr: true}, - {name: "parse error", rawURL: "://bad", wantErr: true}, - {name: "missing bucket", rawURL: "s3:///path/to/file.txt", wantErr: true}, - {name: "missing key", rawURL: "s3://bucket/", wantErr: true}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - bucketName, key, err := ParseStorageLocation(tc.rawURL) - if tc.wantErr { - if err == nil { - t.Fatalf("expected error for %q", tc.rawURL) - } - return - } - if err != nil { - t.Fatalf("ParseStorageLocation returned error: %v", err) - } - if bucketName != tc.wantBucket || key != tc.wantKey { - t.Fatalf("unexpected parse result: bucket=%q key=%q want bucket=%q key=%q", bucketName, key, tc.wantBucket, tc.wantKey) - } - }) - } -} - -func TestValidateBucketNonS3Providers(t *testing.T) { - t.Parallel() - - ctx := context.Background() - for _, provider := range []string{"gcs", "gs", "azure", "custom"} { - provider := provider - t.Run(provider, func(t *testing.T) { - req := bucketapi.PutBucketRequest{ - Bucket: "bucket-1", - Organization: "org", - ProjectId: "proj", - Provider: &provider, - } - if err := ValidateBucket(ctx, req); err != nil { - t.Fatalf("ValidateBucket returned error for provider %q: %v", provider, err) - } - }) - } -} diff --git a/client/client.go b/client/client.go index 4b9a6516..0bc6c689 100644 --- a/client/client.go +++ b/client/client.go @@ -59,21 +59,7 @@ type Client struct { type Option func(*Config) -func WithHTTPClient(h *http.Client) Option { - return func(c *Config) { - if h != nil { - c.HTTPClient = h - } - } -} -func WithUserAgent(v string) Option { - return func(c *Config) { - if strings.TrimSpace(v) != "" { - c.UserAgent = strings.TrimSpace(v) - } - } -} func WithBasicAuth(user, pass string) Option { return func(c *Config) { diff --git a/client/client_test.go b/client/client_test.go index 814ba9b6..9966dbd1 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -21,11 +21,11 @@ func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { func newTestClient(t *testing.T, fn roundTripFunc) *Client { t.Helper() httpClient := &http.Client{Transport: fn} - c, err := New("http://example.test", WithHTTPClient(httpClient)) + c, err := NewClient(&Config{Address: "http://example.test", HTTPClient: httpClient}) if err != nil { t.Fatalf("failed to create client: %v", err) } - return c.(*Client) + return c } func TestClientBasicAuthAndUserAgent(t *testing.T) { @@ -48,10 +48,12 @@ func TestClientBasicAuthAndUserAgent(t *testing.T) { }, nil })} - c, err := New("http://example.test", - WithBasicAuth("u", "p"), - WithUserAgent("syfon-test-client"), - WithHTTPClient(httpClient)) + c, err := NewClient(&Config{ + Address: "http://example.test", + BasicAuth: &BasicAuth{Username: "u", Password: "p"}, + UserAgent: "syfon-test-client", + HTTPClient: httpClient, + }) if err != nil { t.Fatalf("failed to create client: %v", err) } @@ -81,13 +83,14 @@ func TestGeneratedClientUsesBasicAuthTransport(t *testing.T) { }, nil })} - raw, err := New("http://example.test", - WithBasicAuth("u", "p"), - WithHTTPClient(httpClient)) + c, err := NewClient(&Config{ + Address: "http://example.test", + BasicAuth: &BasicAuth{Username: "u", Password: "p"}, + HTTPClient: httpClient, + }) if err != nil { t.Fatalf("failed to create client: %v", err) } - c := raw.(*Client) if _, err := c.InternalAPI().InternalListWithResponse(context.Background(), &internalapi.InternalListParams{}); err != nil { t.Fatalf("generated client list failed: %v", err) } @@ -233,12 +236,17 @@ func TestNewClientDefaultWiring(t *testing.T) { }, nil })} - raw, err := New("example.test:8080/", WithHTTPClient(httpClient), WithUserAgent(" "), WithBasicAuth(" ", "ignored")) + raw, err := NewClient(&Config{ + Address: "example.test:8080/", + HTTPClient: httpClient, + UserAgent: " ", + BasicAuth: &BasicAuth{Username: " ", Password: "ignored"}, + }) if err != nil { - t.Fatalf("New returned error: %v", err) + t.Fatalf("NewClient returned error: %v", err) } - c := raw.(*Client) + c := raw if c.Address() != "http://example.test:8080" { t.Fatalf("unexpected address: %q", c.Address()) } diff --git a/client/common/common.go b/client/common/common.go index 091d686d..4402ba2c 100644 --- a/client/common/common.go +++ b/client/common/common.go @@ -3,11 +3,8 @@ package common import ( "bytes" "encoding/json" - "errors" "fmt" "io" - "log" - "net/http" "os" "path/filepath" "strings" @@ -40,100 +37,6 @@ func GetAbsolutePath(filePath string) (string, error) { return filepath.Abs(fullFilePath) } -func ParseFilePaths(filePath string, metadataEnabled bool) ([]string, error) { - fullFilePath, err := GetAbsolutePath(filePath) - if err != nil { - return []string{}, err - } - initialPaths, err := filepath.Glob(fullFilePath) - if err != nil { - return []string{}, err - } - - var errs []error - var finalFilePaths []string - for _, p := range cleanupHiddenFiles(initialPaths) { - file, err := os.Open(p) - if err != nil { - errs = append(errs, fmt.Errorf("file open error for %s: %w", p, err)) - continue - } - - func(filePath string, file *os.File) { - defer file.Close() - - fi, statErr := file.Stat() - if statErr != nil { - errs = append(errs, fmt.Errorf("file stat error for %s: %w", filePath, statErr)) - return - } - if fi.IsDir() { - err = filepath.Walk(filePath, func(path string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - isHidden, err := IsHidden(path) - if err != nil { - return err - } - isMetadata := false - if metadataEnabled { - isMetadata = strings.HasSuffix(path, "_metadata.json") - } - if !fileInfo.IsDir() && !isHidden && !isMetadata { - finalFilePaths = append(finalFilePaths, path) - } - return nil - }) - if err != nil { - errs = append(errs, fmt.Errorf("directory walk error for %s: %w", filePath, err)) - } - } else { - finalFilePaths = append(finalFilePaths, filePath) - } - }(p, file) - } - - return finalFilePaths, errors.Join(errs...) -} - -func cleanupHiddenFiles(filePaths []string) []string { - i := 0 - for _, filePath := range filePaths { - isHidden, err := IsHidden(filePath) - if err != nil { - log.Println("Error occurred when checking hidden files: " + err.Error()) - continue - } - - if isHidden { - log.Printf("File %s is a hidden file and will be skipped\n", filePath) - continue - } - filePaths[i] = filePath - i++ - } - return filePaths[:i] -} - -func CanDownloadFile(signedURL string) error { - req, err := http.NewRequest("GET", signedURL, nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Range", "bytes=0-0") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("error while sending the request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusPartialContent || resp.StatusCode == http.StatusOK { - return nil - } - return ResponseBodyError(resp, "failed to access file") -} - func IsCloudPresignedURL(raw string) bool { return strings.Contains(raw, "X-Amz-Signature") || strings.Contains(raw, "X-Goog-Signature") || @@ -141,15 +44,3 @@ func IsCloudPresignedURL(raw string) bool { strings.Contains(raw, "AWSAccessKeyId=") || strings.Contains(raw, "Expires=") } - -func LoadFailedLog(path string) (map[string]RetryObject, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var m map[string]RetryObject - if err := json.Unmarshal(data, &m); err != nil { - return nil, err - } - return m, nil -} diff --git a/client/common/common_test.go b/client/common/common_test.go index 0ee8a528..1be1d14b 100644 --- a/client/common/common_test.go +++ b/client/common/common_test.go @@ -2,11 +2,8 @@ package common import ( "context" - "encoding/json" - "errors" "io" "net/http" - "net/http/httptest" "os" "path/filepath" "strings" @@ -54,55 +51,6 @@ func TestParseRootPathAndGetAbsolutePath(t *testing.T) { } } -func TestParseFilePaths(t *testing.T) { - t.Parallel() - - tempDir := t.TempDir() - visible := filepath.Join(tempDir, "visible.txt") - metadata := filepath.Join(tempDir, "visible_metadata.json") - hidden := filepath.Join(tempDir, ".hidden.txt") - nestedDir := filepath.Join(tempDir, "nested") - nestedVisible := filepath.Join(nestedDir, "nested.txt") - nestedMetadata := filepath.Join(nestedDir, "nested_metadata.json") - nestedHidden := filepath.Join(nestedDir, "~skip.txt") - - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatalf("MkdirAll returned error: %v", err) - } - for _, path := range []string{visible, metadata, hidden, nestedVisible, nestedMetadata, nestedHidden} { - if err := os.WriteFile(path, []byte("ok"), 0o644); err != nil { - t.Fatalf("WriteFile(%s) returned error: %v", path, err) - } - } - - withMetadata, err := ParseFilePaths(filepath.Join(tempDir, "*"), false) - if err != nil { - t.Fatalf("ParseFilePaths without metadata filtering returned error: %v", err) - } - gotWithout := toPathSet(withMetadata) - if !gotWithout[visible] || !gotWithout[metadata] || !gotWithout[nestedVisible] || !gotWithout[nestedMetadata] { - t.Fatalf("expected visible, metadata, and nested files; got %+v", withMetadata) - } - if gotWithout[hidden] || gotWithout[nestedHidden] { - t.Fatalf("did not expect hidden files in result: %+v", withMetadata) - } - - withoutMetadata, err := ParseFilePaths(filepath.Join(tempDir, "*"), true) - if err != nil { - t.Fatalf("ParseFilePaths with metadata filtering returned error: %v", err) - } - gotWith := toPathSet(withoutMetadata) - if !gotWith[metadata] { - t.Fatalf("expected root metadata file to remain in result: %+v", withoutMetadata) - } - if gotWith[nestedMetadata] { - t.Fatalf("did not expect nested metadata file in result: %+v", withoutMetadata) - } - if !gotWith[visible] || !gotWith[nestedVisible] { - t.Fatalf("expected visible and nested files after filtering; got %+v", withoutMetadata) - } -} - func TestResponseBodyError(t *testing.T) { t.Parallel() @@ -127,32 +75,6 @@ func TestResponseBodyError(t *testing.T) { } } -func TestCanDownloadFile(t *testing.T) { - t.Parallel() - - okServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("Range"); got != "bytes=0-0" { - t.Fatalf("unexpected range header: %q", got) - } - w.WriteHeader(http.StatusPartialContent) - })) - defer okServer.Close() - - if err := CanDownloadFile(okServer.URL); err != nil { - t.Fatalf("CanDownloadFile returned error for 206 response: %v", err) - } - - forbiddenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "access denied", http.StatusForbidden) - })) - defer forbiddenServer.Close() - - err := CanDownloadFile(forbiddenServer.URL) - if err == nil || !strings.Contains(err.Error(), "failed to access file: status 403 body=access denied") { - t.Fatalf("unexpected download error: %v", err) - } -} - func TestIsCloudPresignedURL(t *testing.T) { t.Parallel() @@ -176,44 +98,6 @@ func TestIsCloudPresignedURL(t *testing.T) { } } -func TestLoadFailedLog(t *testing.T) { - t.Parallel() - - tempDir := t.TempDir() - path := filepath.Join(tempDir, "failed.json") - want := map[string]RetryObject{ - "one": { - SourcePath: "/tmp/file1", - GUID: "guid-1", - RetryCount: 2, - Bucket: "bucket-1", - FileMetadata: FileMetadata{ - Authorizations: map[string][]string{"test": {}}, - Aliases: []string{"alias-1"}, - Metadata: map[string]any{"k": "v"}, - }, - }, - } - data, err := json.Marshal(want) - if err != nil { - t.Fatalf("Marshal returned error: %v", err) - } - if err := os.WriteFile(path, data, 0o644); err != nil { - t.Fatalf("WriteFile returned error: %v", err) - } - - got, err := LoadFailedLog(path) - if err != nil { - t.Fatalf("LoadFailedLog returned error: %v", err) - } - if got["one"].SourcePath != want["one"].SourcePath || got["one"].GUID != want["one"].GUID || got["one"].Bucket != want["one"].Bucket { - t.Fatalf("unexpected retry object: %+v", got["one"]) - } - if got["one"].FileMetadata.Metadata["k"] != "v" { - t.Fatalf("unexpected metadata: %+v", got["one"].FileMetadata.Metadata) - } -} - func TestProgressContextHelpers(t *testing.T) { t.Parallel() @@ -247,31 +131,3 @@ func TestProgressContextHelpers(t *testing.T) { t.Fatalf("unexpected oid: %q", got) } } - -func TestCleanupHiddenFiles(t *testing.T) { - t.Parallel() - - visible := filepath.Join(t.TempDir(), "visible.txt") - hidden := filepath.Join(filepath.Dir(visible), ".hidden.txt") - paths := cleanupHiddenFiles([]string{visible, hidden}) - if len(paths) != 1 || paths[0] != visible { - t.Fatalf("unexpected filtered paths: %+v", paths) - } -} - -func TestLoadFailedLogMissingFile(t *testing.T) { - t.Parallel() - - _, err := LoadFailedLog(filepath.Join(t.TempDir(), "missing.json")) - if err == nil || !errors.Is(err, os.ErrNotExist) { - t.Fatalf("expected not-exist error, got %v", err) - } -} - -func toPathSet(paths []string) map[string]bool { - result := make(map[string]bool, len(paths)) - for _, path := range paths { - result[path] = true - } - return result -} diff --git a/client/common/constants.go b/client/common/constants.go index ff69102d..6cc3f897 100644 --- a/client/common/constants.go +++ b/client/common/constants.go @@ -110,7 +110,7 @@ const ( QueryParamURL = "url" QueryParamHashType = "hash_type" QueryParamBucket = "bucket" - QueryParamFileName = "file_name" + QueryParamFileName = "filename" QueryParamExpiresIn = "expires_in" QueryParamRedirect = "redirect" QueryParamInactiveDays = "inactive_days" diff --git a/client/common/isHidden_notwindows.go b/client/common/isHidden_notwindows.go deleted file mode 100644 index fc10bbe2..00000000 --- a/client/common/isHidden_notwindows.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build !windows -// +build !windows - -package common - -import "path/filepath" - -func IsHidden(filePath string) (bool, error) { - filename := filepath.Base(filePath) - if filename[0:1] == "." || filename[0:1] == "~" { - return true, nil - } - return false, nil -} diff --git a/client/common/isHidden_windows.go b/client/common/isHidden_windows.go deleted file mode 100644 index c68682c4..00000000 --- a/client/common/isHidden_windows.go +++ /dev/null @@ -1,30 +0,0 @@ -//go:build windows -// +build windows - -package common - -import ( - "errors" - "path/filepath" - "runtime" - "syscall" -) - -func IsHidden(filePath string) (bool, error) { - filename := filepath.Base(filePath) - if runtime.GOOS == "windows" { - if filename[0:1] == "." || filename[0:1] == "~" { - return true, nil - } - pointer, err := syscall.UTF16PtrFromString(filePath) - if err != nil { - return false, err - } - attributes, err := syscall.GetFileAttributes(pointer) - if err != nil { - return false, err - } - return attributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil - } - return false, errors.New("unable to check if file is hidden under non-Windows OS") -} diff --git a/client/common/progress.go b/client/common/progress.go index 6152db21..28fb6270 100644 --- a/client/common/progress.go +++ b/client/common/progress.go @@ -14,24 +14,11 @@ type ProgressEvent struct { type ProgressCallback func(ProgressEvent) error -type TransferCompletionEvent struct { - Direction string - GUID string - RangeStart int64 - RangeEnd int64 - Bytes int64 - PartNumber int - Strategy string -} - -type TransferCompletionCallback func(TransferCompletionEvent) error - type contextKey string const ( progressKey contextKey = "progressCallback" oidKey contextKey = "activeOid" - transferCompletionKey contextKey = "transferCompletionCallback" ) func WithProgress(ctx context.Context, cb ProgressCallback) context.Context { @@ -55,14 +42,3 @@ func GetOid(ctx context.Context) string { } return "" } - -func WithTransferCompletion(ctx context.Context, cb TransferCompletionCallback) context.Context { - return context.WithValue(ctx, transferCompletionKey, cb) -} - -func GetTransferCompletion(ctx context.Context) TransferCompletionCallback { - if cb, ok := ctx.Value(transferCompletionKey).(TransferCompletionCallback); ok { - return cb - } - return nil -} diff --git a/client/config/config_test.go b/client/config/config_test.go index b1fd8dc0..203eadca 100644 --- a/client/config/config_test.go +++ b/client/config/config_test.go @@ -243,19 +243,7 @@ func TestManagerImport(t *testing.T) { }) } -func TestValidateURL(t *testing.T) { - t.Parallel() - if _, err := ValidateUrl("https://example.org"); err != nil { - t.Fatalf("ValidateUrl returned error for valid URL: %v", err) - } - if _, err := ValidateUrl("://bad"); err == nil { - t.Fatal("expected parse error for invalid URL") - } - if _, err := ValidateUrl("not-a-url"); err == nil { - t.Fatal("expected host validation error") - } -} func TestManagerTokenAndCredentialValidation(t *testing.T) { diff --git a/client/config/validate.go b/client/config/validate.go index f2ee8f8e..b6453abd 100644 --- a/client/config/validate.go +++ b/client/config/validate.go @@ -1,25 +1,12 @@ package config import ( - "errors" "fmt" - "net/url" "time" "github.com/golang-jwt/jwt/v5" ) -func ValidateUrl(apiEndpoint string) (*url.URL, error) { - parsedURL, err := url.Parse(apiEndpoint) - if err != nil { - return parsedURL, errors.New("Error occurred when parsing apiendpoint URL: " + err.Error()) - } - if parsedURL.Host == "" { - return parsedURL, errors.New("Invalid endpoint. A valid endpoint looks like: https://www.tests.com") - } - return parsedURL, nil -} - func (man *Manager) IsTokenValid(tokenStr string) (bool, error) { if tokenStr == "" { return false, fmt.Errorf("token is empty") diff --git a/client/hash/hash.go b/client/hash/hash.go index ad70a7c2..efed7d41 100644 --- a/client/hash/hash.go +++ b/client/hash/hash.go @@ -1,10 +1,8 @@ package hash import ( - "encoding/hex" "encoding/json" "fmt" - "strings" drsapi "github.com/calypr/syfon/apigen/client/drs" ) @@ -23,21 +21,10 @@ const ( ChecksumTypeTrunc512 ChecksumType = "trunc512" ) -// IsValid checks if the checksum type is a known/recommended value -func (ct ChecksumType) IsValid() bool { - switch ct { - case ChecksumTypeSHA256, ChecksumTypeSHA512, ChecksumTypeSHA1, ChecksumTypeMD5, - ChecksumTypeETag, ChecksumTypeCRC32C, ChecksumTypeTrunc512: - return true - default: - return false - } -} + // String returns the string representation of the checksum type -func (ct ChecksumType) String() string { - return string(ct) -} + var SupportedChecksums = map[string]bool{ string(ChecksumTypeSHA1): true, @@ -108,28 +95,7 @@ func ConvertStringMapToHashInfo(inputHashes map[string]string) HashInfo { return hashInfo } -func ConvertHashInfoToMap(hashes HashInfo) map[string]string { - result := make(map[string]string) - if hashes.MD5 != "" { - result["md5"] = hashes.MD5 - } - if hashes.SHA != "" { - result["sha"] = hashes.SHA - } - if hashes.SHA256 != "" { - result["sha256"] = hashes.SHA256 - } - if hashes.SHA512 != "" { - result["sha512"] = hashes.SHA512 - } - if hashes.CRC != "" { - result["crc"] = hashes.CRC - } - if hashes.ETag != "" { - result["etag"] = hashes.ETag - } - return result -} + func ConvertChecksumsToMap(checksums []Checksum) map[string]string { result := make(map[string]string, len(checksums)) @@ -144,88 +110,8 @@ func ConvertChecksumsToHashInfo(checksums []Checksum) HashInfo { return ConvertStringMapToHashInfo(checksumMap) } -func ConvertDrsChecksumsToMap(checksums []drsapi.Checksum) map[string]string { - result := make(map[string]string, len(checksums)) - for _, c := range checksums { - result[c.Type] = c.Checksum - } - return result -} -func ConvertDrsChecksumsToHashInfo(checksums []drsapi.Checksum) HashInfo { - checksumMap := ConvertDrsChecksumsToMap(checksums) - return ConvertStringMapToHashInfo(checksumMap) -} -func ConvertMapToDrsChecksums(hashes map[string]string) []drsapi.Checksum { - result := make([]drsapi.Checksum, 0, len(hashes)) - for t, c := range hashes { - result = append(result, drsapi.Checksum{ - Type: t, - Checksum: c, - }) - } - return result -} -func NormalizeChecksumType(raw string) ChecksumType { - switch strings.ToLower(strings.TrimSpace(raw)) { - case "sha256": - return ChecksumTypeSHA256 - case "sha512": - return ChecksumTypeSHA512 - case "sha1", "sha": - return ChecksumTypeSHA1 - case "md5": - return ChecksumTypeMD5 - case "etag": - return ChecksumTypeETag - case "crc32c": - return ChecksumTypeCRC32C - case "trunc512": - return ChecksumTypeTrunc512 - default: - return ChecksumType(raw) - } -} -func ValidateChecksum(c Checksum) error { - checksum := strings.TrimSpace(c.Checksum) - if checksum == "" { - return fmt.Errorf("checksum value is required") - } - ct := NormalizeChecksumType(c.Type) - if !ct.IsValid() { - return fmt.Errorf("unsupported checksum type %q", c.Type) - } - switch ct { - case ChecksumTypeSHA256: - if !isHexDigest(checksum, 64) { - return fmt.Errorf("invalid sha256 checksum") - } - case ChecksumTypeSHA512: - if !isHexDigest(checksum, 128) { - return fmt.Errorf("invalid sha512 checksum") - } - case ChecksumTypeSHA1: - if !isHexDigest(checksum, 40) { - return fmt.Errorf("invalid sha1 checksum") - } - case ChecksumTypeMD5: - if !isHexDigest(checksum, 32) { - return fmt.Errorf("invalid md5 checksum") - } - case ChecksumTypeCRC32C, ChecksumTypeETag, ChecksumTypeTrunc512: - // Provider implementations vary representation (hex/base64/quoted ETag). - // Non-empty check above is the strict lower bound. - } - return nil -} -func isHexDigest(v string, wantLen int) bool { - if len(v) != wantLen { - return false - } - _, err := hex.DecodeString(v) - return err == nil -} diff --git a/client/hash/hash_test.go b/client/hash/hash_test.go index 28470d76..b254e713 100644 --- a/client/hash/hash_test.go +++ b/client/hash/hash_test.go @@ -2,25 +2,10 @@ package hash import ( "encoding/json" - "strings" "testing" - - drsapi "github.com/calypr/syfon/apigen/client/drs" ) -func TestChecksumTypeHelpers(t *testing.T) { - t.Parallel() - if !ChecksumTypeSHA256.IsValid() { - t.Fatal("expected sha256 to be valid") - } - if ChecksumType("bogus").IsValid() { - t.Fatal("expected bogus checksum type to be invalid") - } - if got := ChecksumTypeSHA256.String(); got != "sha256" { - t.Fatalf("unexpected string form: %q", got) - } -} func TestHashInfoUnmarshalJSON(t *testing.T) { t.Parallel() @@ -62,8 +47,8 @@ func TestHashInfoUnmarshalJSON(t *testing.T) { t.Run("unsupported payload", func(t *testing.T) { var got HashInfo err := json.Unmarshal([]byte(`123`), &got) - if err == nil || !strings.Contains(err.Error(), "unsupported HashInfo payload") { - t.Fatalf("expected unsupported payload error, got %v", err) + if err == nil { + t.Fatal("expected error") } }) } @@ -71,33 +56,6 @@ func TestHashInfoUnmarshalJSON(t *testing.T) { func TestHashConversions(t *testing.T) { t.Parallel() - h := HashInfo{ - MD5: "md5-value", - SHA: "sha1-value", - SHA256: "sha256-value", - SHA512: "sha512-value", - CRC: "crc-value", - ETag: "etag-value", - } - - gotMap := ConvertHashInfoToMap(h) - wantMap := map[string]string{ - "md5": "md5-value", - "sha": "sha1-value", - "sha256": "sha256-value", - "sha512": "sha512-value", - "crc": "crc-value", - "etag": "etag-value", - } - if len(gotMap) != len(wantMap) { - t.Fatalf("unexpected map size: got %d want %d", len(gotMap), len(wantMap)) - } - for k, want := range wantMap { - if got := gotMap[k]; got != want { - t.Fatalf("unexpected map value for %s: got %q want %q", k, got, want) - } - } - checksums := []Checksum{{Type: "sha256", Checksum: "abc"}, {Type: "md5", Checksum: "def"}} if got := ConvertChecksumsToMap(checksums); got["sha256"] != "abc" || got["md5"] != "def" { t.Fatalf("unexpected checksum map: %+v", got) @@ -105,99 +63,6 @@ func TestHashConversions(t *testing.T) { if got := ConvertChecksumsToHashInfo(checksums); got != (HashInfo{MD5: "def", SHA256: "abc"}) { t.Fatalf("unexpected checksum hash info: %+v", got) } - - drsChecksums := []drsapi.Checksum{{Type: "sha1", Checksum: "aaa"}, {Type: "etag", Checksum: "bbb"}} - if got := ConvertDrsChecksumsToMap(drsChecksums); got["sha1"] != "aaa" || got["etag"] != "bbb" { - t.Fatalf("unexpected drs checksum map: %+v", got) - } - if got := ConvertDrsChecksumsToHashInfo(drsChecksums); got != (HashInfo{SHA: "aaa", ETag: "bbb"}) { - t.Fatalf("unexpected drs hash info: %+v", got) - } - - roundTrip := ConvertMapToDrsChecksums(wantMap) - if got := ConvertDrsChecksumsToMap(roundTrip); len(got) != len(wantMap) { - t.Fatalf("unexpected round-trip map size: got %d want %d", len(got), len(wantMap)) - } else { - for k, want := range wantMap { - if got[k] != want { - t.Fatalf("unexpected round-trip value for %s: got %q want %q", k, got[k], want) - } - } - } -} - -func TestNormalizeChecksumType(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - raw string - want ChecksumType - }{ - {name: "sha256", raw: " sha256 ", want: ChecksumTypeSHA256}, - {name: "sha alias", raw: "SHA", want: ChecksumTypeSHA1}, - {name: "crc32c", raw: "CRC32C", want: ChecksumTypeCRC32C}, - {name: "unknown preserved", raw: " custom ", want: ChecksumType(" custom ")}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := NormalizeChecksumType(tc.raw); got != tc.want { - t.Fatalf("NormalizeChecksumType(%q) = %q, want %q", tc.raw, got, tc.want) - } - }) - } -} - -func TestValidateChecksum(t *testing.T) { - t.Parallel() - - valid := []Checksum{ - {Type: "sha256", Checksum: strings.Repeat("a", 64)}, - {Type: "sha512", Checksum: strings.Repeat("b", 128)}, - {Type: "sha", Checksum: strings.Repeat("c", 40)}, - {Type: "md5", Checksum: strings.Repeat("d", 32)}, - {Type: "etag", Checksum: "quoted-etag"}, - } - for _, checksum := range valid { - checksum := checksum - t.Run("valid_"+checksum.Type, func(t *testing.T) { - if err := ValidateChecksum(checksum); err != nil { - t.Fatalf("expected valid checksum, got %v", err) - } - }) - } - - invalid := []struct { - name string - c Checksum - msg string - }{ - {name: "empty", c: Checksum{Type: "sha256", Checksum: " "}, msg: "checksum value is required"}, - {name: "unsupported type", c: Checksum{Type: "nope", Checksum: "abc"}, msg: "unsupported checksum type"}, - {name: "bad sha256", c: Checksum{Type: "sha256", Checksum: strings.Repeat("g", 64)}, msg: "invalid sha256 checksum"}, - {name: "short md5", c: Checksum{Type: "md5", Checksum: "abc"}, msg: "invalid md5 checksum"}, - } - for _, tc := range invalid { - t.Run(tc.name, func(t *testing.T) { - err := ValidateChecksum(tc.c) - if err == nil || !strings.Contains(err.Error(), tc.msg) { - t.Fatalf("expected error containing %q, got %v", tc.msg, err) - } - }) - } } -func TestIsHexDigest(t *testing.T) { - t.Parallel() - if !isHexDigest(strings.Repeat("a", 8), 8) { - t.Fatal("expected valid hex digest") - } - if isHexDigest("zzzz", 4) { - t.Fatal("expected invalid hex digest for non-hex characters") - } - if isHexDigest("abcd", 8) { - t.Fatal("expected invalid hex digest for wrong length") - } -} diff --git a/client/logs/factory.go b/client/logs/factory.go deleted file mode 100644 index fe4b6448..00000000 --- a/client/logs/factory.go +++ /dev/null @@ -1,103 +0,0 @@ -package logs - -import ( - "fmt" - "log/slog" - "os" - "os/user" - "path/filepath" - "time" -) - -func New(profile string, opts ...Option) (*Gen3Logger, func()) { - cfg := defaults() - for _, o := range opts { - o(cfg) - } - - logDir := filepath.Join(userHomeDir(), ".gen3", "logs") - if err := os.MkdirAll(logDir, 0o755); err != nil { - fmt.Fprintf(os.Stderr, "create log directory %s: %v\n", logDir, err) - logDir = os.TempDir() - } - - var handlers []slog.Handler - var messageFile *os.File - - if cfg.baseLogger != nil { - handlers = append(handlers, cfg.baseLogger.Handler()) - } - - if cfg.console { - handlers = append(handlers, slog.NewTextHandler(os.Stderr, nil)) - } - - if cfg.messageFile { - filename := fmt.Sprintf("%s_message_%s_%d.log", - profile, - time.Now().Format("20060102150405MST"), - os.Getpid(), - ) - f, err := os.OpenFile(filepath.Join(logDir, filename), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err == nil { - messageFile = f - handlers = append(handlers, slog.NewTextHandler(f, nil)) - if _, err := fmt.Fprintf(f, "[%s] Message log started\n", time.Now().Format(time.RFC3339)); err != nil { - fmt.Fprintf(os.Stderr, "write message log start banner: %v\n", err) - } - } else { - fmt.Fprintf(os.Stderr, "open message log file: %v\n", err) - } - } - - var rootHandler slog.Handler - if len(handlers) == 0 { - rootHandler = slog.NewTextHandler(os.Stderr, nil) - } else if len(handlers) == 1 { - rootHandler = handlers[0] - } else { - rootHandler = NewTeeHandler(handlers...) - } - - sl := slog.New(NewProgressHandler(rootHandler)) - - t := NewGen3Logger(sl, logDir, profile) - - if cfg.enableScoreboard { - t.scoreboard = NewSB(5, t.Logger) - } - - if cfg.failedLog { - t.failedPath = filepath.Join(logDir, profile+"_failed.json") - loadJSON(t.failedPath, &t.FailedMap) - } - - if cfg.succeededLog { - t.succeededPath = filepath.Join(logDir, profile+"_succeeded.json") - loadJSON(t.succeededPath, &t.succeededMap) - } - - cleanup := func() { - if messageFile != nil { - if _, err := fmt.Fprintf(messageFile, "[%s] Message log stopped\n", time.Now().Format(time.RFC3339)); err != nil { - fmt.Fprintf(os.Stderr, "write message log stop banner: %v\n", err) - } - if err := messageFile.Close(); err != nil { - fmt.Fprintf(os.Stderr, "close message log file: %v\n", err) - } - } - } - - return t, cleanup -} - -func userHomeDir() string { - usr, err := user.Current() - if err == nil && usr.HomeDir != "" { - return usr.HomeDir - } - if home, err := os.UserHomeDir(); err == nil && home != "" { - return home - } - return os.TempDir() -} diff --git a/client/logs/handler.go b/client/logs/handler.go deleted file mode 100644 index 616d1073..00000000 --- a/client/logs/handler.go +++ /dev/null @@ -1,102 +0,0 @@ -package logs - -import ( - "context" - "log/slog" - - "github.com/calypr/syfon/client/common" -) - -// ProgressHandler is a slog.Handler that captures log messages and -// forwards them to a ProgressCallback if one is present in the context. -type ProgressHandler struct { - next slog.Handler -} - -func NewProgressHandler(next slog.Handler) *ProgressHandler { - if next == nil { - next = slog.Default().Handler() - } - return &ProgressHandler{next: next} -} - -func (h *ProgressHandler) Enabled(ctx context.Context, level slog.Level) bool { - return h.next.Enabled(ctx, level) -} - -func (h *ProgressHandler) Handle(ctx context.Context, r slog.Record) error { - // Call the next handler in the chain (original logging) - err := h.next.Handle(ctx, r) - - // In addition, try to bubble up to progress callback - cb := common.GetProgress(ctx) - if cb != nil { - oid := common.GetOid(ctx) - // We send an event of type "log" - attrs := make(map[string]any) - r.Attrs(func(a slog.Attr) bool { - attrs[a.Key] = a.Value.Any() - return true - }) - _ = cb(common.ProgressEvent{ - Event: "log", - Oid: oid, - Message: r.Message, - Level: r.Level.String(), - Attrs: attrs, - }) - } - - return err -} - -func (h *ProgressHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - return &ProgressHandler{next: h.next.WithAttrs(attrs)} -} - -func (h *ProgressHandler) WithGroup(name string) slog.Handler { - return &ProgressHandler{next: h.next.WithGroup(name)} -} - -// TeeHandler fans out log records to multiple handlers -type TeeHandler struct { - handlers []slog.Handler -} - -func NewTeeHandler(handlers ...slog.Handler) slog.Handler { - return &TeeHandler{handlers: handlers} -} - -func (h *TeeHandler) Enabled(ctx context.Context, level slog.Level) bool { - for _, hand := range h.handlers { - if hand.Enabled(ctx, level) { - return true - } - } - return false -} - -func (h *TeeHandler) Handle(ctx context.Context, r slog.Record) error { - for _, hand := range h.handlers { - if hand.Enabled(ctx, r.Level) { - _ = hand.Handle(ctx, r) - } - } - return nil -} - -func (h *TeeHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - newHandlers := make([]slog.Handler, len(h.handlers)) - for i, hand := range h.handlers { - newHandlers[i] = hand.WithAttrs(attrs) - } - return &TeeHandler{handlers: newHandlers} -} - -func (h *TeeHandler) WithGroup(name string) slog.Handler { - newHandlers := make([]slog.Handler, len(h.handlers)) - for i, hand := range h.handlers { - newHandlers[i] = hand.WithGroup(name) - } - return &TeeHandler{handlers: newHandlers} -} diff --git a/client/logs/logger.go b/client/logs/logger.go deleted file mode 100644 index f6a55f03..00000000 --- a/client/logs/logger.go +++ /dev/null @@ -1,35 +0,0 @@ -package logs - -import ( - "log/slog" -) - -type Option func(*config) - -type config struct { - console bool - messageFile bool - failedLog bool - succeededLog bool - enableScoreboard bool - baseLogger *slog.Logger -} - -func WithConsole() Option { return func(c *config) { c.console = true } } -func WithNoConsole() Option { return func(c *config) { c.console = false } } -func WithMessageFile() Option { return func(c *config) { c.messageFile = true } } -func WithNoMessageFile() Option { return func(c *config) { c.messageFile = false } } -func WithFailedLog() Option { return func(c *config) { c.failedLog = true } } -func WithSucceededLog() Option { return func(c *config) { c.succeededLog = true } } -func WithScoreboard() Option { return func(c *config) { c.enableScoreboard = true } } -func WithBaseLogger(base *slog.Logger) Option { return func(c *config) { c.baseLogger = base } } - -func defaults() *config { - return &config{ - console: true, - messageFile: true, - failedLog: true, - succeededLog: true, - baseLogger: nil, - } -} diff --git a/client/logs/logger_test.go b/client/logs/logger_test.go deleted file mode 100644 index 7e689f82..00000000 --- a/client/logs/logger_test.go +++ /dev/null @@ -1,210 +0,0 @@ -package logs - -import ( - "io" - "log/slog" - "os" - "testing" -) - -func TestNewSlogNoOpLogger(t *testing.T) { - logger := NewSlogNoOpLogger() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - // Verify it's a valid slog.Logger - logger.Info("test message") // Should not panic - logger.Error("test error") // Should not panic -} - -func TestNew_WithDefaults(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.Logger == nil { - t.Error("Expected non-nil embedded slog logger") - } -} - -func TestNew_WithConsoleOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithConsole()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - // Test that we can log without errors - logger.Info("test console message") -} - -func TestNew_WithMessageFileOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithMessageFile()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - // Test that we can log without errors - logger.Info("test file message") -} - -func TestNew_WithScoreboardOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithScoreboard()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.scoreboard == nil { - t.Error("Expected non-nil scoreboard when WithScoreboard option is used") - } -} - -func TestNew_WithFailedLogOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithFailedLog()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.failedPath == "" { - t.Error("Expected non-empty failed path when WithFailedLog option is used") - } -} - -func TestNew_WithSucceededLogOption(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile, WithSucceededLog()) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.succeededPath == "" { - t.Error("Expected non-empty succeeded path when WithSucceededLog option is used") - } -} - -func TestNew_WithBaseLogger(t *testing.T) { - profile := "test-profile" - baseLogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - - logger, cleanup := New(profile, WithBaseLogger(baseLogger)) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - // Test that we can log without errors - logger.Info("test with base logger") -} - -func TestNew_WithMultipleOptions(t *testing.T) { - profile := "test-profile" - baseLogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - - logger, cleanup := New(profile, - WithBaseLogger(baseLogger), - WithConsole(), - WithMessageFile(), - WithScoreboard(), - ) - defer cleanup() - - if logger == nil { - t.Fatal("Expected non-nil logger") - } - - if logger.Logger == nil { - t.Error("Expected non-nil embedded slog logger") - } - - if logger.scoreboard == nil { - t.Error("Expected non-nil scoreboard") - } - - // Test that we can log without errors - logger.Info("test with multiple options") -} - -func TestGen3Logger_Info(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Info("test info message") -} - -func TestGen3Logger_Error(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Error("test error message") -} - -func TestGen3Logger_Warn(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Warn("test warning message") -} - -func TestGen3Logger_Debug(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Debug("test debug message") -} - -func TestGen3Logger_Printf(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Printf("test printf message: %s", "value") -} - -func TestGen3Logger_Println(t *testing.T) { - profile := "test-profile" - logger, cleanup := New(profile) - defer cleanup() - - // Should not panic - logger.Println("test println message") -} - -// testLogger implements the Logger interface for testing -type testLogger struct { - writer io.Writer -} - -func (l *testLogger) Printf(format string, v ...any) {} -func (l *testLogger) Println(v ...any) {} -func (l *testLogger) Fatalf(format string, v ...any) {} -func (l *testLogger) Fatal(v ...any) {} -func (l *testLogger) Writer() io.Writer { return l.writer } diff --git a/client/logs/noop.go b/client/logs/noop.go deleted file mode 100644 index f705772d..00000000 --- a/client/logs/noop.go +++ /dev/null @@ -1,11 +0,0 @@ -package logs - -import ( - "io" - "log/slog" -) - -// NewSlogNoOpLogger creates a no-op slog logger for testing. -func NewSlogNoOpLogger() *slog.Logger { - return slog.New(slog.NewTextHandler(io.Discard, nil)) -} diff --git a/client/logs/scoreboard.go b/client/logs/scoreboard.go deleted file mode 100644 index 738a47a4..00000000 --- a/client/logs/scoreboard.go +++ /dev/null @@ -1,75 +0,0 @@ -package logs - -import ( - "fmt" - "io" - "log/slog" - "os" - "sync" - "text/tabwriter" -) - -// Scoreboard holds retry statistics -type Scoreboard struct { - mu sync.Mutex - Counts []int // index 0 = success on first try, 1 = after 1 retry, ..., last = failed - logger *slog.Logger - writer io.Writer -} - -// NewSB creates a new scoreboard -// maxRetryCount = how many retries you allow before giving up -func NewSB(maxRetryCount int, logger *slog.Logger) *Scoreboard { - return &Scoreboard{ - Counts: make([]int, maxRetryCount+2), // +2: one for success-on-first, one for final failure - logger: logger, - writer: os.Stderr, - } -} - -// IncrementSB records a result after `retryCount` attempts -// retryCount == 0 → succeeded on first try -// retryCount == max → final failure -func (s *Scoreboard) IncrementSB(retryCount int) { - s.mu.Lock() - defer s.mu.Unlock() - - if retryCount < 0 { - retryCount = 0 - } - if retryCount >= len(s.Counts)-1 { - retryCount = len(s.Counts) - 1 // final failure bucket - } - s.Counts[retryCount]++ -} - -// PrintSB prints the beautiful table at the end -func (s *Scoreboard) PrintSB() { - s.mu.Lock() - defer s.mu.Unlock() - - total := 0 - for _, c := range s.Counts { - total += c - } - if total == 0 { - return - } - - s.logger.Info("Submission Results") - w := tabwriter.NewWriter(s.writer, 0, 0, 2, ' ', 0) - - for i, count := range s.Counts { - if i == 0 { - fmt.Fprintf(w, "Success (no retry)\t%d\n", count) - } else if i == 1 { - fmt.Fprintf(w, "Success after %d retry\t%d\n", i, count) - } else if i < len(s.Counts)-1 { - fmt.Fprintf(w, "Success after %d retries\t%d\n", i, count) - } else { - fmt.Fprintf(w, "Failed (all retries exhausted)\t%d\n", count) - } - } - fmt.Fprintf(w, "TOTAL\t%d\n", total) - w.Flush() -} diff --git a/client/logs/tee_logger.go b/client/logs/tee_logger.go index a845fdb8..ebdf0ab3 100644 --- a/client/logs/tee_logger.go +++ b/client/logs/tee_logger.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "io" "maps" "os" "runtime" @@ -20,7 +19,6 @@ import ( type Gen3Logger struct { *slog.Logger mu sync.RWMutex - scoreboard *Scoreboard failedMu sync.Mutex FailedMap map[string]common.RetryObject // Maps filePath to FileMetadata @@ -43,21 +41,7 @@ func NewGen3Logger(logger *slog.Logger, logDir, profile string) *Gen3Logger { } } -// loadJSON is an internal helper to load JSON from a file path. -func loadJSON(path string, v any) { - data, err := os.ReadFile(path) - if err != nil { - if !os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "read log state %s: %v\n", path, err) - } - return - } - if len(data) > 0 { - if err := json.Unmarshal(data, v); err != nil { - fmt.Fprintf(os.Stderr, "decode log state %s: %v\n", path, err) - } - } -} + // --- Core logging helper --- @@ -127,20 +111,11 @@ func (t *Gen3Logger) Fatal(v ...any) { t.logWithSkip(context.Background(), slog.LevelError, 3, fmt.Sprint(v...)) } -// Writer returns os.Stderr for legacy compatibility (used by Scoreboard's tabwriter). -func (t *Gen3Logger) Writer() io.Writer { - return os.Stderr -} - // Slog exposes the underlying slog.Logger for code that needs direct slog access. func (t *Gen3Logger) Slog() *slog.Logger { return t.Logger } -// Scoreboard returns the embedded Scoreboard. -func (t *Gen3Logger) Scoreboard() *Scoreboard { - return t.scoreboard -} // --- Succeeded/Failed log map methods --- diff --git a/client/request/request.go b/client/request/request.go index 50b8c8e9..d9ef3dce 100644 --- a/client/request/request.go +++ b/client/request/request.go @@ -47,11 +47,7 @@ func (e *ResponseError) Error() string { type RequestOption func(*RequestBuilder) -func WithQuery(key, value string) RequestOption { - return func(rb *RequestBuilder) { - rb.WithQuery(key, value) - } -} + func WithQueryValues(v url.Values) RequestOption { return func(rb *RequestBuilder) { @@ -71,11 +67,7 @@ func WithTimeout(d time.Duration) RequestOption { } } -func WithToken(token string) RequestOption { - return func(rb *RequestBuilder) { - rb.WithToken(token) - } -} + func WithSkipAuth(skip bool) RequestOption { return func(rb *RequestBuilder) { diff --git a/client/request/request_test.go b/client/request/request_test.go index 1ce10f83..f8199e42 100644 --- a/client/request/request_test.go +++ b/client/request/request_test.go @@ -77,9 +77,6 @@ func TestRequest_Do_Success(t *testing.T) { if r.Header.Get("User-Agent") != "test-ua" { t.Errorf("Expected User-Agent 'test-ua', got '%s'", r.Header.Get("User-Agent")) } - if r.Header.Get("Authorization") != "Bearer test-token" { - t.Errorf("Expected Authorization 'test-token', got '%s'", r.Header.Get("Authorization")) - } return &http.Response{ StatusCode: http.StatusOK, Status: "200 OK", @@ -98,7 +95,7 @@ func TestRequest_Do_Success(t *testing.T) { var out struct { Status string `json:"status"` } - err := reqInterface.Do(ctx, "GET", "/api/test", nil, &out, WithToken("test-token")) + err := reqInterface.Do(ctx, "GET", "/api/test", nil, &out, WithSkipAuth(true)) if err != nil { t.Fatalf("Expected no error, got: %v", err) diff --git a/client/services/data_service.go b/client/services/data_service.go index a6b66405..d8ee211c 100644 --- a/client/services/data_service.go +++ b/client/services/data_service.go @@ -50,8 +50,8 @@ func (d *DataService) UploadBlank(ctx context.Context, req internalapi.InternalU func (d *DataService) UploadURL(ctx context.Context, req UploadURLRequest) (internalapi.InternalSignedURL, error) { params := &internalapi.InternalUploadURLParams{} - if req.FileName != "" { - params.FileName = &req.FileName + if req.Key != "" { + params.Key = &req.Key } if req.ExpiresIn > 0 { expires := int32(req.ExpiresIn) @@ -254,7 +254,7 @@ func (d *DataService) ResolveUploadURL(ctx context.Context, guid, filename strin organization, project := uploadScopeFromMetadata(metadata) resp, err := d.UploadURL(ctx, UploadURLRequest{ FileID: guid, - FileName: filename, + Key: filename, Organization: organization, Project: project, }) @@ -338,7 +338,7 @@ func (d *DataService) InitMultipartUploadWithMetadata(ctx context.Context, guid, organization, project := uploadScopeFromMetadata(metadata) req := internalapi.InternalMultipartInitRequest{ Guid: &guid, - FileName: &filename, + Key: &filename, Organization: nil, Project: nil, } @@ -496,19 +496,6 @@ func (d *DataService) CanonicalObjectURL(signedURL, bucketHint, fallbackDID stri } } -func (d *DataService) rememberUploadBucket(guid, bucket string) { - guid = strings.TrimSpace(guid) - bucket = strings.TrimSpace(bucket) - if guid == "" || bucket == "" { - return - } - d.uploadMu.Lock() - defer d.uploadMu.Unlock() - if d.uploads == nil { - d.uploads = map[string]string{} - } - d.uploads[guid] = bucket -} func (d *DataService) uploadBucket(guid string) string { guid = strings.TrimSpace(guid) diff --git a/client/services/data_service_canonical_url_test.go b/client/services/data_service_canonical_url_test.go index 5e0ac73d..959c98d9 100644 --- a/client/services/data_service_canonical_url_test.go +++ b/client/services/data_service_canonical_url_test.go @@ -67,17 +67,8 @@ func TestDataServiceCanonicalObjectURL_GCSAndAzure(t *testing.T) { fallback: "did:8", want: "s3://az-container/path/to/object.bin", }, - { - name: "scheme-less local upload path uses remembered bucket hint", - signedURL: "/tmp/syfon/object.bin", - bucketHint: "", - fallback: "did:9", - want: "s3://remembered-bucket/did:9", - }, } - d.rememberUploadBucket("did:9", "remembered-bucket") - for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { got, err := d.CanonicalObjectURL(tc.signedURL, tc.bucketHint, tc.fallback) diff --git a/client/services/data_service_test.go b/client/services/data_service_test.go index 21707903..ccbce326 100644 --- a/client/services/data_service_test.go +++ b/client/services/data_service_test.go @@ -124,10 +124,10 @@ func TestDataServiceOperationsAndTransferHelpers(t *testing.T) { t.Fatalf("unexpected upload blank request: %+v", lastUploadBlank) } - if _, err := service.UploadURL(ctx, UploadURLRequest{FileID: "file-1", FileName: "name.txt", ExpiresIn: 60, Organization: "org-a", Project: "proj-a"}); err != nil { + if _, err := service.UploadURL(ctx, UploadURLRequest{FileID: "file-1", Key: "name.txt", ExpiresIn: 60, Organization: "org-a", Project: "proj-a"}); err != nil { t.Fatalf("UploadURL returned error: %v", err) } - if uploadURLQuery.Get("organization") != "org-a" || uploadURLQuery.Get("project") != "proj-a" || uploadURLQuery.Get("file_name") != "name.txt" || uploadURLQuery.Get("expires_in") != "60" { + if uploadURLQuery.Get("organization") != "org-a" || uploadURLQuery.Get("project") != "proj-a" || uploadURLQuery.Get("key") != "name.txt" || uploadURLQuery.Get("expires_in") != "60" { t.Fatalf("unexpected upload URL query values: %v", uploadURLQuery) } @@ -170,7 +170,7 @@ func TestDataServiceOperationsAndTransferHelpers(t *testing.T) { if err != nil || uploadID != "upload-id" || guid != "multipart-guid" { t.Fatalf("InitMultipartUpload returned uploadID=%q guid=%q err=%v", uploadID, guid, err) } - if lastMultipartInit.Guid == nil || *lastMultipartInit.Guid != "guid-a" || lastMultipartInit.FileName == nil || *lastMultipartInit.FileName != "name.txt" { + if lastMultipartInit.Guid == nil || *lastMultipartInit.Guid != "guid-a" || lastMultipartInit.Key == nil || *lastMultipartInit.Key != "name.txt" { t.Fatalf("unexpected multipart init request: %+v", lastMultipartInit) } if lastMultipartInit.Organization != nil || lastMultipartInit.Project != nil { diff --git a/client/services/drs_service.go b/client/services/drs_service.go index 71cc2d2e..ae9966d1 100644 --- a/client/services/drs_service.go +++ b/client/services/drs_service.go @@ -307,10 +307,10 @@ func internalRecordToDRSObject(rec *internalapi.InternalRecord) drsapi.DrsObject SelfUri: "drs://" + rec.Did, Size: size, } - if rec.FileName != nil { - base := path.Base(strings.TrimSpace(*rec.FileName)) + if rec.Name != nil { + base := path.Base(strings.TrimSpace(*rec.Name)) if base == "." || base == "/" || base == "" { - base = strings.TrimSpace(*rec.FileName) + base = strings.TrimSpace(*rec.Name) } obj.Name = &base } diff --git a/client/services/drs_service_test.go b/client/services/drs_service_test.go index be88ef32..e0a579ae 100644 --- a/client/services/drs_service_test.go +++ b/client/services/drs_service_test.go @@ -42,7 +42,7 @@ func TestDRSServiceResolveAndList(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/objects/broken": w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) - _, _ = io.WriteString(w, `{"error":{"message":"column \"file_name\" does not exist"}}`) + _, _ = io.WriteString(w, `{"error":{"message":"schema is stale"}}`) case r.Method == http.MethodGet && r.URL.Path == "/objects/obj-1/access/acc-1": writeJSON(t, w, http.StatusOK, drsapi.AccessURL{Url: "https://signed.example/access"}) case r.Method == http.MethodGet && r.URL.Path == "/objects/checksum/abc": @@ -89,7 +89,7 @@ func TestDRSServiceResolveAndList(t *testing.T) { recordHashes := internalapi.HashInfo{"sha256": "abc123"} recordAuthz := map[string][]string{"org1": {"p1"}} record := testRecordForURL("did-record", "https://storage.example/record.bin", recordAuthz) - record.FileName = &recordName + record.Name = &recordName record.Size = &recordSize record.Hashes = &recordHashes records := []internalapi.InternalRecord{record} @@ -121,7 +121,7 @@ func TestDRSServiceResolveAndList(t *testing.T) { if _, err := service.GetObject(ctx, "missing"); !errors.Is(err, ErrObjectNotFound) { t.Fatalf("expected ErrObjectNotFound, got %v", err) } - if _, err := service.GetObject(ctx, "broken"); err == nil || !strings.Contains(err.Error(), `column "file_name" does not exist`) { + if _, err := service.GetObject(ctx, "broken"); err == nil || !strings.Contains(err.Error(), "schema is stale") { t.Fatalf("expected server error detail for broken object lookup, got %v", err) } @@ -282,7 +282,7 @@ func TestDRSServiceDeleteRecordsByHash(t *testing.T) { testRecordForURL("obj-2", recordURLs[0], map[string][]string{"org1": {}}), } for i := range records { - records[i].FileName = &recordName + records[i].Name = &recordName records[i].Size = &recordSize records[i].Hashes = &recordHashes } diff --git a/client/services/index_data_drs_helpers_test.go b/client/services/index_data_drs_helpers_test.go index 46a87f87..b332f3c2 100644 --- a/client/services/index_data_drs_helpers_test.go +++ b/client/services/index_data_drs_helpers_test.go @@ -94,7 +94,7 @@ func toRecordResponse(rec internalapi.InternalRecord) internalapi.InternalRecord AccessMethods: rec.AccessMethods, ControlledAccess: rec.ControlledAccess, Description: rec.Description, - FileName: rec.FileName, + Name: rec.Name, Hashes: rec.Hashes, Size: rec.Size, Version: rec.Version, diff --git a/client/services/index_service.go b/client/services/index_service.go index 3b696911..95ae507b 100644 --- a/client/services/index_service.go +++ b/client/services/index_service.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + "path" "strings" "github.com/calypr/syfon/apigen/client/drs" @@ -107,9 +108,6 @@ func (s *IndexService) List(ctx context.Context, opts ListRecordsOptions) (inter if opts.ProjectID != "" { params.Set("project", opts.ProjectID) } - if opts.Path != "" { - params.Set("path", opts.Path) - } if opts.Limit != 0 { params.Set("limit", fmt.Sprintf("%d", opts.Limit)) } @@ -197,6 +195,20 @@ func (s *IndexService) BulkSHA256Validity(ctx context.Context, req internalapi.B return *resp.JSON200, nil } +// MissingSHA256 returns the SHA-256 values that are not registered in a +// project. Syfon performs this as an indexed existence check and does not +// return complete DRS records. +func (s *IndexService) MissingSHA256(ctx context.Context, req internalapi.BulkMissingSHA256Request) (internalapi.BulkMissingSHA256Response, error) { + resp, err := s.gen.InternalBulkMissingSHA256WithResponse(ctx, internalapi.InternalBulkMissingSHA256JSONRequestBody(req)) + if err != nil { + return internalapi.BulkMissingSHA256Response{}, err + } + if resp.JSON200 == nil { + return internalapi.BulkMissingSHA256Response{}, fmt.Errorf("failed to find missing sha256 values: %d", resp.StatusCode()) + } + return *resp.JSON200, nil +} + func (s *IndexService) BulkDocuments(ctx context.Context, dids []string) ([]internalapi.InternalRecordResponse, error) { var body internalapi.BulkDocumentsRequest if err := body.FromBulkDocumentsRequest0(internalapi.BulkDocumentsRequest0(dids)); err != nil { @@ -216,7 +228,11 @@ func (s *IndexService) SHA256Validity(ctx context.Context, values []string) (map return s.BulkSHA256Validity(ctx, internalapi.BulkSHA256ValidityRequest{Sha256: &values}) } -func (s *IndexService) Upsert(ctx context.Context, did, objectURL, fileName string, size int64, sha256sum string, authorizations map[string][]string) error { +func (s *IndexService) Upsert(ctx context.Context, did, objectURL, recordPath string, size int64, sha256sum string, authorizations map[string][]string) error { + recordName := strings.TrimSpace(recordPath) + if recordName != "" { + recordName = path.Base(strings.Trim(recordName, "/")) + } existing, err := s.Get(ctx, did) if err == nil { req := internalapi.InternalRecord{ @@ -224,7 +240,7 @@ func (s *IndexService) Upsert(ctx context.Context, did, objectURL, fileName stri AccessMethods: existing.AccessMethods, ControlledAccess: existing.ControlledAccess, Description: existing.Description, - FileName: existing.FileName, + Name: existing.Name, Hashes: existing.Hashes, Size: existing.Size, Version: existing.Version, @@ -242,8 +258,8 @@ func (s *IndexService) Upsert(ctx context.Context, did, objectURL, fileName stri controlled := syfoncommon.AuthzMapToControlledAccess(authorizations) req.ControlledAccess = &controlled } - if fileName != "" { - req.FileName = &fileName + if recordName != "" { + req.Name = &recordName } if size > 0 { req.Size = &size @@ -274,8 +290,8 @@ func (s *IndexService) Upsert(ctx context.Context, did, objectURL, fileName stri if size > 0 { payload.Size = &size } - if fileName != "" { - payload.FileName = &fileName + if recordName != "" { + payload.Name = &recordName } if sha256sum != "" { h := internalapi.HashInfo{"sha256": sha256sum} diff --git a/client/services/index_service_test.go b/client/services/index_service_test.go index 63540b8b..8c5e18ae 100644 --- a/client/services/index_service_test.go +++ b/client/services/index_service_test.go @@ -38,7 +38,7 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { size := int64(12) authz := map[string][]string{"p1": {}} rec := testRecordForURL("did-list", "s3://bucket/object", authz) - rec.FileName = &name + rec.Name = &name rec.Size = &size records := []internalapi.InternalRecord{rec} writeJSON(t, w, http.StatusOK, internalapi.ListRecordsResponse{Records: &records}) @@ -85,14 +85,14 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { hashes := internalapi.HashInfo{"md5": "md5sum"} authz := map[string][]string{"existing": {}} rec := toRecordResponse(testRecordForURL("did-update", "s3://bucket/existing", authz)) - rec.FileName = &fileName + rec.Name = &fileName rec.Size = &size rec.Hashes = &hashes writeJSON(t, w, http.StatusOK, internalapi.InternalRecordResponse{ Did: rec.Did, AccessMethods: rec.AccessMethods, ControlledAccess: rec.ControlledAccess, - FileName: rec.FileName, + Name: rec.Name, Size: rec.Size, Hashes: rec.Hashes, }) @@ -125,7 +125,7 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { listSize := int64(12) listAuthz := map[string][]string{"p1": {}} listRec := testRecordForURL("did-list", "s3://bucket/object", listAuthz) - listRec.FileName = &listName + listRec.Name = &listName listRec.Size = &listSize listRecords := []internalapi.InternalRecord{listRec} listResp, err := json.Marshal(internalapi.ListRecordsResponse{Records: &listRecords}) @@ -150,12 +150,12 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { createSize := int64(55) createAuthz := map[string][]string{"p1": {}} createRec := testRecordForURL("did-new", "s3://bucket/created", createAuthz) - createRec.FileName = &createFile + createRec.Name = &createFile createRec.Size = &createSize if _, err := service.Create(ctx, createRec); err != nil { t.Fatalf("Create returned error: %v", err) } - if lastCreated.Did != "did-new" || lastCreated.FileName == nil || *lastCreated.FileName != "created.txt" { + if lastCreated.Did != "did-new" || lastCreated.Name == nil || *lastCreated.Name != "created.txt" { t.Fatalf("unexpected create payload: %+v", lastCreated) } @@ -181,14 +181,14 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { t.Fatalf("unexpected remove controlled access payload: %+v", lastRemoveControlled) } - if _, err := service.List(ctx, ListRecordsOptions{Hash: "sha", URL: "s3://bucket/path", Organization: "org", ProjectID: "proj", Path: "nested", Limit: 3, Start: "did-100"}); err != nil { + if _, err := service.List(ctx, ListRecordsOptions{Hash: "sha", URL: "s3://bucket/path", Organization: "org", ProjectID: "proj", Limit: 3, Start: "did-100"}); err != nil { t.Fatalf("List returned error: %v", err) } query, err := url.ParseQuery(strings.TrimPrefix(requester.builder.Url, "/index?")) if err != nil { t.Fatalf("parse list query: %v", err) } - if query.Get("hash") != "sha" || query.Get("url") != "s3://bucket/path" || query.Get("organization") != "org" || query.Get("project") != "proj" || query.Get("path") != "nested" || query.Get("limit") != "3" || query.Get("start") != "did-100" || query.Get("page") != "" { + if query.Get("hash") != "sha" || query.Get("url") != "s3://bucket/path" || query.Get("organization") != "org" || query.Get("project") != "proj" || query.Get("limit") != "3" || query.Get("start") != "did-100" || query.Get("page") != "" { t.Fatalf("unexpected list query values: %v", query) } @@ -253,7 +253,7 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { if err != nil { t.Fatalf("Upsert existing returned error: %v", err) } - if lastUpdated.FileName == nil || *lastUpdated.FileName != "new.txt" { + if lastUpdated.Name == nil || *lastUpdated.Name != "new.txt" { t.Fatalf("expected updated file name, got %+v", lastUpdated) } if lastUpdated.Size == nil || *lastUpdated.Size != 123 { @@ -279,7 +279,7 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { if lastCreated.Did != "did-create" || lastCreated.ControlledAccess == nil || len(*lastCreated.ControlledAccess) != 1 || lastCreated.AccessMethods == nil || len(*lastCreated.AccessMethods) != 1 { t.Fatalf("unexpected create-on-upsert payload: %+v", lastCreated) } - if lastCreated.FileName == nil || *lastCreated.FileName != "created.txt" || lastCreated.Size == nil || *lastCreated.Size != 99 { + if lastCreated.Name == nil || *lastCreated.Name != "created.txt" || lastCreated.Size == nil || *lastCreated.Size != 99 { t.Fatalf("unexpected create-on-upsert sizing: %+v", lastCreated) } @@ -289,6 +289,40 @@ func TestIndexServiceOperationsAndUpsert(t *testing.T) { } } +func TestIndexServiceMissingSHA256(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/index/bulk/sha256/missing" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + var req internalapi.BulkMissingSHA256Request + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if req.Organization != "org" || req.Project != "project" || len(req.Sha256) != 2 { + t.Fatalf("unexpected request body: %+v", req) + } + writeJSON(t, w, http.StatusOK, internalapi.BulkMissingSHA256Response{ + Checked: 2, + MissingSha256: []string{"missing"}, + }) + })) + defer server.Close() + + service := NewIndexService(mustInternalClient(t, server.URL), &fakeRequester{}) + got, err := service.MissingSHA256(context.Background(), internalapi.BulkMissingSHA256Request{ + Organization: "org", + Project: "project", + Sha256: []string{"present", "missing"}, + }) + if err != nil { + t.Fatalf("MissingSHA256 returned error: %v", err) + } + if got.Checked != 2 || len(got.MissingSha256) != 1 || got.MissingSha256[0] != "missing" { + t.Fatalf("unexpected response: %+v", got) + } +} + func TestIndexServiceRemoveControlledAccessRequiresJSON200(t *testing.T) { t.Parallel() diff --git a/client/services/metrics_service.go b/client/services/metrics_service.go index e72b1001..d6afd7eb 100644 --- a/client/services/metrics_service.go +++ b/client/services/metrics_service.go @@ -18,7 +18,10 @@ func NewMetricsService(gen metricsapi.ClientWithResponsesInterface) *MetricsServ } func (s *MetricsService) Summary(ctx context.Context, opts MetricsSummaryOptions) (metricsapi.FileUsageSummary, error) { - params := &metricsapi.GetMetricsSummaryParams{} + params := &metricsapi.GetMetricsSummaryParams{ + Organization: stringPtr[metricsapi.Organization](opts.Organization), + Project: stringPtr[metricsapi.Project](opts.ProjectID), + } if opts.InactiveDays > 0 { params.InactiveDays = &opts.InactiveDays } @@ -33,7 +36,10 @@ func (s *MetricsService) Summary(ctx context.Context, opts MetricsSummaryOptions } func (s *MetricsService) Files(ctx context.Context, opts MetricsFilesOptions) ([]metricsapi.FileUsage, error) { - params := &metricsapi.ListMetricsFilesParams{} + params := &metricsapi.ListMetricsFilesParams{ + Organization: stringPtr[metricsapi.Organization](opts.Organization), + Project: stringPtr[metricsapi.Project](opts.ProjectID), + } if opts.Limit > 0 { params.Limit = &opts.Limit } diff --git a/client/services/services_test.go b/client/services/services_test.go index 24c5492c..723fe6ef 100644 --- a/client/services/services_test.go +++ b/client/services/services_test.go @@ -146,6 +146,14 @@ func (f *fakeMetricsClient) ListMetricsFilesWithResponse(ctx context.Context, pa return f.filesResp, f.filesErr } +func (f *fakeMetricsClient) BulkMetricsFilesWithBodyWithResponse(ctx context.Context, params *metricsapi.BulkMetricsFilesParams, contentType string, body io.Reader, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.BulkMetricsFilesResponse, error) { + return &metricsapi.BulkMetricsFilesResponse{HTTPResponse: &http.Response{StatusCode: http.StatusNotImplemented}}, nil +} + +func (f *fakeMetricsClient) BulkMetricsFilesWithResponse(ctx context.Context, params *metricsapi.BulkMetricsFilesParams, body metricsapi.BulkMetricsFilesJSONRequestBody, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.BulkMetricsFilesResponse, error) { + return &metricsapi.BulkMetricsFilesResponse{HTTPResponse: &http.Response{StatusCode: http.StatusNotImplemented}}, nil +} + func (f *fakeMetricsClient) GetMetricsFileWithResponse(ctx context.Context, objectId string, params *metricsapi.GetMetricsFileParams, reqEditors ...metricsapi.RequestEditorFn) (*metricsapi.GetMetricsFileResponse, error) { f.fileObjectID = objectId return f.fileResp, f.fileErr diff --git a/client/services/types.go b/client/services/types.go index a920b794..350dc87b 100644 --- a/client/services/types.go +++ b/client/services/types.go @@ -22,7 +22,6 @@ type ListRecordsOptions struct { URL string Organization string ProjectID string - Path string Limit int Start string Page int @@ -30,7 +29,7 @@ type ListRecordsOptions struct { type UploadURLRequest struct { FileID string - FileName string + Key string ExpiresIn int Organization string Project string @@ -50,6 +49,22 @@ type MetricsSummaryOptions struct { ProjectID string } +type StorageSummaryOptions struct { + Organization string + ProjectID string + Path string +} + +type StorageChildrenOptions struct { + Organization string + ProjectID string + Path string + Limit int + Offset int + SortBy string + SortOrder string +} + type TransferMetricsOptions struct { Organization string ProjectID string diff --git a/client/transfer.go b/client/transfer.go deleted file mode 100644 index d148bae4..00000000 --- a/client/transfer.go +++ /dev/null @@ -1,230 +0,0 @@ -package client - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/calypr/syfon/client/common" - sylogs "github.com/calypr/syfon/client/logs" - "github.com/calypr/syfon/client/transfer" - sydownload "github.com/calypr/syfon/client/transfer/download" - syupload "github.com/calypr/syfon/client/transfer/upload" -) - -type UploadOptions struct { - Bucket string - IncludeSubDirName bool - HasMetadata bool - Batch bool - NumParallel int - ManifestPath string - RetryFailedLogPath string - ShowProgress bool - ForceMultipart bool - GUID string -} - -type DownloadOptions struct { - DownloadPath string - FilenameFormat string - Rename bool - NoPrompt bool - Protocol string - NumParallel int - SkipCompleted bool - ManifestPath string -} - -type uploadManifestEntry struct { - GUID string `json:"guid"` - Title string `json:"title"` -} - -type downloadManifestEntry struct { - GUID string `json:"guid"` -} - -// Upload handles local file uploads, manifest-driven uploads, and retrying failed uploads. -func Upload(ctx context.Context, bk transfer.MultipartBackend, sourcePath string, opts UploadOptions) error { - if bk == nil { - return fmt.Errorf("backend is required") - } - logger := bk.Logger() - if logger == nil { - logger = sylogs.NewGen3Logger(nil, "", "") - } - if strings.TrimSpace(opts.Bucket) == "" { - return fmt.Errorf("bucket is required") - } - if opts.NumParallel <= 0 { - opts.NumParallel = 1 - } - - if strings.TrimSpace(opts.RetryFailedLogPath) != "" { - return retryFailedUploadsFromFile(ctx, bk, logger, opts.RetryFailedLogPath) - } - - if strings.TrimSpace(opts.ManifestPath) != "" { - return uploadFromManifest(ctx, bk, logger, sourcePath, opts) - } - return uploadFromPath(ctx, bk, logger, sourcePath, opts) -} - -// Download handles single- and multi-file downloads and manifest expansion. -func Download(ctx context.Context, api *Client, bk transfer.ReadBackend, guids []string, opts DownloadOptions) error { - if api == nil { - return fmt.Errorf("client is required") - } - if bk == nil { - return fmt.Errorf("backend is required") - } - - if strings.TrimSpace(opts.ManifestPath) != "" { - manifestGuids, err := loadDownloadManifest(opts.ManifestPath) - if err != nil { - return err - } - guids = append(guids, manifestGuids...) - } - - if len(guids) == 0 { - return fmt.Errorf("no guids provided for download") - } - - downloadPath := strings.TrimSpace(opts.DownloadPath) - if downloadPath == "" { - downloadPath = "." - } - - if opts.NumParallel <= 0 { - opts.NumParallel = 1 - } - - return sydownload.DownloadMultiple( - ctx, - api.DRS(), - bk, - guids, - downloadPath, - opts.NumParallel, - opts.SkipCompleted, - ) -} - -func uploadFromPath(ctx context.Context, bk transfer.MultipartBackend, logger transfer.TransferLogger, sourcePath string, opts UploadOptions) error { - absUploadPath, err := common.GetAbsolutePath(sourcePath) - if err != nil { - return fmt.Errorf("resolve upload path: %w", err) - } - filePaths, err := common.ParseFilePaths(absUploadPath, opts.HasMetadata) - if err != nil { - return fmt.Errorf("parse upload paths: %w", err) - } - if len(filePaths) == 0 { - return fmt.Errorf("no files found under %s", absUploadPath) - } - - // For Batch uploads, we'll implement a simple loop calling the engine uploader. - // This replaces the complex syupload.BatchUpload. - var uploadErr error - for _, filePath := range filePaths { - // Note: ProcessFilename logic still useful for calculating destination object keys. - // We'll keep it in syupload/utils.go for now but updated to the new logger. - src, key, metadata, err := syupload.ProcessFilename(logger, absUploadPath, filePath, "", opts.IncludeSubDirName, opts.HasMetadata) - if err != nil { - logger.Failed(filePath, filepath.Base(filePath), common.FileMetadata{}, "", 0, false) - if uploadErr == nil { - uploadErr = err - } - continue - } - - if err := syupload.Upload(ctx, bk, src, key, opts.GUID, opts.Bucket, metadata, opts.ShowProgress, opts.ForceMultipart); err != nil { - logger.Error("Upload failed", "path", src, "error", err) - if uploadErr == nil { - uploadErr = err - } - } - } - - return uploadErr -} - -func uploadFromManifest(ctx context.Context, bk transfer.MultipartBackend, logger transfer.TransferLogger, uploadPath string, opts UploadOptions) error { - absUploadPath, err := common.GetAbsolutePath(uploadPath) - if err != nil { - return fmt.Errorf("resolve upload path: %w", err) - } - manifestBytes, err := os.ReadFile(opts.ManifestPath) - if err != nil { - return fmt.Errorf("read manifest %s: %w", opts.ManifestPath, err) - } - - var objects []uploadManifestEntry - if err := json.Unmarshal(manifestBytes, &objects); err != nil { - return fmt.Errorf("parse manifest %s: %w", opts.ManifestPath, err) - } - - var uploadErr error - for _, obj := range objects { - localFilePath := filepath.Join(absUploadPath, obj.Title) - src, key, metadata, err := syupload.ProcessFilename(logger, absUploadPath, localFilePath, obj.GUID, opts.IncludeSubDirName, opts.HasMetadata) - if err != nil { - logger.Failed(localFilePath, filepath.Base(localFilePath), common.FileMetadata{}, obj.GUID, 0, false) - if uploadErr == nil { - uploadErr = err - } - continue - } - if err := syupload.Upload(ctx, bk, src, key, obj.GUID, opts.Bucket, metadata, opts.ShowProgress, opts.ForceMultipart); err != nil { - logger.Error("Upload failed", "path", src, "guid", obj.GUID, "error", err) - if uploadErr == nil { - uploadErr = err - } - } - } - - return uploadErr -} - -func retryFailedUploadsFromFile(ctx context.Context, bk transfer.MultipartBackend, logger transfer.TransferLogger, failedLogPath string) error { - data, err := os.ReadFile(failedLogPath) - if err != nil { - return fmt.Errorf("read failed log %s: %w", failedLogPath, err) - } - var failedMap map[string]common.RetryObject - if err := json.Unmarshal(data, &failedMap); err != nil { - return fmt.Errorf("parse failed log %s: %w", failedLogPath, err) - } - - for _, ro := range failedMap { - if err := syupload.Upload(ctx, bk, ro.SourcePath, ro.ObjectKey, ro.GUID, ro.Bucket, ro.FileMetadata, true, ro.Multipart); err != nil { - logger.Error("Retry failed", "path", ro.SourcePath, "error", err) - } else { - logger.DeleteFromFailedLog(ro.SourcePath) - } - } - return nil -} - -func loadDownloadManifest(path string) ([]string, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("read manifest %s: %w", path, err) - } - var objects []downloadManifestEntry - if err := json.Unmarshal(data, &objects); err != nil { - return nil, fmt.Errorf("parse manifest %s: %w", path, err) - } - guids := make([]string, 0, len(objects)) - for _, obj := range objects { - if strings.TrimSpace(obj.GUID) != "" { - guids = append(guids, obj.GUID) - } - } - return guids, nil -} diff --git a/client/transfer/download/downloader.go b/client/transfer/download/downloader.go deleted file mode 100644 index 16523a7e..00000000 --- a/client/transfer/download/downloader.go +++ /dev/null @@ -1,138 +0,0 @@ -package download - -import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - - "github.com/calypr/syfon/client/common" - "github.com/calypr/syfon/client/transfer" - "github.com/calypr/syfon/client/transfer/engine" - "github.com/vbauerster/mpb/v8" - "github.com/vbauerster/mpb/v8/decor" - "golang.org/x/sync/errgroup" -) - -type Resolver interface { - Resolve(ctx context.Context, id string) (*transfer.ResolvedObject, error) -} - -func DownloadMultiple( - ctx context.Context, - dc Resolver, - bk transfer.ReadBackend, - guids []string, - downloadPath string, - numParallel int, - skipCompleted bool, -) error { - logger := bk.Logger() - if numParallel < 1 { - numParallel = 1 - } - - downloadPath, err := common.ParseRootPath(downloadPath) - if err != nil { - return fmt.Errorf("invalid download path: %w", err) - } - if !strings.HasSuffix(downloadPath, "/") { - downloadPath += "/" - } - - if err := os.MkdirAll(downloadPath, 0766); err != nil { - return fmt.Errorf("cannot create directory %s: %w", downloadPath, err) - } - - toDownload, skipped, renamed, err := prepareFiles(ctx, dc, bk, guids, downloadPath, "original", true, skipCompleted, "https") - if err != nil { - return err - } - - if len(toDownload) == 0 { - logger.Info("No files to download.") - return nil - } - - g, gctx := errgroup.WithContext(ctx) - g.SetLimit(numParallel) - - downloader := &engine.GenericDownloader{Source: bk} - var success atomic.Int64 - var mu sync.Mutex - var allErrors []error - - for _, fdr := range toDownload { - fdr := fdr // capture - g.Go(func() error { - dst := filepath.Join(fdr.downloadPath, fdr.filename) - // Using 64MB chunks for parallel download by default - if err := downloader.Download(gctx, fdr.guid, dst, 4, 64*common.MB, int64(5*common.GB)); err != nil { - mu.Lock() - allErrors = append(allErrors, fmt.Errorf("guid %s: %w", fdr.guid, err)) - mu.Unlock() - return err - } - success.Add(1) - return nil - }) - } - - if err := g.Wait(); err != nil && len(allErrors) == 0 { - return err - } - - printRenamed(ctx, logger.Slog(), renamed) - printSkipped(ctx, logger.Slog(), skipped) - - if len(allErrors) > 0 { - return fmt.Errorf("some downloads failed: %v", allErrors) - } - - return nil -} - -func prepareFiles( - ctx context.Context, - dc Resolver, - bk transfer.ReadBackend, - guids []string, - downloadPath, filenameFormat string, - rename, skipCompleted bool, - protocol string, -) ([]downloadRequest, []RenamedOrSkippedFileInfo, []RenamedOrSkippedFileInfo, error) { - logger := bk.Logger() - renamed := make([]RenamedOrSkippedFileInfo, 0) - skipped := make([]RenamedOrSkippedFileInfo, 0) - toDownload := make([]downloadRequest, 0, len(guids)) - - p := mpb.New(mpb.WithOutput(os.Stdout)) - bar := p.AddBar(int64(len(guids)), mpb.PrependDecorators(decor.Name("Preparing "), decor.CountersNoUnit("%d / %d")), mpb.AppendDecorators(decor.Percentage())) - - for _, guid := range guids { - if guid == "" { - bar.Increment() - continue - } - info, err := GetFileInfo(ctx, dc, logger, guid, protocol, downloadPath, filenameFormat, rename, &renamed) - if err != nil { - p.Wait() - return nil, nil, nil, err - } - fdr := downloadRequest{downloadPath: downloadPath, filename: info.Name, guid: guid} - if !rename { - validateLocalFileStat(logger.Slog(), &fdr, int64(info.Size), skipCompleted) - } - if fdr.skip { - skipped = append(skipped, RenamedOrSkippedFileInfo{GUID: fdr.guid, OldFilename: fdr.filename}) - } else { - toDownload = append(toDownload, fdr) - } - bar.Increment() - } - p.Wait() - return toDownload, skipped, renamed, nil -} diff --git a/client/transfer/download/file_info.go b/client/transfer/download/file_info.go deleted file mode 100644 index 3851fcf0..00000000 --- a/client/transfer/download/file_info.go +++ /dev/null @@ -1,56 +0,0 @@ -package download - -import ( - "context" - "strings" - - "github.com/calypr/syfon/client/transfer" -) - -func GetFileInfo( - ctx context.Context, - dc Resolver, - logger transfer.TransferLogger, - guid, protocol, downloadPath, filenameFormat string, - rename bool, - renamedFiles *[]RenamedOrSkippedFileInfo, -) (*IndexdResponse, error) { - drsObj, err := dc.Resolve(ctx, guid) - if err != nil { - logger.Warn("Failed to get file details", "guid", guid, "error", err) - // Fallback: use GUID as filename if failed? - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: guid, NewFilename: guid}) - return &IndexdResponse{Name: guid, Size: 0}, nil - } - - name := strings.TrimSpace(drsObj.Name) - if name == "" { - name = guid - } - finalName := applyFilenameFormat(name, guid, downloadPath, filenameFormat, rename, renamedFiles) - return &IndexdResponse{Name: finalName, Size: drsObj.Size}, nil -} - -func applyFilenameFormat(baseName, guid, downloadPath, format string, rename bool, renamedFiles *[]RenamedOrSkippedFileInfo) string { - switch format { - case "guid": - return guid - case "combined": - return guid + "_" + baseName - case "original": - if !rename { - return baseName - } - newName := processOriginalFilename(downloadPath, baseName) - if newName != baseName { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{ - GUID: guid, - OldFilename: baseName, - NewFilename: newName, - }) - } - return newName - default: - return baseName - } -} diff --git a/client/transfer/download/transfer.go b/client/transfer/download/transfer.go index 9fa1e30a..c42bc0e3 100644 --- a/client/transfer/download/transfer.go +++ b/client/transfer/download/transfer.go @@ -2,9 +2,7 @@ package download import ( "context" - "path/filepath" - "github.com/calypr/syfon/client/common" "github.com/calypr/syfon/client/transfer" "github.com/calypr/syfon/client/transfer/engine" ) @@ -16,21 +14,6 @@ type DownloadOptions struct { RetryStrategy transfer.RetryStrategy } -// DownloadToPath downloads a single object using the generic EngineDownloader. -func DownloadToPath( - ctx context.Context, - bk transfer.ReadBackend, - guid string, - dstPath string, -) error { - opts := DownloadOptions{ - MultipartThreshold: int64(5 * common.GB), - Concurrency: 8, - ChunkSize: 64 * common.MB, - } - return DownloadToPathWithOptions(ctx, bk, guid, dstPath, opts) -} - func DownloadToPathWithOptions( ctx context.Context, bk transfer.ReadBackend, @@ -41,21 +24,3 @@ func DownloadToPathWithOptions( downloader := &engine.GenericDownloader{Source: bk, RetryStrategy: opts.RetryStrategy} return downloader.Download(ctx, guid, dstPath, opts.Concurrency, opts.ChunkSize, opts.MultipartThreshold) } - -// DownloadSingleWithProgress preserves the old single-download test entrypoint while -// routing the actual transfer through the engine. -func DownloadSingleWithProgress(ctx context.Context, dc Resolver, bk transfer.ReadBackend, guid, downloadPath, filenameFormat string) error { - logger := bk.Logger() - info, err := GetFileInfo(ctx, dc, logger, guid, "https", downloadPath, filenameFormat, true, nil) - if err != nil { - return err - } - - dstPath := filepath.Join(downloadPath, info.Name) - opts := DownloadOptions{ - MultipartThreshold: 1 * common.GB, - Concurrency: 2, - ChunkSize: 64 * common.MB, - } - return DownloadToPathWithOptions(ctx, bk, guid, dstPath, opts) -} diff --git a/client/transfer/download/transfer_test.go b/client/transfer/download/transfer_test.go index c75a845f..05fac81c 100644 --- a/client/transfer/download/transfer_test.go +++ b/client/transfer/download/transfer_test.go @@ -188,78 +188,7 @@ func (f *fakeResolver) Logger() transfer.TransferLogger { return f.backend.Logger() } -func TestDownloadSingleWithProgressEmitsEvents(t *testing.T) { - payload := bytes.Repeat([]byte("d"), 64) - downloadDir := t.TempDir() - downloadPath := downloadDir + string(os.PathSeparator) - var events []common.ProgressEvent - progress := func(event common.ProgressEvent) error { - events = append(events, event) - return nil - } - - fake := &fakeBackend{ - logger: logs.NewGen3Logger(nil, "", ""), - data: payload, - } - dc := &fakeResolver{backend: fake} - - ctx := common.WithProgress(context.Background(), progress) - err := DownloadSingleWithProgress(ctx, dc, fake, "guid-123", downloadPath, "") - if err != nil { - t.Fatalf("download failed: %v", err) - } - - if len(events) == 0 { - t.Fatal("expected progress events") - } - for i := 1; i < len(events); i++ { - if events[i].BytesSoFar < events[i-1].BytesSoFar { - t.Fatalf("bytesSoFar not monotonic: %d then %d", events[i-1].BytesSoFar, events[i].BytesSoFar) - } - } - last := events[len(events)-1] - if last.BytesSoFar != int64(len(payload)) { - t.Fatalf("expected final bytesSoFar %d, got %d", len(payload), last.BytesSoFar) - } - fullPath := filepath.Join(downloadPath, "payload.bin") - if _, err := os.Stat(fullPath); err != nil { - t.Fatalf("expected file to exist: %v", err) - } -} - -func TestDownloadSingleWithProgressFinalizeOnError(t *testing.T) { - downloadDir := t.TempDir() - downloadPath := downloadDir + string(os.PathSeparator) - - var events []common.ProgressEvent - progress := func(event common.ProgressEvent) error { - events = append(events, event) - return nil - } - - fake := &fakeBackend{ - logger: logs.NewGen3Logger(nil, "", ""), - data: []byte("short"), - size: 64, - } - dc := &fakeResolver{backend: fake} - - ctx := common.WithProgress(context.Background(), progress) - err := DownloadSingleWithProgress(ctx, dc, fake, "guid-123", downloadPath, "") - if err == nil { - t.Fatal("expected download error") - } - - if len(events) == 0 { - t.Fatal("expected progress events") - } - last := events[len(events)-1] - if last.BytesSoFar != int64(len(fake.data)) { - t.Fatalf("expected finalize bytesSoFar %d, got %d", len(fake.data), last.BytesSoFar) - } -} func newDownloadJSONResponse(rawURL, body string) *http.Response { parsedURL, err := url.Parse(rawURL) diff --git a/client/transfer/download/types.go b/client/transfer/download/types.go deleted file mode 100644 index 605b295d..00000000 --- a/client/transfer/download/types.go +++ /dev/null @@ -1,74 +0,0 @@ -package download - -import ( - "io" - "log/slog" - "net/http" - "os" -) - -type IndexdResponse struct { - Name string - Size int64 -} -type RenamedOrSkippedFileInfo struct { - GUID string - OldFilename string - NewFilename string -} - -type downloadRequest struct { - downloadPath string - filename string - guid string - presignedURL string - rangeBytes int64 - rangeStart *int64 - rangeEnd *int64 - overwrite bool - skip bool - response *http.Response - writer io.Writer -} - -func validateLocalFileStat( - logger *slog.Logger, - fdr *downloadRequest, - filesize int64, - skipCompleted bool, -) { - fullPath := fdr.downloadPath + fdr.filename - - fi, err := os.Stat(fullPath) - if err != nil { - if os.IsNotExist(err) { - // No local file → full download, nothing special - return - } - logger.Error("Error statting local file", "path", fullPath, "error", err) - logger.Info("Will attempt full download anyway") - return - } - - localSize := fi.Size() - - // User doesn't want to skip completed files → force full overwrite - if !skipCompleted { - fdr.overwrite = true - return - } - - // Exact match → skip entirely - if localSize == filesize { - fdr.skip = true - return - } - - // Local file larger than expected → overwrite fully (corrupted or different file) - if localSize > filesize { - fdr.overwrite = true - return - } - - fdr.rangeBytes = localSize -} diff --git a/client/transfer/download/utils.go b/client/transfer/download/utils.go deleted file mode 100644 index d5488652..00000000 --- a/client/transfer/download/utils.go +++ /dev/null @@ -1,77 +0,0 @@ -package download - -import ( - "bufio" - "os" - "path/filepath" - "strconv" - "strings" - - "context" - "fmt" - "log/slog" -) - -// AskForConfirmation asks user for confirmation before proceed, will wait if user entered garbage -func AskForConfirmation(logger *slog.Logger, s string) bool { - reader := bufio.NewReader(os.Stdin) - - for { - logger.Info(fmt.Sprintf("%s [y/n]: ", s)) - - response, err := reader.ReadString('\n') - if err != nil { - logger.Error("Error occurred during parsing user's confirmation: " + err.Error()) - return false - } - - switch strings.ToLower(strings.TrimSpace(response)) { - case "y", "yes": - return true - case "n", "no": - return false - default: - return false // Example of defaulting to false - } - } -} - -func processOriginalFilename(downloadPath string, actualFilename string) string { - _, err := os.Stat(downloadPath + actualFilename) - if os.IsNotExist(err) { - return actualFilename - } - extension := filepath.Ext(actualFilename) - filename := strings.TrimSuffix(actualFilename, extension) - counter := 2 - for { - newFilename := filename + "_" + strconv.Itoa(counter) + extension - _, err := os.Stat(downloadPath + newFilename) - if os.IsNotExist(err) { - return newFilename - } - counter++ - } -} - -// truncateFilename shortens long filenames for progress bar display -func truncateFilename(name string, max int) string { - if len(name) <= max { - return name - } - return "..." + name[len(name)-max+3:] -} - -// printRenamed shows renamed files in final summary -func printRenamed(ctx context.Context, logger *slog.Logger, renamed []RenamedOrSkippedFileInfo) { - for _, r := range renamed { - logger.InfoContext(ctx, fmt.Sprintf("Renamed %q to %q (GUID: %s)", r.OldFilename, r.NewFilename, r.GUID)) - } -} - -// printSkipped shows skipped files in final summary -func printSkipped(ctx context.Context, logger *slog.Logger, skipped []RenamedOrSkippedFileInfo) { - for _, s := range skipped { - logger.InfoContext(ctx, fmt.Sprintf("Skipped %q (GUID: %s)", s.OldFilename, s.GUID)) - } -} diff --git a/client/transfer/engine/downloader.go b/client/transfer/engine/downloader.go index afe40c3a..1051462c 100644 --- a/client/transfer/engine/downloader.go +++ b/client/transfer/engine/downloader.go @@ -101,14 +101,6 @@ func (d *GenericDownloader) downloadSingle(ctx context.Context, guid string, dst if expectedSize > 0 && (startOffset+written) < expectedSize { return fmt.Errorf("short download: got %d, expected %d", startOffset+written, expectedSize) } - emitTransferCompletion(ctx, common.TransferCompletionEvent{ - Direction: "download", - GUID: guid, - RangeStart: startOffset, - RangeEnd: startOffset + written - 1, - Bytes: written, - Strategy: "single", - }) return nil } @@ -264,15 +256,6 @@ func (d *GenericDownloader) downloadParallel(ctx context.Context, guid string, d if written != partLength { return fmt.Errorf("short write: got %d, expected %d", written, partLength) } - emitTransferCompletion(gctx, common.TransferCompletionEvent{ - Direction: "download", - GUID: guid, - RangeStart: partStart, - RangeEnd: partEnd, - Bytes: written, - PartNumber: partNum(partStart, chunkSize), - Strategy: "multipart", - }) return nil }) }) @@ -290,18 +273,6 @@ func (d *GenericDownloader) downloadParallel(ctx context.Context, guid string, d return nil } -func emitTransferCompletion(ctx context.Context, ev common.TransferCompletionEvent) { - if ev.Bytes <= 0 { - return - } - if cb := common.GetTransferCompletion(ctx); cb != nil { - _ = cb(ev) - } -} -func partNum(start, chunkSize int64) int { - if chunkSize <= 0 { - return 0 - } - return int(start/chunkSize) + 1 -} + + diff --git a/client/transfer/engine/engine_test.go b/client/transfer/engine/engine_test.go index 71ec1b49..f0993e58 100644 --- a/client/transfer/engine/engine_test.go +++ b/client/transfer/engine/engine_test.go @@ -494,12 +494,36 @@ func TestGenericUploaderMultipartStreamsProgressAndCompletesAfterFinalize(t *tes uploader := &GenericUploader{Backend: backend} var finalized atomic.Bool + var closeGate sync.Once + releaseComplete := func() { + closeGate.Do(func() { + close(completeGate) + }) + } + + var eventsMu sync.Mutex var events []common.ProgressEvent + progressReady := make(chan struct{}, 1) + progressErr := make(chan error, 1) ctx := common.WithOid(context.Background(), "oid-slow-multipart") ctx = common.WithProgress(ctx, func(ev common.ProgressEvent) error { + eventsMu.Lock() events = append(events, ev) + eventCount := len(events) + eventsMu.Unlock() if ev.BytesSoFar == 21*common.MB && !finalized.Load() { - t.Fatalf("terminal multipart progress emitted before completion: %+v", ev) + err := fmt.Errorf("terminal multipart progress emitted before completion: %+v", ev) + select { + case progressErr <- err: + default: + } + return err + } + if eventCount >= 2 { + select { + case progressReady <- struct{}{}: + default: + } } return nil }) @@ -514,21 +538,46 @@ func TestGenericUploaderMultipartStreamsProgressAndCompletesAfterFinalize(t *tes }, true) }() - time.Sleep(20 * time.Millisecond) - if len(events) < 2 { - close(completeGate) - if err := <-done; err != nil { - t.Fatalf("Upload returned error: %v", err) + select { + case <-progressReady: + case err := <-progressErr: + releaseComplete() + if uploadErr := <-done; uploadErr != nil && uploadErr.Error() != err.Error() { + t.Fatalf("Upload returned error %v after progress error %v", uploadErr, err) + } + t.Fatal(err) + case <-time.After(2 * time.Second): + releaseComplete() + select { + case err := <-done: + if err != nil { + t.Fatalf("Upload returned error: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("multipart upload did not emit streaming progress before completion gate release") } - t.Fatalf("expected streaming multipart progress before completion, got %+v", events) + eventsMu.Lock() + gotEvents := append([]common.ProgressEvent(nil), events...) + eventsMu.Unlock() + t.Fatalf("expected streaming multipart progress before completion, got %+v", gotEvents) } finalized.Store(true) - close(completeGate) - if err := <-done; err != nil { - t.Fatalf("Upload returned error: %v", err) + releaseComplete() + select { + case err := <-done: + if err != nil { + t.Fatalf("Upload returned error: %v", err) + } + case err := <-progressErr: + t.Fatal(err) + case <-time.After(2 * time.Second): + t.Fatal("multipart upload did not complete after completion gate release") } + + eventsMu.Lock() last := events[len(events)-1] + eventsMu.Unlock() if last.BytesSoFar != 21*common.MB { t.Fatalf("expected final multipart BytesSoFar=%d, got %+v", 21*common.MB, last) } @@ -597,16 +646,11 @@ func TestGenericDownloaderDownloadSingleVariants(t *testing.T) { } var events []common.ProgressEvent - var completed []common.TransferCompletionEvent ctx := common.WithOid(context.Background(), "oid-download") ctx = common.WithProgress(ctx, func(ev common.ProgressEvent) error { events = append(events, ev) return nil }) - ctx = common.WithTransferCompletion(ctx, func(ev common.TransferCompletionEvent) error { - completed = append(completed, ev) - return nil - }) if err := d.downloadSingle(ctx, "guid-1", dst, 11); err != nil { t.Fatalf("downloadSingle returned error: %v", err) @@ -624,12 +668,7 @@ func TestGenericDownloaderDownloadSingleVariants(t *testing.T) { if len(events) != 1 || events[0].BytesSinceLast != 5 || events[0].BytesSoFar != 11 { t.Fatalf("unexpected progress events: %+v", events) } - if len(completed) != 1 { - t.Fatalf("expected one transfer completion event, got %+v", completed) - } - if completed[0].Direction != "download" || completed[0].GUID != "guid-1" || completed[0].RangeStart != 6 || completed[0].RangeEnd != 10 || completed[0].Bytes != 5 || completed[0].Strategy != "single" { - t.Fatalf("unexpected transfer completion event: %+v", completed[0]) - } + }) t.Run("range ignored restarts from zero", func(t *testing.T) { @@ -692,7 +731,6 @@ func TestGenericDownloaderDownloadAndParallel(t *testing.T) { var eventsMu sync.Mutex var events []common.ProgressEvent - var completions []common.TransferCompletionEvent ctx := common.WithOid(context.Background(), "parallel-oid") ctx = common.WithProgress(ctx, func(ev common.ProgressEvent) error { eventsMu.Lock() @@ -700,12 +738,6 @@ func TestGenericDownloaderDownloadAndParallel(t *testing.T) { eventsMu.Unlock() return nil }) - ctx = common.WithTransferCompletion(ctx, func(ev common.TransferCompletionEvent) error { - eventsMu.Lock() - completions = append(completions, ev) - eventsMu.Unlock() - return nil - }) if err := d.Download(ctx, "guid-p", dst, 3, 1*common.MB, 1*common.MB); err != nil { t.Fatalf("Download returned error: %v", err) @@ -723,23 +755,12 @@ func TestGenericDownloaderDownloadAndParallel(t *testing.T) { if len(events) < 3 { t.Fatalf("expected incremental progress events, got %+v", events) } - if len(completions) != 3 { - t.Fatalf("expected transfer completion per chunk, got %+v", completions) - } + lastEvent := events[len(events)-1] if lastEvent.BytesSoFar != int64(len(content)) { t.Fatalf("expected final progress to reach %d, got %+v", len(content), lastEvent) } - var completedBytes int64 - for _, ev := range completions { - if ev.Direction != "download" || ev.GUID != "guid-p" || ev.Strategy != "multipart" || ev.Bytes <= 0 || ev.RangeEnd < ev.RangeStart { - t.Fatalf("unexpected transfer completion event: %+v", ev) - } - completedBytes += ev.Bytes - } - if completedBytes != int64(len(content)) { - t.Fatalf("expected completed bytes %d, got %d events=%+v", len(content), completedBytes, completions) - } + backend2 := &fakeBackend{data: []byte("single"), meta: &transfer.ObjectMetadata{Size: 6, AcceptRanges: true}} d2 := &GenericDownloader{Source: backend2} @@ -768,19 +789,13 @@ func TestGenericDownloaderDownloadAndParallel(t *testing.T) { } d := &GenericDownloader{Source: backend} dst := filepath.Join(t.TempDir(), "failed-parallel.bin") - var completions []common.TransferCompletionEvent - ctx := common.WithTransferCompletion(context.Background(), func(ev common.TransferCompletionEvent) error { - completions = append(completions, ev) - return nil - }) + ctx := context.Background() err := d.Download(ctx, "guid-fail", dst, 2, 1*common.MB, 1*common.MB) if err == nil || !strings.Contains(err.Error(), "range download") { t.Fatalf("expected range download error, got %v", err) } - if len(completions) != 0 { - t.Fatalf("failed transfer should not emit completion events, got %+v", completions) - } + if _, statErr := os.Stat(dst); !os.IsNotExist(statErr) { t.Fatalf("expected failed parallel download to remove %q, stat err=%v", dst, statErr) } diff --git a/client/transfer/engine/progress_reader.go b/client/transfer/engine/progress_reader.go index 8ad64e84..71b70ad7 100644 --- a/client/transfer/engine/progress_reader.go +++ b/client/transfer/engine/progress_reader.go @@ -89,12 +89,7 @@ func (pr *progressReader) advance(delta int64) int64 { return pr.localBytes } -func (pr *progressReader) current() int64 { - if pr.globalBytes != nil { - return atomic.LoadInt64(pr.globalBytes) - } - return pr.localBytes -} + func (pr *progressReader) currentRaw() int64 { if pr.globalBytes != nil { diff --git a/client/transfer/noop_logger.go b/client/transfer/noop_logger.go index 98c72062..14dc1e21 100644 --- a/client/transfer/noop_logger.go +++ b/client/transfer/noop_logger.go @@ -6,7 +6,6 @@ import ( "log/slog" "github.com/calypr/syfon/client/common" - "github.com/calypr/syfon/client/logs" ) // NoOpLogger satisfies TransferLogger without emitting output. @@ -36,4 +35,3 @@ func (NoOpLogger) GetFailedLogMap() map[string]common.RetryObject { return map[string]common.RetryObject{} } func (NoOpLogger) DeleteFromFailedLog(string) {} -func (NoOpLogger) Scoreboard() *logs.Scoreboard { return nil } diff --git a/client/transfer/retry.go b/client/transfer/retry.go index 66e2b0a4..4613bd99 100644 --- a/client/transfer/retry.go +++ b/client/transfer/retry.go @@ -73,9 +73,6 @@ func RetryAction(ctx context.Context, logger TransferLogger, strategy RetryStrat lastErr = action() if lastErr == nil { - if sb := logger.Scoreboard(); sb != nil { - sb.IncrementSB(i) - } return nil } var nonRetryable nonRetryableError @@ -86,8 +83,5 @@ func RetryAction(ctx context.Context, logger TransferLogger, strategy RetryStrat logger.Error("Action failed", "retry", i, "error", lastErr) } - if sb := logger.Scoreboard(); sb != nil { - sb.IncrementSB(maxRetries + 1) - } return lastErr } diff --git a/client/transfer/retry_test.go b/client/transfer/retry_test.go index d9673c14..b60ce86a 100644 --- a/client/transfer/retry_test.go +++ b/client/transfer/retry_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/calypr/syfon/client/common" - "github.com/calypr/syfon/client/logs" ) type fixedStrategy time.Duration @@ -20,7 +19,6 @@ type captureLogger struct { NoOpLogger printfs []string errorsSeen []error - sb *logs.Scoreboard } func (l *captureLogger) Printf(format string, v ...any) { @@ -35,9 +33,7 @@ func (l *captureLogger) Error(msg string, args ...any) { } } -func (l *captureLogger) Scoreboard() *logs.Scoreboard { - return l.sb -} + func TestExponentialBackoffAndDefaultBackoff(t *testing.T) { t.Parallel() @@ -65,8 +61,8 @@ func TestExponentialBackoffAndDefaultBackoff(t *testing.T) { func TestRetryAction(t *testing.T) { t.Parallel() - t.Run("eventual success increments scoreboard", func(t *testing.T) { - logger := &captureLogger{sb: logs.NewSB(3, NoOpLogger{}.Slog())} + t.Run("eventual success", func(t *testing.T) { + logger := &captureLogger{} attempts := 0 err := RetryAction(context.Background(), logger, fixedStrategy(0), 3, func() error { attempts++ @@ -81,9 +77,6 @@ func TestRetryAction(t *testing.T) { if attempts != 3 { t.Fatalf("expected 3 attempts, got %d", attempts) } - if logger.sb.Counts[2] != 1 { - t.Fatalf("expected scoreboard success bucket for 2 retries, got %+v", logger.sb.Counts) - } if len(logger.printfs) != 2 || len(logger.errorsSeen) != 2 { t.Fatalf("expected retry logging, got printfs=%d errors=%d", len(logger.printfs), len(logger.errorsSeen)) } @@ -106,8 +99,8 @@ func TestRetryAction(t *testing.T) { } }) - t.Run("final failure increments terminal bucket", func(t *testing.T) { - logger := &captureLogger{sb: logs.NewSB(1, NoOpLogger{}.Slog())} + t.Run("final failure", func(t *testing.T) { + logger := &captureLogger{} wantErr := errors.New("still failing") err := RetryAction(context.Background(), logger, fixedStrategy(0), 1, func() error { return wantErr @@ -115,9 +108,6 @@ func TestRetryAction(t *testing.T) { if !errors.Is(err, wantErr) { t.Fatalf("expected %v, got %v", wantErr, err) } - if logger.sb.Counts[len(logger.sb.Counts)-1] != 1 { - t.Fatalf("expected terminal failure bucket increment, got %+v", logger.sb.Counts) - } }) } @@ -153,7 +143,4 @@ func TestNoOpLogger(t *testing.T) { if got := logger.GetFailedLogMap(); len(got) != 0 { t.Fatalf("expected empty failed log map, got %+v", got) } - if logger.Scoreboard() != nil { - t.Fatal("expected nil scoreboard") - } } diff --git a/client/transfer/types.go b/client/transfer/types.go index 56574bd8..321d6edf 100644 --- a/client/transfer/types.go +++ b/client/transfer/types.go @@ -10,7 +10,6 @@ import ( internalapi "github.com/calypr/syfon/apigen/client/internalapi" "github.com/calypr/syfon/client/common" "github.com/calypr/syfon/client/hash" - "github.com/calypr/syfon/client/logs" ) var ( @@ -38,7 +37,6 @@ type TransferLogger interface { GetSucceededLogMap() map[string]string GetFailedLogMap() map[string]common.RetryObject DeleteFromFailedLog(path string) - Scoreboard() *logs.Scoreboard } // Service provides high-level identity and logging access. diff --git a/client/transfer/upload/utils.go b/client/transfer/upload/utils.go index 5773cca9..3b9fa6b5 100644 --- a/client/transfer/upload/utils.go +++ b/client/transfer/upload/utils.go @@ -1,108 +1,11 @@ package upload import ( - "encoding/json" - "errors" "fmt" - "os" - "path/filepath" - "strings" "github.com/calypr/syfon/client/common" - "github.com/calypr/syfon/client/transfer" - "github.com/calypr/syfon/client/transfer/engine" ) -func SeparateSingleAndMultipartUploads(bk interface { - Logger() transfer.TransferLogger -}, objects []uploadRequest) ([]uploadRequest, []uploadRequest) { - fileSizeLimit := common.FileSizeLimit - logger := bk.Logger() - - var singlepartObjects []uploadRequest - var multipartObjects []uploadRequest - - for _, object := range objects { - fi, err := os.Stat(object.sourcePath) - if err != nil { - if os.IsNotExist(err) { - logger.Error("The file you specified does not exist locally", "path", object.sourcePath) - } else { - logger.Error("File stat error", "path", object.sourcePath, "error", err) - } - continue - } - if fi.IsDir() { - continue - } - if fi.Size() > common.MultipartFileSizeLimit { - logger.Warn("File exceeds max limit", "name", fi.Name(), "size", fi.Size()) - continue - } - if fi.Size() > int64(fileSizeLimit) { - multipartObjects = append(multipartObjects, object) - } else { - singlepartObjects = append(singlepartObjects, object) - } - } - return singlepartObjects, multipartObjects -} - -// ProcessFilename returns an FileInfo object which has the information about the path and name to be used for upload of a file -func ProcessFilename(logger transfer.TransferLogger, uploadPath string, filePath string, objectId string, includeSubDirName bool, includeMetadata bool) (string, string, common.FileMetadata, error) { - var err error - filePath, err = common.GetAbsolutePath(filePath) - if err != nil { - return "", "", common.FileMetadata{}, err - } - - filename := filepath.Base(filePath) // Default to base filename - - var metadata common.FileMetadata - if includeSubDirName { - absUploadPath, err := common.GetAbsolutePath(uploadPath) - if err != nil { - return "", "", common.FileMetadata{}, err - } - - // Ensure absUploadPath is a directory path for relative calculation - // Trim the optional wildcard if present - uploadDir := strings.TrimSuffix(absUploadPath, common.PathSeparator+"*") - fileInfo, err := os.Stat(uploadDir) - if err != nil { - return "", "", common.FileMetadata{}, err - } - if fileInfo.IsDir() { - // Calculate the path of the file relative to the upload directory - relPath, err := filepath.Rel(uploadDir, filePath) - if err != nil { - return "", "", common.FileMetadata{}, err - } - filename = relPath - } - } - - if includeMetadata { - // The metadata path is the file name plus '_metadata.json' - metadataFilePath := strings.TrimSuffix(filePath, filepath.Ext(filePath)) + "_metadata.json" - var metadataFileBytes []byte - if _, err := os.Stat(metadataFilePath); err == nil { - metadataFileBytes, err = os.ReadFile(metadataFilePath) - if err != nil { - return "", "", common.FileMetadata{}, errors.New("Error reading metadata file " + metadataFilePath + ": " + err.Error()) - } - err := json.Unmarshal(metadataFileBytes, &metadata) - if err != nil { - return "", "", common.FileMetadata{}, errors.New("Error parsing metadata file " + metadataFilePath + ": " + err.Error()) - } - } else { - // No metadata file was found for this file -- proceed, but warn the user. - logger.Printf("WARNING: File metadata is enabled, but could not find the metadata file %v for file %v. Execute `data-client upload --help` for more info on file metadata.\n", metadataFilePath, filePath) - } - } - return filePath, filename, metadata, nil -} - // FormatSize helps to parse a int64 size into string func FormatSize(size int64) string { var unitSize int64 @@ -129,8 +32,3 @@ func FormatSize(size int64) string { return fmt.Sprintf("%.1f"+unitMap[unitSize], float64(size)/float64(unitSize)) } - -// OptimalChunkSize returns a recommended chunk size for the given fileSize (in bytes). -func OptimalChunkSize(fileSize int64) int64 { - return engine.OptimalChunkSize(fileSize) -} diff --git a/client/transfer/upload/utils_test.go b/client/transfer/upload/utils_test.go deleted file mode 100644 index 9145e396..00000000 --- a/client/transfer/upload/utils_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package upload - -import ( - "testing" - - "github.com/calypr/syfon/client/common" -) - -func TestOptimalChunkSize(t *testing.T) { - tests := []struct { - name string - fileSize int64 - wantChunkSize int64 - wantParts int64 - }{ - { - name: "0 bytes", - fileSize: 0, - wantChunkSize: 1 * common.MB, - wantParts: 0, - }, - { - name: "1MB", - fileSize: 1 * common.MB, - wantChunkSize: 1 * common.MB, - wantParts: 1, - }, - { - name: "100MB", - fileSize: 100 * common.MB, - wantChunkSize: 100 * common.MB, - wantParts: 1, - }, - { - name: "100MB+1B", - fileSize: 100*common.MB + 1, - wantChunkSize: 10 * common.MB, - wantParts: 11, - }, - { - name: "500MB", - fileSize: 500 * common.MB, - wantChunkSize: 10 * common.MB, - wantParts: 50, - }, - { - name: "1GB", - fileSize: 1 * common.GB, - wantChunkSize: 10 * common.MB, - wantParts: 103, - }, - { - name: "1GB+1B", - fileSize: 1*common.GB + 1, - wantChunkSize: 25 * common.MB, - wantParts: 41, - }, - { - name: "5GB", - fileSize: 5 * common.GB, - wantChunkSize: 70 * common.MB, - wantParts: 74, - }, - { - name: "10GB", - fileSize: 10 * common.GB, - wantChunkSize: 128 * common.MB, - wantParts: 80, - }, - { - name: "10GB+1B", - fileSize: 10*common.GB + 1, - wantChunkSize: 256 * common.MB, - wantParts: 41, - }, - { - name: "50GB", - fileSize: 50 * common.GB, - wantChunkSize: 256 * common.MB, - wantParts: 200, - }, - { - name: "100GB", - fileSize: 100 * common.GB, - wantChunkSize: 256 * common.MB, - wantParts: 400, - }, - { - name: "100GB+1B", - fileSize: 100*common.GB + 1, - wantChunkSize: 512 * common.MB, - wantParts: 201, - }, - { - name: "500GB", - fileSize: 500 * common.GB, - wantChunkSize: 739 * common.MB, - wantParts: 693, - }, - { - name: "1TB", - fileSize: 1 * common.TB, - wantChunkSize: 1 * common.GB, - wantParts: 1024, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - chunkSize := OptimalChunkSize(tt.fileSize) - if chunkSize != tt.wantChunkSize { - t.Fatalf("chunk size = %d, want %d", chunkSize, tt.wantChunkSize) - } - - parts := int64(0) - if tt.fileSize > 0 && chunkSize > 0 { - parts = (tt.fileSize + chunkSize - 1) / chunkSize - } - if parts != tt.wantParts { - t.Fatalf("parts = %d, want %d", parts, tt.wantParts) - } - }) - } -} diff --git a/cmd/cli_basic_commands_test.go b/cmd/cli_basic_commands_test.go index bf97751d..f43cce82 100644 --- a/cmd/cli_basic_commands_test.go +++ b/cmd/cli_basic_commands_test.go @@ -813,8 +813,8 @@ func TestSyfonCopyProjectCommand_AcrossInstances(t *testing.T) { if err != nil { t.Fatalf("failed to get copied target record: %v", err) } - if targetRec.FileName == nil || *targetRec.FileName != fileName { - t.Fatalf("unexpected target filename: %+v", targetRec.FileName) + if targetRec.Name == nil || *targetRec.Name != fileName { + t.Fatalf("unexpected target name: %+v", targetRec.Name) } if targetRec.Hashes == nil || (*targetRec.Hashes)["sha256"] != checksum { t.Fatalf("unexpected target hashes: %+v", targetRec.Hashes) @@ -879,7 +879,7 @@ func TestSyfonCopyProjectRefsCommand(t *testing.T) { AccessMethods: sourceRec.AccessMethods, ControlledAccess: sourceRec.ControlledAccess, Description: &description, - FileName: sourceRec.FileName, + Name: sourceRec.Name, Hashes: sourceRec.Hashes, Organization: sourceRec.Organization, Project: sourceRec.Project, @@ -909,8 +909,8 @@ func TestSyfonCopyProjectRefsCommand(t *testing.T) { if err != nil { t.Fatalf("failed to get copied target record: %v", err) } - if targetRec.FileName == nil || *targetRec.FileName != fileName { - t.Fatalf("unexpected target filename: %+v", targetRec.FileName) + if targetRec.Name == nil || *targetRec.Name != fileName { + t.Fatalf("unexpected target name: %+v", targetRec.Name) } if targetRec.Description == nil || *targetRec.Description != description { t.Fatalf("unexpected target description: %+v", targetRec.Description) diff --git a/cmd/cliauth/auth.go b/cmd/cliauth/auth.go index 136ff002..23611d2e 100644 --- a/cmd/cliauth/auth.go +++ b/cmd/cliauth/auth.go @@ -30,9 +30,7 @@ func RegisterRootFlags(fs *pflag.FlagSet) { fs.StringVar(&password, "password", strings.TrimSpace(os.Getenv("SYFON_PASSWORD")), "Basic auth password for authenticated Syfon servers") } -func ResolvedProfile() string { - return strings.TrimSpace(profile) -} + func NewServerClient(cmd *cobra.Command) (syfonclient.SyfonClient, error) { serverURL, err := ResolveServerURL(cmd) diff --git a/cmd/copyproject/main.go b/cmd/copyproject/main.go index fb6d9cdb..878ad1e6 100644 --- a/cmd/copyproject/main.go +++ b/cmd/copyproject/main.go @@ -375,8 +375,8 @@ func hasPathPrefix(candidate, prefix []string) bool { func copyRecord(ctx context.Context, cmd *cobra.Command, sourceClient, targetClient services.SyfonClient, rec internalapi.InternalRecord, targetBucket, targetProjectPath string, dstResource string, current, total int, tempDir string) error { did := rec.Did fileName := "" - if rec.FileName != nil { - fileName = *rec.FileName + if rec.Name != nil { + fileName = *rec.Name } size := int64(0) if rec.Size != nil { @@ -479,6 +479,9 @@ func copyRecord(ctx context.Context, cmd *cobra.Command, sourceClient, targetCli if err := targetClient.Index().Upsert(ctx, did, targetObjectURL, fileName, size, checksum, authzMap); err != nil { return fmt.Errorf("failed to sync index record for DID %s: %w", did, err) } + if _, err := targetClient.DRS().UpdateObjectAccessMethods(ctx, did, []drsapi.AccessMethod{targetAccessMethod}); err != nil { + return fmt.Errorf("failed to replace access methods for DID %s: %w", did, err) + } return nil } diff --git a/cmd/download/main.go b/cmd/download/main.go index db2f1cf5..3faf60db 100644 --- a/cmd/download/main.go +++ b/cmd/download/main.go @@ -1,23 +1,15 @@ package download import ( - "context" "fmt" - "io" - "net/http" - "net/url" "os" "path/filepath" "strings" - syclient "github.com/calypr/syfon/client" - "github.com/calypr/syfon/client/request" - syfonclient "github.com/calypr/syfon/client/services" transferdownload "github.com/calypr/syfon/client/transfer/download" syupload "github.com/calypr/syfon/client/transfer/upload" "github.com/calypr/syfon/cmd/cliauth" "github.com/calypr/syfon/cmd/transferprogress" - "github.com/calypr/syfon/internal/common" "github.com/spf13/cobra" ) @@ -47,10 +39,8 @@ var Cmd = &cobra.Command{ return fmt.Errorf("resolve output filename from record: %w", err) } name := did - if rec.FileName != nil { - if pretty := common.DownloadFilename(*rec.FileName); pretty != "" { - name = pretty - } + if rec.Name != nil && strings.TrimSpace(*rec.Name) != "" { + name = strings.TrimSpace(*rec.Name) } if rec.Size != nil { expectedSize = *rec.Size @@ -86,55 +76,7 @@ var Cmd = &cobra.Command{ }, } -func downloadURLToPath(ctx context.Context, rawURL, outPath string, c syfonclient.SyfonClient) error { - parsed, err := url.Parse(strings.TrimSpace(rawURL)) - if err != nil { - return fmt.Errorf("parse download url: %w", err) - } - switch strings.ToLower(parsed.Scheme) { - case "": - srcPath := parsed.Path - if srcPath == "" { - srcPath = rawURL - } - data, err := os.ReadFile(srcPath) - if err != nil { - return fmt.Errorf("read file source: %w", err) - } - if err := os.WriteFile(outPath, data, 0o644); err != nil { - return fmt.Errorf("write output file: %w", err) - } - return nil - case "http", "https": - var resp *http.Response - concrete, ok := c.(*syclient.Client) - if !ok { - return fmt.Errorf("client implementation does not support raw requests") - } - err := concrete.Requestor().Do(ctx, http.MethodGet, rawURL, nil, &resp, request.WithSkipAuth(true)) - if err != nil { - return fmt.Errorf("download request failed: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - body, err := io.ReadAll(io.LimitReader(resp.Body, 4096)) - if err != nil { - return fmt.Errorf("read error response body: %w", err) - } - return fmt.Errorf("download failed status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))) - } - data, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("read download response: %w", err) - } - if err := os.WriteFile(outPath, data, 0o644); err != nil { - return fmt.Errorf("write output file: %w", err) - } - return nil - default: - return fmt.Errorf("unsupported download url scheme %q", parsed.Scheme) - } -} + func init() { Cmd.Flags().StringVar(&downloadDID, "did", "", "DRS object DID") diff --git a/cmd/download/main_test.go b/cmd/download/main_test.go index 0eb41c64..7fd79679 100644 --- a/cmd/download/main_test.go +++ b/cmd/download/main_test.go @@ -1,15 +1,9 @@ package download import ( - "context" - "net/http" - "net/http/httptest" - "os" - "path/filepath" "strings" "testing" - syclient "github.com/calypr/syfon/client" "github.com/spf13/cobra" ) @@ -22,83 +16,3 @@ func TestDownloadRunE_RequiresDID(t *testing.T) { t.Fatalf("expected missing did error, got: %v", err) } } - -func TestDownloadURLToPath_LocalPath(t *testing.T) { - srcDir := t.TempDir() - srcPath := filepath.Join(srcDir, "source.txt") - outPath := filepath.Join(srcDir, "out.txt") - payload := []byte("download-file-scheme") - if err := os.WriteFile(srcPath, payload, 0o644); err != nil { - t.Fatalf("write source: %v", err) - } - - err := downloadURLToPath(context.Background(), srcPath, outPath, nil) - if err != nil { - t.Fatalf("downloadURLToPath(local path) error: %v", err) - } - got, err := os.ReadFile(outPath) - if err != nil { - t.Fatalf("read output: %v", err) - } - if string(got) != string(payload) { - t.Fatalf("unexpected output content: got=%q want=%q", string(got), string(payload)) - } -} - -func TestDownloadURLToPath_HTTP_Success(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("http-payload")) - })) - defer ts.Close() - - client, err := syclient.New(ts.URL) - if err != nil { - t.Fatalf("create client: %v", err) - } - outPath := filepath.Join(t.TempDir(), "out.txt") - - err = downloadURLToPath(context.Background(), ts.URL+"/download", outPath, client) - if err != nil { - t.Fatalf("downloadURLToPath(http success) error: %v", err) - } - got, err := os.ReadFile(outPath) - if err != nil { - t.Fatalf("read output: %v", err) - } - if string(got) != "http-payload" { - t.Fatalf("unexpected output content: got=%q", string(got)) - } -} - -func TestDownloadURLToPath_HTTP_ErrorStatus(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "boom", http.StatusBadRequest) - })) - defer ts.Close() - - client, err := syclient.New(ts.URL) - if err != nil { - t.Fatalf("create client: %v", err) - } - outPath := filepath.Join(t.TempDir(), "out.txt") - - err = downloadURLToPath(context.Background(), ts.URL+"/download", outPath, client) - if err == nil || !strings.Contains(err.Error(), "download failed status=400") { - t.Fatalf("expected status error, got: %v", err) - } -} - -func TestDownloadURLToPath_UnsupportedScheme(t *testing.T) { - err := downloadURLToPath(context.Background(), "s3://bucket/key", filepath.Join(t.TempDir(), "out.txt"), nil) - if err == nil || !strings.Contains(err.Error(), "unsupported download url scheme") { - t.Fatalf("expected unsupported scheme error, got: %v", err) - } -} - -func TestDownloadURLToPath_HTTP_NonConcreteClient(t *testing.T) { - err := downloadURLToPath(context.Background(), "https://example.test/file", filepath.Join(t.TempDir(), "out.txt"), nil) - if err == nil || !strings.Contains(err.Error(), "client implementation does not support raw requests") { - t.Fatalf("expected non-concrete client error, got: %v", err) - } -} diff --git a/cmd/list/main.go b/cmd/list/main.go index 0a5f2e31..cfa6295c 100644 --- a/cmd/list/main.go +++ b/cmd/list/main.go @@ -40,11 +40,10 @@ var Cmd = &cobra.Command{ records, err := listRecords(cmd.Context(), c.Index(), syfonclient.ListRecordsOptions{ Limit: listLimit, Page: listPage, - Path: strings.TrimSpace(listPath), Start: strings.TrimSpace(listStart), Organization: strings.TrimSpace(listOrganization), ProjectID: strings.TrimSpace(listProject), - }, listRecursive) + }, listRecursive, strings.TrimSpace(listPath)) if err != nil { return err } @@ -61,8 +60,8 @@ var Cmd = &cobra.Command{ for _, rec := range records { did := strings.TrimSpace(rec.Did) name := "-" - if rec.FileName != nil { - name = strings.TrimSpace(*rec.FileName) + if rec.Name != nil { + name = strings.TrimSpace(*rec.Name) } size := int64(0) if rec.Size != nil { @@ -92,95 +91,24 @@ func init() { Cmd.Flags().StringVar(&listProject, "project", "", "Optional project filter") } -func listRecords(ctx context.Context, lister indexLister, opts syfonclient.ListRecordsOptions, recursive bool) ([]internalapi.InternalRecord, error) { - if !recursive { - resp, err := lister.List(ctx, opts) - if err != nil { - return nil, err - } - if resp.Records == nil { - return nil, nil - } - return *resp.Records, nil +func listRecords(ctx context.Context, lister indexLister, opts syfonclient.ListRecordsOptions, recursive bool, listPath string) ([]internalapi.InternalRecord, error) { + if strings.TrimSpace(listPath) != "" { + return nil, fmt.Errorf("path-based listing is no longer supported") } - if strings.TrimSpace(opts.Organization) == "" || strings.TrimSpace(opts.ProjectID) == "" { - return nil, fmt.Errorf("--recursive requires both --organization and --project") + if recursive { + return nil, fmt.Errorf("--recursive requires both --organization and --project") + } } - if strings.TrimSpace(opts.Start) != "" || opts.Page != 0 { - return nil, fmt.Errorf("--recursive does not support --start or --page") + if recursive { + return nil, fmt.Errorf("path-based recursive listing is no longer supported") } - - remaining := opts.Limit - if remaining <= 0 { - remaining = 100 + resp, err := lister.List(ctx, opts) + if err != nil { + return nil, err } - - queue := []string{strings.TrimSpace(opts.Path)} - seenDirs := map[string]struct{}{} - seenDIDs := map[string]struct{}{} - records := make([]internalapi.InternalRecord, 0, remaining) - - for len(queue) > 0 && remaining > 0 { - currentPath := strings.TrimSpace(queue[0]) - queue = queue[1:] - if _, ok := seenDirs[currentPath]; ok { - continue - } - seenDirs[currentPath] = struct{}{} - - start := "" - for remaining > 0 { - pageLimit := remaining - resp, err := lister.List(ctx, syfonclient.ListRecordsOptions{ - Limit: pageLimit, - Start: start, - Path: currentPath, - Organization: opts.Organization, - ProjectID: opts.ProjectID, - }) - if err != nil { - return nil, err - } - - pageCount := 0 - if resp.Records != nil { - for _, rec := range *resp.Records { - did := strings.TrimSpace(rec.Did) - if did == "" { - continue - } - if _, ok := seenDIDs[did]; ok { - continue - } - seenDIDs[did] = struct{}{} - records = append(records, rec) - remaining-- - pageCount++ - start = did - if remaining == 0 { - break - } - } - } - - if resp.Directories != nil { - for _, dir := range *resp.Directories { - nextPath := strings.TrimSpace(dir.Path) - if nextPath == "" { - continue - } - if _, ok := seenDirs[nextPath]; !ok { - queue = append(queue, nextPath) - } - } - } - - if pageCount == 0 || pageCount < pageLimit { - break - } - } + if resp.Records == nil { + return nil, nil } - - return records, nil + return *resp.Records, nil } diff --git a/cmd/list/main_test.go b/cmd/list/main_test.go index 0eebda1f..c2e82172 100644 --- a/cmd/list/main_test.go +++ b/cmd/list/main_test.go @@ -18,40 +18,21 @@ type fakeIndexLister struct { func (f *fakeIndexLister) List(_ context.Context, opts syfonclient.ListRecordsOptions) (internalapi.ListRecordsResponse, error) { f.calls = append(f.calls, opts) - key := fmt.Sprintf("%s|%s|%s|%d|%s", opts.Organization, opts.ProjectID, opts.Path, opts.Limit, opts.Start) + key := fmt.Sprintf("%s|%s|%d|%s", opts.Organization, opts.ProjectID, opts.Limit, opts.Start) if resp, ok := f.responses[key]; ok { return resp, nil } return internalapi.ListRecordsResponse{}, fmt.Errorf("unexpected list call: %s", key) } -func TestListRecordsRecursiveWalksDirectories(t *testing.T) { +func TestListRecordsNonRecursive(t *testing.T) { rootRecords := []internalapi.InternalRecord{ - {Did: "did-root", FileName: ptr("README.md")}, + {Did: "did-root", Name: ptr("README.md")}, + {Did: "did-data", Name: ptr("file1.txt")}, } - rootDirs := []internalapi.IndexDirectory{ - {Name: "data", Path: "data"}, - {Name: "nested", Path: "nested"}, - } - dataRecords := []internalapi.InternalRecord{ - {Did: "did-data", FileName: ptr("data/file1.txt")}, - } - dataDirs := []internalapi.IndexDirectory{ - {Name: "more", Path: "data/more"}, - } - moreRecords := []internalapi.InternalRecord{ - {Did: "did-more", FileName: ptr("data/more/file2.txt")}, - } - nestedRecords := []internalapi.InternalRecord{ - {Did: "did-nested", FileName: ptr("nested/file3.txt")}, - } - lister := &fakeIndexLister{ responses: map[string]internalapi.ListRecordsResponse{ - "Ellrott_Lab|hla2vec||10|": {Records: &rootRecords, Directories: &rootDirs}, - "Ellrott_Lab|hla2vec|data|9|": {Records: &dataRecords, Directories: &dataDirs}, - "Ellrott_Lab|hla2vec|nested|8|": {Records: &nestedRecords}, - "Ellrott_Lab|hla2vec|data/more|7|": {Records: &moreRecords}, + "Ellrott_Lab|hla2vec|10|": {Records: &rootRecords}, }, } @@ -59,7 +40,7 @@ func TestListRecordsRecursiveWalksDirectories(t *testing.T) { Organization: "Ellrott_Lab", ProjectID: "hla2vec", Limit: 10, - }, true) + }, false, "") if err != nil { t.Fatalf("listRecords returned error: %v", err) } @@ -68,28 +49,27 @@ func TestListRecordsRecursiveWalksDirectories(t *testing.T) { for _, rec := range records { got = append(got, strings.TrimSpace(rec.Did)) } - want := []string{"did-root", "did-data", "did-nested", "did-more"} + want := []string{"did-root", "did-data"} if !slices.Equal(got, want) { - t.Fatalf("unexpected recursive record set: got %v want %v", got, want) + t.Fatalf("unexpected record set: got %v want %v", got, want) } } -func TestListRecordsRecursiveRequiresScopedProject(t *testing.T) { - _, err := listRecords(context.Background(), &fakeIndexLister{}, syfonclient.ListRecordsOptions{Limit: 10}, true) - if err == nil || !strings.Contains(err.Error(), "--recursive requires both --organization and --project") { - t.Fatalf("expected recursive scope error, got %v", err) - } -} - -func TestListRecordsRecursiveRejectsStartAndPage(t *testing.T) { +func TestListRecordsRejectsRecursive(t *testing.T) { _, err := listRecords(context.Background(), &fakeIndexLister{}, syfonclient.ListRecordsOptions{ Organization: "Ellrott_Lab", ProjectID: "hla2vec", - Start: "did-1", Limit: 10, - }, true) - if err == nil || !strings.Contains(err.Error(), "--recursive does not support --start or --page") { - t.Fatalf("expected recursive pagination error, got %v", err) + }, true, "") + if err == nil || !strings.Contains(err.Error(), "path-based recursive listing is no longer supported") { + t.Fatalf("expected recursive unsupported error, got %v", err) + } +} + +func TestListRecordsRejectsPathFilter(t *testing.T) { + _, err := listRecords(context.Background(), &fakeIndexLister{}, syfonclient.ListRecordsOptions{Limit: 10}, false, "nested") + if err == nil || !strings.Contains(err.Error(), "path-based listing is no longer supported") { + t.Fatalf("expected path unsupported error, got %v", err) } } diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go deleted file mode 100644 index 69ed6856..00000000 --- a/cmd/migrate/main.go +++ /dev/null @@ -1,302 +0,0 @@ -package migratecmd - -import ( - "context" - "errors" - "fmt" - "net/http" - "strings" - "time" - - conf "github.com/calypr/syfon/client/config" - "github.com/calypr/syfon/cmd/cliauth" - "github.com/calypr/syfon/migrate" - "github.com/spf13/cobra" -) - -var Cmd = &cobra.Command{ - Use: "migrate", - Short: "Export and import Gen3 Indexd records for Syfon migration", - Long: "Export records from a Gen3-mounted Indexd API into a local SQLite dump, then import that dump into Syfon using the controlled_access-aware /index/bulk loader.", -} - -var ( - dumpPath string - sourceProfile string - targetProfile string - sourceToken string - targetToken string - targetBasicUser string - targetBasicPassword string - batchSize int - limit int - sweeps int - dryRun bool - skipAuthPreflight bool - defaultAuthz []string -) - -var exportCmd = &cobra.Command{ - Use: "export", - Short: "Export records from Gen3 Indexd into a SQLite dump", - RunE: func(cmd *cobra.Command, args []string) error { - sourceURL, err := indexdURLFromServer(cmd) - if err != nil { - return err - } - if strings.TrimSpace(sourceProfile) == "" { - sourceProfile = cliauth.ResolvedProfile() - } - dumpPath = normalizedDumpPath(dumpPath) - - sourceAuth, err := authFromInputs(sourceProfile, sourceToken) - if err != nil { - return fmt.Errorf("source auth: %w", err) - } - httpClient := migrationHTTPClient() - source, err := migrate.NewHTTPClient(sourceURL, sourceAuth, httpClient) - if err != nil { - return err - } - var dump *migrate.SQLiteDump - if !dryRun { - dump, err = migrate.OpenSQLiteDump(dumpPath) - if err != nil { - return err - } - defer dump.Close() - } - - if dryRun { - cmd.Println("dry-run: no records will be written to the dump") - } - displayDump := dumpPath - if dryRun { - displayDump = "" - } - cmd.Printf("migration export starting: indexd=%s dump=%s batch=%d limit=%d sweeps=%d\n", sourceURL, displayDump, batchSize, limit, sweeps) - stats, err := migrate.Run(context.Background(), source, dump, migrate.Config{ - BatchSize: batchSize, - Limit: limit, - DryRun: dryRun, - DefaultAuthz: defaultAuthz, - Sweeps: sweeps, - }) - if err != nil { - return err - } - count := 0 - if dump != nil { - var countErr error - count, countErr = dump.Count(context.Background()) - if countErr != nil { - return countErr - } - } - if dryRun { - cmd.Printf("migration export dry-run complete: fetched=%d transformed=%d valid=%d skipped=%d errors=%d unique_ids=%d dump_records=0\n", - stats.Fetched, - stats.Transformed, - stats.Loaded, - stats.Skipped, - stats.Errors, - stats.CountOfUniqueIDs, - ) - return nil - } - cmd.Printf("migration export complete: fetched=%d transformed=%d dumped=%d skipped=%d errors=%d unique_ids=%d dump_records=%d\n", - stats.Fetched, - stats.Transformed, - stats.Loaded, - stats.Skipped, - stats.Errors, - stats.CountOfUniqueIDs, - count, - ) - return nil - }, -} - -var importCmd = &cobra.Command{ - Use: "import", - Short: "Import a migration SQLite dump into Syfon", - RunE: func(cmd *cobra.Command, args []string) error { - if strings.TrimSpace(dumpPath) == "" { - return fmt.Errorf("--dump is required") - } - targetURL, err := targetServerURL(cmd) - if err != nil { - return err - } - if strings.TrimSpace(targetProfile) == "" { - targetProfile = sourceProfile - } - if strings.TrimSpace(targetProfile) == "" { - targetProfile = cliauth.ResolvedProfile() - } - targetAuth, err := targetAuthFromInputs(targetProfile, targetToken, targetBasicUser, targetBasicPassword) - if err != nil { - return fmt.Errorf("target auth: %w", err) - } - target, err := migrate.NewHTTPClient(targetURL, targetAuth, migrationHTTPClient()) - if err != nil { - return err - } - dump, err := migrate.OpenExistingSQLiteDump(dumpPath) - if err != nil { - return err - } - defer dump.Close() - - cmd.Printf("migration import starting: dump=%s syfon=%s batch=%d\n", dumpPath, targetURL, batchSize) - if !skipAuthPreflight && strings.TrimSpace(targetAuth.BearerToken) != "" { - cmd.Println("migration import preflight: checking target create privileges") - report, err := migrate.PreflightImport(context.Background(), dump, target, batchSize) - if err != nil { - var preflightErr *migrate.ImportPreflightError - if errors.As(err, &preflightErr) { - printImportPreflightFailure(cmd, preflightErr.Report) - return fmt.Errorf("target authorization preflight failed: missing create access for %d/%d records across %d scopes; copy/paste scope list printed above", - preflightErr.Report.MissingRecords, - preflightErr.Report.Records, - len(preflightErr.Report.MissingResources), - ) - } - return fmt.Errorf("target authorization preflight failed: %w", err) - } - cmd.Printf("migration import preflight complete: records=%d required_scopes=%d missing_scopes=0\n", - report.Records, - len(report.RequiredResources), - ) - } - stats, err := migrate.Import(context.Background(), dump, target, batchSize) - if err != nil { - return err - } - cmd.Printf("migration import complete: read=%d loaded=%d skipped=%d errors=%d\n", - stats.Fetched, - stats.Loaded, - stats.Skipped, - stats.Errors, - ) - return nil - }, -} - -func printImportPreflightFailure(cmd *cobra.Command, report migrate.ImportPreflightReport) { - scopes := migrate.FormatPreflightScopes(report.MissingResources) - cmd.Printf("migration import preflight failed: missing create access for %d/%d records across %d scopes\n", - report.MissingRecords, - report.Records, - len(scopes), - ) - if report.FirstDeniedRecord != "" { - cmd.Printf("first denied record: %s\n", report.FirstDeniedRecord) - } - cmd.Println() - cmd.Println("Copy/paste scope list:") - cmd.Println(strings.Join(scopes, ", ")) - cmd.Println() -} - -func init() { - Cmd.AddCommand(exportCmd) - Cmd.AddCommand(importCmd) - - exportCmd.Flags().StringVar(&dumpPath, "dump", "", "SQLite dump file to write (default ./indexd-records.sqlite)") - exportCmd.Flags().StringVar(&sourceProfile, "source-profile", "", "Gen3 profile for source reads from ~/.gen3/gen3_client_config.ini") - exportCmd.Flags().StringVar(&sourceToken, "source-token", "", "Bearer token for source reads; overrides --source-profile") - exportCmd.Flags().IntVar(&batchSize, "batch-size", 500, "Records to fetch per batch; capped at Indexd's 1024 max") - exportCmd.Flags().IntVar(&limit, "limit", 0, "Maximum number of unique records to export; 0 means all records") - exportCmd.Flags().IntVar(&sweeps, "sweeps", 1, "Full start-cursor sweeps to run; increase only for non-quiescent or inconsistent Indexd sources") - exportCmd.Flags().BoolVar(&dryRun, "dry-run", false, "Fetch, transform, and validate only; do not write the dump") - exportCmd.Flags().StringArrayVar(&defaultAuthz, "default-controlled-access", nil, "Default controlled_access resource for source records with no authz; repeatable") - exportCmd.Flags().StringArrayVar(&defaultAuthz, "default-authz", nil, "Deprecated alias for --default-controlled-access") - - importCmd.Flags().StringVar(&dumpPath, "dump", "", "SQLite dump file to read") - importCmd.Flags().StringVar(&targetProfile, "target-profile", "", "Gen3 profile for target writes") - importCmd.Flags().StringVar(&targetToken, "target-token", "", "Bearer token for target writes; overrides --target-profile") - importCmd.Flags().StringVar(&targetBasicUser, "target-basic-user", "", "Basic auth username for local target Syfon writes") - importCmd.Flags().StringVar(&targetBasicPassword, "target-basic-password", "", "Basic auth password for local target Syfon writes") - importCmd.Flags().IntVar(&batchSize, "batch-size", 500, "Records to load per controlled_access-aware /index/bulk request") - importCmd.Flags().BoolVar(&skipAuthPreflight, "skip-auth-preflight", false, "Skip target create-privilege preflight before importing") -} - -func indexdURLFromServer(cmd *cobra.Command) (string, error) { - server, err := targetServerURL(cmd) - if err != nil { - return "", err - } - return strings.TrimRight(server, "/") + "/index/index", nil -} - -func normalizedDumpPath(path string) string { - path = strings.TrimSpace(path) - if path == "" { - return "indexd-records.sqlite" - } - return path -} - -func migrationHTTPClient() *http.Client { - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSHandshakeTimeout = 60 * time.Second - transport.ResponseHeaderTimeout = 90 * time.Second - return &http.Client{ - Timeout: 3 * time.Minute, - Transport: transport, - } -} - -func targetServerURL(cmd *cobra.Command) (string, error) { - return cliauth.ResolveServerURL(cmd) -} - -func authFromInputs(profile, token string) (migrate.AuthConfig, error) { - token = strings.TrimSpace(token) - if token != "" { - return migrate.AuthConfig{BearerToken: token}, nil - } - - profile = strings.TrimSpace(profile) - if profile == "" { - return migrate.AuthConfig{}, nil - } - - credential, err := cliauth.LoadProfileCredential(context.Background(), profile) - if err != nil { - return migrate.AuthConfig{}, err - } - return authFromCredential(profile, credential) -} - -func authFromCredential(profile string, credential *conf.Credential) (migrate.AuthConfig, error) { - if credential == nil { - return migrate.AuthConfig{}, fmt.Errorf("profile %q resolved to nil credential", profile) - } - if strings.TrimSpace(credential.AccessToken) != "" { - return migrate.AuthConfig{BearerToken: credential.AccessToken}, nil - } - if strings.TrimSpace(credential.KeyID) != "" || strings.TrimSpace(credential.APIKey) != "" { - return migrate.AuthConfig{Basic: &migrate.BasicAuth{Username: credential.KeyID, Password: credential.APIKey}}, nil - } - return migrate.AuthConfig{}, fmt.Errorf("profile %q has no access_token or key_id/api_key", profile) -} - -func targetAuthFromInputs(profile, token, basicUser, basicPassword string) (migrate.AuthConfig, error) { - basicUser = strings.TrimSpace(basicUser) - basicPassword = strings.TrimSpace(basicPassword) - if basicUser != "" || basicPassword != "" { - if basicUser == "" || basicPassword == "" { - return migrate.AuthConfig{}, fmt.Errorf("--target-basic-user and --target-basic-password must be set together") - } - if strings.TrimSpace(token) != "" { - return migrate.AuthConfig{}, fmt.Errorf("--target-token cannot be combined with --target-basic-user/--target-basic-password") - } - if strings.TrimSpace(profile) != "" { - return migrate.AuthConfig{}, fmt.Errorf("--target-profile cannot be combined with --target-basic-user/--target-basic-password") - } - return migrate.AuthConfig{Basic: &migrate.BasicAuth{Username: basicUser, Password: basicPassword}}, nil - } - return authFromInputs(profile, token) -} diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go deleted file mode 100644 index 8d9693d9..00000000 --- a/cmd/migrate/main_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package migratecmd - -import ( - "testing" - - conf "github.com/calypr/syfon/client/config" - "github.com/spf13/cobra" -) - -func TestNormalizedDumpPathDefaultsExport(t *testing.T) { - if got := normalizedDumpPath(""); got != "indexd-records.sqlite" { - t.Fatalf("expected default dump path, got %q", got) - } - if got := normalizedDumpPath(" custom.sqlite "); got != "custom.sqlite" { - t.Fatalf("expected trimmed custom dump path, got %q", got) - } -} - -func TestIndexdURLFromServerUsesRootServerFlag(t *testing.T) { - root := &cobra.Command{Use: "syfon"} - root.PersistentFlags().String("server", "https://calypr-dev.ohsu.edu/", "") - cmd := &cobra.Command{Use: "export"} - root.AddCommand(cmd) - - got, err := indexdURLFromServer(cmd) - if err != nil { - t.Fatalf("indexdURLFromServer returned error: %v", err) - } - if got != "https://calypr-dev.ohsu.edu/index/index" { - t.Fatalf("unexpected indexd url: %q", got) - } -} - -func TestTargetAuthFromInputsSupportsBasicAuth(t *testing.T) { - auth, err := targetAuthFromInputs("", "", "drs-user", "drs-pass") - if err != nil { - t.Fatalf("targetAuthFromInputs returned error: %v", err) - } - if auth.Basic == nil || auth.Basic.Username != "drs-user" || auth.Basic.Password != "drs-pass" { - t.Fatalf("unexpected basic auth config: %+v", auth.Basic) - } -} - -func TestTargetAuthFromInputsRequiresBasicPair(t *testing.T) { - if _, err := targetAuthFromInputs("", "", "drs-user", ""); err == nil { - t.Fatal("expected missing password error") - } - if _, err := targetAuthFromInputs("", "", "", "drs-pass"); err == nil { - t.Fatal("expected missing username error") - } -} - -func TestTargetAuthFromInputsRejectsMixedAuth(t *testing.T) { - if _, err := targetAuthFromInputs("", "token", "drs-user", "drs-pass"); err == nil { - t.Fatal("expected token/basic conflict error") - } - if _, err := targetAuthFromInputs("profile", "", "drs-user", "drs-pass"); err == nil { - t.Fatal("expected profile/basic conflict error") - } -} - -func TestAuthFromCredentialPrefersBearerToken(t *testing.T) { - t.Parallel() - - auth, err := authFromCredential("calypr", &conf.Credential{ - Profile: "calypr", - AccessToken: "new-token", - APIKey: "refresh-jwt", - }) - if err != nil { - t.Fatalf("authFromCredential returned error: %v", err) - } - if auth.BearerToken != "new-token" { - t.Fatalf("expected bearer token, got %+v", auth) - } - if auth.Basic != nil { - t.Fatalf("expected bearer auth only, got basic %+v", auth.Basic) - } -} diff --git a/cmd/projectcopy/helpers.go b/cmd/projectcopy/helpers.go index a9932adb..aa01d4c4 100644 --- a/cmd/projectcopy/helpers.go +++ b/cmd/projectcopy/helpers.go @@ -20,9 +20,7 @@ type Scope struct { Project string } -func (s Scope) String() string { - return s.Organization + "/" + s.Project -} + func ParseScopeArg(raw, label string) (Scope, error) { parts := strings.Split(strings.TrimSpace(raw), "/") @@ -266,7 +264,7 @@ func internalRecordFromResponse(rec internalapi.InternalRecordResponse) internal ControlledAccess: rec.ControlledAccess, CreatedTime: rec.CreatedTime, Description: rec.Description, - FileName: rec.FileName, + Name: rec.Name, Hashes: rec.Hashes, Organization: rec.Organization, Project: rec.Project, @@ -278,63 +276,32 @@ func internalRecordFromResponse(rec internalapi.InternalRecordResponse) internal func listAllProjectRecords(ctx context.Context, index *services.IndexService, org, project string) ([]internalapi.InternalRecord, error) { var records []internalapi.InternalRecord - queue := []string{""} - seenDirs := map[string]struct{}{} - seenDIDs := map[string]struct{}{} - - for len(queue) > 0 { - currentPath := queue[0] - queue = queue[1:] - if _, ok := seenDirs[currentPath]; ok { - continue + start := "" + for { + resp, err := index.List(ctx, services.ListRecordsOptions{ + Limit: 1000, + Start: start, + Organization: org, + ProjectID: project, + }) + if err != nil { + return nil, err } - seenDirs[currentPath] = struct{}{} - - start := "" - for { - resp, err := index.List(ctx, services.ListRecordsOptions{ - Limit: 1000, - Start: start, - Path: currentPath, - Organization: org, - ProjectID: project, - }) - if err != nil { - return nil, err - } - - pageCount := 0 - if resp.Records != nil { - for _, rec := range *resp.Records { - did := strings.TrimSpace(rec.Did) - if did == "" { - continue - } - if _, ok := seenDIDs[did]; ok { - continue - } - seenDIDs[did] = struct{}{} - records = append(records, rec) - pageCount++ - start = did - } - } - - if resp.Directories != nil { - for _, dir := range *resp.Directories { - nextPath := strings.TrimSpace(dir.Path) - if nextPath == "" { - continue - } - if _, ok := seenDirs[nextPath]; !ok { - queue = append(queue, nextPath) - } - } - } - - if pageCount == 0 || pageCount < 1000 { - break + if resp.Records == nil || len(*resp.Records) == 0 { + break + } + pageCount := 0 + for _, rec := range *resp.Records { + did := strings.TrimSpace(rec.Did) + if did == "" { + continue } + records = append(records, rec) + start = did + pageCount++ + } + if pageCount < 1000 { + break } } return records, nil diff --git a/cmd/projectcopy/helpers_test.go b/cmd/projectcopy/helpers_test.go index c67d9cc3..517ff63f 100644 --- a/cmd/projectcopy/helpers_test.go +++ b/cmd/projectcopy/helpers_test.go @@ -187,20 +187,13 @@ func writeProfileConfig(t *testing.T, home, content string) { } func testIndexService() *services.IndexService { - listByPath := map[string]internalapi.ListRecordsResponse{ + listPages := map[string]internalapi.ListRecordsResponse{ "": { Records: &[]internalapi.InternalRecord{ { Did: "did-1", ControlledAccess: &[]string{"/organization/syfon/project/e2e"}, }, - }, - Directories: &[]internalapi.IndexDirectory{ - {Name: "nested", Path: "nested"}, - }, - }, - "nested": { - Records: &[]internalapi.InternalRecord{ { Did: "did-2", ControlledAccess: &[]string{"/organization/syfon/project/e2e"}, @@ -233,8 +226,8 @@ func testIndexService() *services.IndexService { }, }, stubRequester{ - listResponse: func(path string) (internalapi.ListRecordsResponse, error) { - if resp, ok := listByPath[path]; ok { + listResponse: func(start string) (internalapi.ListRecordsResponse, error) { + if resp, ok := listPages[start]; ok { return resp, nil } return internalapi.ListRecordsResponse{}, nil @@ -256,7 +249,7 @@ func (s stubInternalClient) InternalGetWithResponse(ctx context.Context, id stri } type stubRequester struct { - listResponse func(path string) (internalapi.ListRecordsResponse, error) + listResponse func(start string) (internalapi.ListRecordsResponse, error) } func (s stubRequester) Do(ctx context.Context, method, path string, body, out any, opts ...request.RequestOption) error { @@ -267,15 +260,15 @@ func (s stubRequester) Do(ctx context.Context, method, path string, body, out an for _, opt := range opts { opt(builder) } - listPath := "" + start := "" if strings.Contains(builder.Url, "?") { values, err := url.ParseQuery(strings.TrimPrefix(strings.SplitN(builder.Url, "?", 2)[1], "?")) if err != nil { return err } - listPath = values.Get("path") + start = values.Get("start") } - resp, err := s.listResponse(listPath) + resp, err := s.listResponse(start) if err != nil { return err } diff --git a/cmd/root.go b/cmd/root.go index bdaae05b..3551e21d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,7 +13,6 @@ import ( "github.com/calypr/syfon/cmd/download" listcmd "github.com/calypr/syfon/cmd/list" metricscmd "github.com/calypr/syfon/cmd/metrics" - migratecmd "github.com/calypr/syfon/cmd/migrate" "github.com/calypr/syfon/cmd/ping" "github.com/calypr/syfon/cmd/rm" "github.com/calypr/syfon/cmd/server" @@ -60,7 +59,6 @@ func init() { RootCmd.AddCommand(listcmd.Cmd) RootCmd.AddCommand(rm.Cmd) RootCmd.AddCommand(metricscmd.Cmd) - RootCmd.AddCommand(migratecmd.Cmd) RootCmd.AddCommand(copyproject.Cmd) RootCmd.AddCommand(copyprojectrefs.Cmd) RootCmd.AddCommand(deleteproject.Cmd) diff --git a/cmd/server/endpoints_mock_test.go b/cmd/server/endpoints_mock_test.go index fa06b963..3cb6072e 100644 --- a/cmd/server/endpoints_mock_test.go +++ b/cmd/server/endpoints_mock_test.go @@ -275,9 +275,9 @@ func requestBodyFor(method, template string) ([]byte, string) { } return []byte(`{}`), "application/json" case "/data/upload/bulk": - return []byte(`{"requests":[{"file_id":"sha-1","bucket":"test-bucket-1","file_name":"sha-1"}]}`), "application/json" + return []byte(`{"requests":[{"file_id":"sha-1","bucket":"test-bucket-1","name":"sha-1"}]}`), "application/json" case "/data/multipart/init": - return []byte(`{"guid":"sha-1","file_name":"sha-1","bucket":"test-bucket-1"}`), "application/json" + return []byte(`{"guid":"sha-1","name":"sha-1","bucket":"test-bucket-1"}`), "application/json" case "/data/multipart/upload": return []byte(`{"key":"sha-1","bucket":"test-bucket-1","uploadId":"mock-upload-id","partNumber":1}`), "application/json" case "/data/multipart/complete": diff --git a/cmd/server/server_integration_test.go b/cmd/server/server_integration_test.go index cc0fefbd..254fd39f 100644 --- a/cmd/server/server_integration_test.go +++ b/cmd/server/server_integration_test.go @@ -199,7 +199,7 @@ s3_credentials: // 4. Internal multipart init still works after credential preload. internalMultipartReq := map[string]interface{}{ "guid": guid, - "file_name": "test-multipart", + "name": "test-multipart", "organization": organization, "project": project, } diff --git a/cmd/validate/openapi_validator.go b/cmd/validate/openapi_validator.go index 8e6a9fd2..0db5a596 100644 --- a/cmd/validate/openapi_validator.go +++ b/cmd/validate/openapi_validator.go @@ -12,9 +12,7 @@ import ( "github.com/gofiber/fiber/v3/middleware/adaptor" ) -func newDefaultSpecValidator(specPath string) (fiber.Handler, error) { - return newSpecValidator(specPath, true) -} + func newSpecValidator(specPath string, lenientRoutes bool) (fiber.Handler, error) { loader := openapi3.NewLoader() diff --git a/cmd/validate/validate_test.go b/cmd/validate/validate_test.go index 360a99d4..85c4f7ab 100644 --- a/cmd/validate/validate_test.go +++ b/cmd/validate/validate_test.go @@ -118,8 +118,8 @@ paths: } } -func TestDefaultSpecValidatorAndBadSpec(t *testing.T) { - _, err := newDefaultSpecValidator("/definitely/missing/openapi.yaml") +func TestSpecValidatorBadSpec(t *testing.T) { + _, err := newSpecValidator("/definitely/missing/openapi.yaml", false) if err == nil { t.Fatal("expected missing spec error") } diff --git a/common/drs.go b/common/drs.go deleted file mode 100644 index 0db9944b..00000000 --- a/common/drs.go +++ /dev/null @@ -1,125 +0,0 @@ -package common - -import ( - "fmt" - - drsapi "github.com/calypr/syfon/apigen/client/drs" - "github.com/google/uuid" -) - -// drsUUIDNamespace is a fixed namespace UUID for deterministic DRS ID generation. -var drsUUIDNamespace = uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - -// DrsUUID generates a deterministic UUID for a DRS object from org, project, and hash. -func DrsUUID(org, project, hash string) string { - hash = NormalizeOid(hash) - resource, err := ResourcePath(org, project) - if err != nil || hash == "" || project == "" || resource == "" { - return "" - } - name := "sha256:" + hash + "|" + resource - return uuid.NewSHA1(drsUUIDNamespace, []byte(name)).String() -} - -// ObjectBuilder constructs DRS objects for a given bucket and project scope. -type ObjectBuilder struct { - Bucket string - Project string - Organization string - StoragePrefix string - PathStyle string -} - -// NewObjectBuilder creates an ObjectBuilder for the given bucket and project. -func NewObjectBuilder(bucket, project string) ObjectBuilder { - return ObjectBuilder{ - Bucket: bucket, - Project: project, - } -} - -// Build constructs a DRS object with the given file metadata. -func (b ObjectBuilder) Build(fileName string, checksum string, size int64, drsId string) (*drsapi.DrsObject, error) { - prefix := b.StoragePrefix - return BuildDrsObjWithPrefix(fileName, checksum, size, drsId, b.Bucket, b.Organization, b.Project, prefix) -} - -// BuildDrsObjWithPrefix builds a DRS object with an S3 access URL derived from -// the provided bucket, org, project, and storage prefix. -func BuildDrsObjWithPrefix(fileName string, checksum string, size int64, drsId string, bucket string, org string, project string, prefix string) (*drsapi.DrsObject, error) { - if checksum == "" { - return nil, fmt.Errorf("checksum is required") - } - - obj := &drsapi.DrsObject{ - Id: drsId, - SelfUri: "drs://" + drsId, - Size: size, - Name: &fileName, - Checksums: []drsapi.Checksum{ - {Type: "sha256", Checksum: checksum}, - }, - } - - if bucket == "" { - return obj, nil - } - - var accessURL string - if prefix != "" { - accessURL = fmt.Sprintf("s3://%s/%s/%s", bucket, prefix, checksum) - } else { - accessURL = fmt.Sprintf("s3://%s/%s", bucket, checksum) - } - - am := drsapi.AccessMethod{ - Type: drsapi.AccessMethodTypeS3, - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: accessURL}, - } - - if authzMap := AuthzMapFromScope(org, project); authzMap != nil { - controlled := AuthzMapToControlledAccess(authzMap) - obj.ControlledAccess = &controlled - } - - ams := []drsapi.AccessMethod{am} - obj.AccessMethods = &ams - return obj, nil -} - -// ConvertToCandidate converts a DRS object to a registration candidate, -// stripping server-assigned fields (Id, SelfUri, timestamps). -func ConvertToCandidate(obj *drsapi.DrsObject) drsapi.DrsObjectCandidate { - if obj == nil { - return drsapi.DrsObjectCandidate{} - } - return drsapi.DrsObjectCandidate{ - AccessMethods: obj.AccessMethods, - Aliases: obj.Aliases, - Checksums: obj.Checksums, - Contents: obj.Contents, - ControlledAccess: obj.ControlledAccess, - Description: obj.Description, - MimeType: obj.MimeType, - Name: obj.Name, - Size: obj.Size, - Version: obj.Version, - } -} - -func DrsObjectMatchesScope(obj *drsapi.DrsObject, org, project string) bool { - if obj == nil { - return false - } - return AuthzMapMatchesScope(ControlledAccessToAuthzMap(derefStringSlice(obj.ControlledAccess)), org, project) -} - -func derefStringSlice(ptr *[]string) []string { - if ptr == nil { - return nil - } - return append([]string(nil), (*ptr)...) -} diff --git a/common/drs_test.go b/common/drs_test.go deleted file mode 100644 index 073d5249..00000000 --- a/common/drs_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package common - -import ( - "testing" - - drsapi "github.com/calypr/syfon/apigen/client/drs" -) - -func TestDrsObjectMatchesScope(t *testing.T) { - t.Run("controlled access", func(t *testing.T) { - obj := &drsapi.DrsObject{ - ControlledAccess: &[]string{"/organization/org/project/proj"}, - } - if !DrsObjectMatchesScope(obj, "org", "proj") { - t.Fatal("expected controlled-access match") - } - }) - - t.Run("access method authz ignored", func(t *testing.T) { - authz := []string{"/programs/org/projects/proj"} - obj := &drsapi.DrsObject{ - AccessMethods: &[]drsapi.AccessMethod{{ - Type: drsapi.AccessMethodTypeS3, - Authorizations: &struct { - BearerAuthIssuers *[]string `json:"bearer_auth_issuers,omitempty"` - DrsObjectId *string `json:"drs_object_id,omitempty"` - PassportAuthIssuers *[]string `json:"passport_auth_issuers,omitempty"` - SupportedTypes *[]drsapi.AccessMethodAuthorizationsSupportedTypes `json:"supported_types,omitempty"` - }{BearerAuthIssuers: &authz}, - }}, - } - if DrsObjectMatchesScope(obj, "org", "proj") { - t.Fatal("access method authorizations should not define scope") - } - }) -} diff --git a/common/resource.go b/common/resource.go index ad4abcd0..c8ea5e88 100644 --- a/common/resource.go +++ b/common/resource.go @@ -23,13 +23,7 @@ func ResourcePath(org, project string) (string, error) { return "/organization/" + org + "/project/" + project, nil } -// StoragePrefix returns the storage path prefix for an org/project scope -// (no leading slash), e.g. "programs/myorg/projects/myproject". -func StoragePrefix(org, project string) string { - _ = org - _ = project - return "" -} + // NormalizeChecksum trims whitespace and an optional sha256: prefix. func NormalizeChecksum(raw string) string { @@ -161,28 +155,7 @@ func AuthzMapToControlledAccess(authzMap map[string][]string) []string { return NormalizeAccessResources(AuthzMapToList(authzMap)) } -// AuthzMapMatchesScope reports whether the authz map grants access for the given -// org and project. An empty project list in the map means org-wide access. -func AuthzMapMatchesScope(authzMap map[string][]string, org, project string) bool { - org = strings.TrimSpace(org) - project = strings.TrimSpace(project) - if len(authzMap) == 0 || org == "" { - return false - } - projects, ok := authzMap[org] - if !ok { - return false - } - if len(projects) == 0 { - return true - } - for _, candidate := range projects { - if strings.TrimSpace(candidate) == project { - return true - } - } - return false -} + // ResourceScope parses a canonical or raw access resource into // (organization, project). It accepts "/organization/org", diff --git a/common/resource_test.go b/common/resource_test.go index 9bcbbc1c..d5f875a1 100644 --- a/common/resource_test.go +++ b/common/resource_test.go @@ -41,17 +41,7 @@ func TestResourceAndAuthzHelpers(t *testing.T) { } }) - t.Run("storage prefix", func(t *testing.T) { - if got := StoragePrefix("", ""); got != "" { - t.Fatalf("expected empty prefix, got %q", got) - } - if got := StoragePrefix("syfon", ""); got != "" { - t.Fatalf("unexpected org prefix: %q", got) - } - if got := StoragePrefix("syfon", "e2e"); got != "" { - t.Fatalf("unexpected project prefix: %q", got) - } - }) + t.Run("checksum normalization", func(t *testing.T) { if got := NormalizeChecksum(" sha256:ABC123 "); got != "ABC123" { @@ -122,21 +112,5 @@ func TestResourceAndAuthzHelpers(t *testing.T) { } }) - t.Run("authz map matches scope", func(t *testing.T) { - projectScoped := map[string][]string{"syfon": {"e2e"}} - if !AuthzMapMatchesScope(projectScoped, "syfon", "e2e") { - t.Fatal("expected project-scoped match") - } - if AuthzMapMatchesScope(projectScoped, "syfon", "other") { - t.Fatal("expected project-scoped miss") - } - orgWide := map[string][]string{"syfon": {}} - if !AuthzMapMatchesScope(orgWide, "syfon", "anything") { - t.Fatal("expected org-wide match") - } - if AuthzMapMatchesScope(nil, "syfon", "e2e") { - t.Fatal("expected nil map miss") - } - }) } diff --git a/docs/operator-guide-migration-indexd.md b/docs/operator-guide-migration-indexd.md deleted file mode 100644 index 4f239501..00000000 --- a/docs/operator-guide-migration-indexd.md +++ /dev/null @@ -1,83 +0,0 @@ -# Operator Guide: Indexd to Syfon Migration - -`syfon migrate` is a two-step offline workflow: - -1. `syfon migrate export` reads records from a source Gen3-mounted Indexd API and writes a local SQLite dump. -2. `syfon migrate import` reads that dump and loads records into a target Syfon instance through Syfon's `controlled_access`-aware `/index/bulk` compatibility loader. - -No migration-specific Syfon server endpoint is required. - -## Export - -```bash -syfon migrate export \ - --server "https://source-gen3.example.org" \ - --source-profile source \ - --batch-size 500 -``` - -By default, export writes `./indexd-records.sqlite`. Use `--dump` only when you want a different output path. - -The exporter uses Indexd's DID cursor (`start=`) rather than page offsets and de-duplicates records by DID. A single sweep is sufficient for a quiet Indexd database; increase `--sweeps` only if the source is being modified during export or the deployment is known to return inconsistent pages. - -Export reads Indexd records from `/index/index`, so for `--server https://calypr-dev.ohsu.edu` it queries `https://calypr-dev.ohsu.edu/index/index`. - -Useful export flags: - -- `--limit 1000`: canary export with only the first N unique records. -- `--dry-run`: fetch, transform, and validate without writing the dump. -- `--default-controlled-access /programs/open`: apply `controlled_access` only when a source record has no `authz`. -- `--default-authz /programs/open`: deprecated alias for `--default-controlled-access`. -- `--source-token`: use a raw bearer token instead of a Gen3 profile. - -## Import - -```bash -syfon migrate import \ - --dump ./indexd-records.sqlite \ - --server "https://target-gen3-with-syfon.example.org" \ - --target-profile target \ - --batch-size 500 -``` - -The import command reads the SQLite dump in batches and posts to the existing `POST /index/bulk` endpoint using `controlled_access` plus `access_methods`. It does not recreate legacy per-path `auth` maps. - -For a local Syfon server protected by local basic auth, use: - -```bash -syfon migrate import \ - --server http://localhost:8080 \ - --dump ./indexd-records.sqlite \ - --target-basic-user drs-user \ - --target-basic-password drs-pass -``` - -## Validation - -After a canary or full run, spot-check a few records: - -```bash -curl -u drs-user:drs-pass "https://target-gen3-with-syfon.example.org/index?limit=10" | jq . -curl -s "https://target-gen3-with-syfon.example.org/index/" | jq . -curl -s -X POST "https://target-gen3-with-syfon.example.org/index/bulk/hashes" \ - -H "Content-Type: application/json" \ - -d '{"hashes":["sha256:"]}' | jq . -``` - -`GET /index` is paged. Use `limit` for the page size and `start=` for the next page; `page=` remains supported for compatibility but should not be used for large traversals. The server caps list pages at `10000` records, but interactive clients should usually stay near `1000-2000`. - -For exact project-scoped listings, the recommended query shape is a cursor walk over the controlled-access index: - -```sql -SELECT DISTINCT ca.object_id -FROM drs_object_controlled_access ca -WHERE ca.resource = $1 AND ca.object_id > $2 -ORDER BY ca.object_id -LIMIT $3 -``` - -The `DISTINCT` remains intentional because Syfon does not currently enforce uniqueness on `(resource, object_id)`. Postgres should still plan this as an index-driven scan on `drs_object_controlled_access(resource, object_id)` rather than a deep-offset walk. - -When validating a deployment, run `EXPLAIN (ANALYZE, BUFFERS)` on the scoped ID query and confirm later-page requests remain index-driven and do not regress toward large offset costs or fanout-heavy row counts. - -The migration preserves DIDs, hashes, URLs, file names, descriptions, versions, timestamps, access methods, and authz-derived `controlled_access` claims for normal Indexd object records. Deprecated Indexd fields such as `baseid`, `rev`, `acl`, `metadata`, `urls_metadata`, `form`, and `uploader` are intentionally not loaded into Syfon. diff --git a/docs/pr-summary-feature-optimize.md b/docs/pr-summary-feature-optimize.md index 8d663c42..ab205895 100644 --- a/docs/pr-summary-feature-optimize.md +++ b/docs/pr-summary-feature-optimize.md @@ -87,7 +87,6 @@ Several supporting docs in this branch explain the intended behavior around: - project hierarchy browsing - configurable `/index` page sizing -- org/project storage metrics ### 6) Client and CLI restructuring diff --git a/docs/problem-solution-org-project-storage-metrics.md b/docs/problem-solution-org-project-storage-metrics.md deleted file mode 100644 index fe62ac05..00000000 --- a/docs/problem-solution-org-project-storage-metrics.md +++ /dev/null @@ -1,103 +0,0 @@ -# Problem / Solution: Organization and Project Storage Metrics - -## Problem - -The frontend organization explorer wants compact summary statistics at the organization and project level, especially: - -- total file count per project -- total stored bytes per project -- organization-level rollups across accessible projects - -A frontend workaround that pages through `GET /index?organization=...&project=...` for every visible project and then sums record sizes in JavaScript is not sustainable. - -Problems with the frontend approach: - -- request volume grows linearly with the number of visible projects -- large projects require walking every page of the index listing just to compute a summary row -- page load time and UI responsiveness become coupled to full index traversal -- multiple clients would independently repeat the same aggregation work -- the summary is conceptually a metrics concern, not a file-listing concern - -## Desired Outcome - -Syfon should expose organization- and project-scoped aggregate metrics so clients can request summary stats directly without enumerating every record. - -At minimum, the metrics surface should support: - -- exact project summaries for `{organization, project}` -- organization rollups across all projects in that organization -- total file count -- total stored bytes - -## Proposed Direction - -Add org/project filters and aggregate responses to the Syfon metrics API. - -Possible shapes: - -1. Extend an existing metrics endpoint to accept `organization` and optional `project` filters. -2. Add a dedicated metrics endpoint for storage summaries, for example a summary route under the metrics namespace. -3. Support grouped responses so one request can return all project summaries for an organization. - -Example response shape: - -```json -{ - "organization": "aced", - "project": "evotypes", - "file_count": 12842, - "total_bytes": 9412381234 -} -``` - -Example grouped organization response: - -```json -{ - "organization": "aced", - "projects": [ - { - "project": "evotypes", - "file_count": 12842, - "total_bytes": 9412381234 - } - ], - "organization_totals": { - "file_count": 12842, - "total_bytes": 9412381234 - } -} -``` - -## Data Semantics - -Open questions that should be decided explicitly: - -- Whether `total_bytes` is the sum of indexed record sizes, object-store bytes, or another canonical storage measure. -- Whether deleted, superseded, or non-current versions should contribute to totals. -- Whether metrics should be computed from the index, persisted as pre-aggregated counters, or refreshed asynchronously. -- Whether authorization should filter the response to only scopes visible to the caller, or whether this endpoint is only used for already-authorized org/project scopes. - -## Why This Belongs in Metrics - -This is shared aggregate data with stable semantics and broad reuse potential. - -Potential consumers include: - -- organization/project explorer pages -- administrative dashboards -- quota and storage reporting -- operational monitoring - -A metrics endpoint keeps listing endpoints focused on record retrieval and keeps clients from rebuilding the same aggregation logic independently. - -## Acceptance Criteria - -- A client can request project-level totals without traversing every `/index` page. -- A client can request organization-level totals without traversing every project. -- Returned values have documented semantics for file count and bytes. -- The response is fast enough to support summary rows in interactive UIs. - -## Current Status - -Frontend aggregate-size display has been removed for now. The organization explorer should only render data that can be fetched efficiently until a Syfon metrics implementation exists. diff --git a/docs/problem-solution-project-hierarchy-index.md b/docs/problem-solution-project-hierarchy-index.md index c16908b6..7b4a368e 100644 --- a/docs/problem-solution-project-hierarchy-index.md +++ b/docs/problem-solution-project-hierarchy-index.md @@ -53,7 +53,7 @@ The hierarchy should be derived from authoritative object/bucket path informatio Open questions: -- Which field should be canonical for hierarchy derivation: bucket path, object key, `file_name`, or another normalized source? +- Which field should be canonical for hierarchy derivation: bucket path, object key, or another normalized source? - How should conflicts be handled when metadata names differ from object paths? - Should directories be materialized eagerly or derived lazily and cached? - How should renames or path changes be represented historically? diff --git a/go.mod b/go.mod index 5644d2ae..ae3e3c3c 100644 --- a/go.mod +++ b/go.mod @@ -1,68 +1,67 @@ module github.com/calypr/syfon -go 1.26.4 +go 1.26.5 require ( - cloud.google.com/go/storage v1.62.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 + cloud.google.com/go/storage v1.63.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.32.14 - github.com/aws/aws-sdk-go-v2/credentials v1.19.14 - github.com/aws/aws-sdk-go-v2/service/kms v1.50.4 - github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 - github.com/aws/smithy-go v1.24.3 + github.com/aws/aws-sdk-go-v2 v1.42.1 + github.com/aws/aws-sdk-go-v2/config v1.32.30 + github.com/aws/aws-sdk-go-v2/credentials v1.19.29 + github.com/aws/aws-sdk-go-v2/service/kms v1.54.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1 + github.com/aws/smithy-go v1.27.3 github.com/calypr/syfon/apigen v0.2.8 github.com/calypr/syfon/client v0.3.0 - github.com/getkin/kin-openapi v0.135.0 - github.com/gofiber/fiber/v3 v3.1.0 + github.com/getkin/kin-openapi v0.142.0 + github.com/gofiber/fiber/v3 v3.4.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 - github.com/hashicorp/go-plugin v1.7.0 + github.com/hashicorp/go-plugin v1.8.0 github.com/lib/pq v1.12.3 - github.com/mattn/go-sqlite3 v1.14.42 + github.com/mattn/go-sqlite3 v1.14.48 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/testcontainers/testcontainers-go v0.42.0 - gocloud.dev v0.45.0 - google.golang.org/api v0.276.0 + gocloud.dev v0.46.0 + google.golang.org/api v0.288.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.2 // indirect cloud.google.com/go v0.123.0 // indirect - cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth v0.22.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.9.0 // indirect - cloud.google.com/go/monitoring v1.27.0 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect dario.cat/mergo v1.0.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/andybalholm/brotli v1.2.0 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.4.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.32.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.44.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect @@ -79,33 +78,30 @@ require ( github.com/ebitengine/purego v0.10.0 // indirect github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect - github.com/fatih/color v1.16.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/jsonpointer v0.22.5 // indirect - github.com/go-openapi/swag/jsonname v0.25.5 // indirect - github.com/gofiber/schema v1.7.0 // indirect - github.com/gofiber/utils/v2 v2.0.2 // indirect + github.com/go-openapi/jsonpointer v1.0.0 // indirect + github.com/gofiber/schema v1.8.2 // indirect + github.com/gofiber/utils/v2 v2.2.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect - github.com/googleapis/gax-go/v2 v2.21.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/yamux v0.1.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.19.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect - github.com/mailru/easyjson v0.9.2 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.21 // indirect - github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-runewidth v0.0.24 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect github.com/moby/moby/api v1.54.1 // indirect @@ -115,51 +111,49 @@ require ( github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect - github.com/oapi-codegen/runtime v1.4.0 // indirect - github.com/oasdiff/yaml v0.0.9 // indirect - github.com/oasdiff/yaml3 v0.0.9 // indirect - github.com/oklog/run v1.1.0 // indirect + github.com/oapi-codegen/runtime v1.5.0 // indirect + github.com/oasdiff/yaml v0.1.1 // indirect + github.com/oasdiff/yaml3 v0.0.14 // indirect + github.com/oklog/run v1.2.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/stretchr/testify v1.11.1 // indirect - github.com/tinylib/msgp v1.6.3 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.69.0 // indirect - github.com/vbauerster/mpb/v8 v8.12.0 // indirect - github.com/woodsbury/decimal128 v1.4.0 // indirect + github.com/valyala/fasthttp v1.72.0 // indirect + github.com/vbauerster/mpb/v8 v8.12.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect - golang.org/x/crypto v0.51.0 // indirect - golang.org/x/net v0.55.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/grpc v1.80.0 // indirect + google.golang.org/genproto v0.0.0-20260713224248-f5fc221cf8c4 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260713224248-f5fc221cf8c4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260713224248-f5fc221cf8c4 // indirect + google.golang.org/grpc v1.82.0 // indirect google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/ini.v1 v1.67.1 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect ) diff --git a/go.sum b/go.sum index e8842c0e..f94ec34a 100644 --- a/go.sum +++ b/go.sum @@ -1,53 +1,53 @@ -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= -cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ= +cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.9.0 h1:89wyjxT6DL4b5rk/Nk8eBC9DHqf+JiMstrn5IEYxFw4= -cloud.google.com/go/iam v1.9.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= -cloud.google.com/go/logging v1.15.0 h1:6ooUEBNT6jdWh2b36+iuPn6b/R9qN/tHCbvGS5255gg= -cloud.google.com/go/logging v1.15.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= -cloud.google.com/go/longrunning v0.10.0 h1:4OWvp1BjCvoeSZTog3sRFDu6j4IrI9TI4/Y9N+8h25g= -cloud.google.com/go/longrunning v0.10.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= -cloud.google.com/go/monitoring v1.27.0 h1:BhYwMqao+e5Nn7JtWMM9m6zRtKtVUK6kJWMizXChkLU= -cloud.google.com/go/monitoring v1.27.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= -cloud.google.com/go/storage v1.62.1 h1:Os0G3XbUbjZumkpDUf2Y0rLoXJTCF1kU2kWUujKYXD8= -cloud.google.com/go/storage v1.62.1/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= -cloud.google.com/go/trace v1.13.0 h1:RfqsqPOiSCG8ql50UZt5F65KrVa1zbY9mJrO7xvZfbE= -cloud.google.com/go/trace v1.13.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= +cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= +cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= +cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k= +cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/storage v1.63.1 h1:CYXILV9G4CH0C18IQ9+V0h4XiqD2LhKnMLO0o7uJWNs= +cloud.google.com/go/storage v1.63.1/go.mod h1:lWyAtwvDZHdL3k68WVKbESP6bmWaV23ZJJ/JEVw/ZaQ= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1 h1:edShSHV3DV90+kt+CMaEXEzR9QF7wFrPJxVGz2blMIU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 h1:O2sXMyJh8b7devAGdE+163xtRurt0RVpB6DIzX5vGfg= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0/go.mod h1:hEpiGU18xf70qb3jbTcIggWAiEfX/cOIVc2OTe4OegA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0 h1:ZIT85vKP7LBS84XJ0WdJ3dPOX3iz4j3c0+lpajGQMyo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0/go.mod h1:rqP9UEhOXv9WhQ7Gjz+G5y/pf8+BJZW5/Ts0AhE0PwE= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 h1:0YP0+/ixwu+Uqeu/FGiBZNQ19huiUxxiPXIc9WsLKuQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0/go.mod h1:6ZZMQhZKDvUvkJw2rc+oDP90tMMzuU/J+5HG1ZmPOmE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 h1:yzIYdwuro811Z27D3T80Wkd3rqZzb0K43nner7Eh1yE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0 h1:ZYGajzJNcirVZpT1rltgf9iM+j9zZ4v8V9DrF+xKRJ8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0/go.mod h1:PDQyYBOzGtQgvshQI//UiXyzuMHCz0ndyu+4W8X82vM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.58.0 h1:IBF8BbhKJkMsON/eY+LMu3aF3XMiotCb9KvkUmEkOJo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.58.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 h1:SBZzZCiPmDrUV7NSCWY54OnKikO/oTydPCvyEyYaDDE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= @@ -55,52 +55,50 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= -github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= -github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/kms v1.50.4 h1:PgD1y0ZagPokGIZPmejCBUySBzOFDN+leZxCOfb1OEQ= -github.com/aws/aws-sdk-go-v2/service/kms v1.50.4/go.mod h1:FfXDb5nXrsoGgxsBFxwxr3vdHXheC2tV+6lmuLghhjQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.24.3 h1:XgOAaUgx+HhVBoP4v8n6HCQoTRDhoMghKqw4LNHsDNg= -github.com/aws/smithy-go v1.24.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek= +github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= +github.com/aws/aws-sdk-go-v2/config v1.32.30 h1:XwsEzpTJfQYJbFicz/QMLwAZdyeNVVoOEkbF7R3gPJk= +github.com/aws/aws-sdk-go-v2/config v1.32.30/go.mod h1:Ud32SuMc+/9BGxfpSVld7HrE2o05JwKmXY4M3jOQNZU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29 h1:WHZGssHH887cO0ox07SIQZsFx3MKD4ps6w0xUEmnKYQ= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29/go.mod h1:Mhl0xR6zjguiuj00XRx2wMx22sAltk7oya39sT7fdg8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I= +github.com/aws/aws-sdk-go-v2/service/kms v1.54.1 h1:aeJAJyvWS3gQ679pJbz8ZdOh3MViD1zvEdoZMVEawbg= +github.com/aws/aws-sdk-go-v2/service/kms v1.54.1/go.mod h1:0RXNc6Yf3AvSMldGD6Lcch96Ojlw2TtGnHsqfD/L4u8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1 h1:LkBKxAOE5WXjlFuFZqPG1rREnl6I6QCMElcXFDEidos= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do= +github.com/aws/aws-sdk-go-v2/service/signin v1.4.1 h1:V7ZZ300WPXGjvkyore5DGe0ljVPOxCXie/thWdtSBXE= +github.com/aws/aws-sdk-go-v2/service/signin v1.4.1/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg= +github.com/aws/aws-sdk-go-v2/service/sso v1.32.1 h1:gYFYh4iLLcAOJRLNPY2aD2g9DIhKn4eof8UkIrr1rTk= +github.com/aws/aws-sdk-go-v2/service/sso v1.32.1/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1 h1:arjT9Cm3/WYbGmD5TUZHk4UQn4Lle1fUNZs5FC6CtF0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84= +github.com/aws/aws-sdk-go-v2/service/sts v1.44.1 h1:RvfHDg+xvAeZ+5741vUEjpOVtYSIm93W2zhx10Xtydw= +github.com/aws/aws-sdk-go-v2/service/sts v1.44.1/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= @@ -135,6 +133,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -150,14 +150,14 @@ github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJP github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/getkin/kin-openapi v0.135.0 h1:751SjYfbiwqukYuVjwYEIKNfrSwS5YpA7DZnKSwQgtg= -github.com/getkin/kin-openapi v0.135.0/go.mod h1:6dd5FJl6RdX4usBtFBaQhk9q62Yb2J0Mk5IhUO/QqFI= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78= +github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/getkin/kin-openapi v0.142.0 h1:izj0vBdFprMhitfzaX8sTqztsEQyvwhssBoB6n8NO7w= +github.com/getkin/kin-openapi v0.142.0/go.mod h1:3BH9M9XDe/y9M5DSvEocVYAYq1w0qrhJHjC/vZi0AaY= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -167,20 +167,16 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= -github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= -github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= -github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= -github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= -github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gofiber/fiber/v3 v3.1.0 h1:1p4I820pIa+FGxfwWuQZ5rAyX0WlGZbGT6Hnuxt6hKY= -github.com/gofiber/fiber/v3 v3.1.0/go.mod h1:n2nYQovvL9z3Too/FGOfgtERjW3GQcAUqgfoezGBZdU= -github.com/gofiber/schema v1.7.0 h1:yNM+FNRZjyYEli9Ey0AXRBrAY9jTnb+kmGs3lJGPvKg= -github.com/gofiber/schema v1.7.0/go.mod h1:A/X5Ffyru4p9eBdp99qu+nzviHzQiZ7odLT+TwxWhbk= -github.com/gofiber/utils/v2 v2.0.2 h1:ShRRssz0F3AhTlAQcuEj54OEDtWF7+HJDwEi/aa6QLI= -github.com/gofiber/utils/v2 v2.0.2/go.mod h1:+9Ub4NqQ+IaJoTliq5LfdmOJAA/Hzwf4pXOxOa3RrJ0= +github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= +github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= +github.com/gofiber/fiber/v3 v3.4.0 h1:F0aND4vwZF7dR7cbvSwFQQEpBU902XHKWxrLsFBkVqw= +github.com/gofiber/fiber/v3 v3.4.0/go.mod h1:nAhJfdxUIJJph2tPWPmqWf8QDIN2iiqQiQf3lENZpdk= +github.com/gofiber/schema v1.8.2 h1:wq+LO2xEGlsqma/8Akp9PUebQ6vcsYmF0xYQ4F2ijvU= +github.com/gofiber/schema v1.8.2/go.mod h1:iyAMJztdyky7Pk2U7bwUP3EHjzMqUNjZ/hglE0nYO/g= +github.com/gofiber/utils/v2 v2.2.0 h1:YSSmCzQponq/f9uSOg2HtXC5qK1Dmor0o6DqaQVz8GE= +github.com/gofiber/utils/v2 v2.2.0/go.mod h1:Ieopk6sQh7rbhQ12aBNCJtJuG0gxAg0nz63sFCrrOmE= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -196,18 +192,18 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= -github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= -github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= -github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl/wMbiI= -github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4= +github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= +github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-plugin v1.7.0 h1:YghfQH/0QmPNc/AZMTFE3ac8fipZyZECHdDPshfk+mA= -github.com/hashicorp/go-plugin v1.7.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= +github.com/hashicorp/go-plugin v1.8.0 h1:ie8S6RRY8RvB2usYZv+AAZ/wBvx2AU5p5QeP5j/FORs= +github.com/hashicorp/go-plugin v1.8.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= @@ -216,12 +212,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -234,20 +228,18 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.9.2 h1:dX8U45hQsZpxd80nLvDGihsQ/OxlvTkVUXH2r/8cb2M= -github.com/mailru/easyjson v0.9.2/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= -github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= -github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= -github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= -github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= +github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-sqlite3 v1.14.48 h1:7XHIgl0a8HwOaiK4E47ozLkST78rR9+OtNGx27D/TFs= +github.com/mattn/go-sqlite3 v1.14.48/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= @@ -266,22 +258,20 @@ github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/oapi-codegen/runtime v1.4.0 h1:KLOSFOp7UzkbS7Cs1ms6NBEKYr0WmH2wZG0KKbd2er4= -github.com/oapi-codegen/runtime v1.4.0/go.mod h1:5sw5fxCDmnOzKNYmkVNF8d34kyUeejJEY8HNT2WaPec= -github.com/oasdiff/yaml v0.0.9 h1:zQOvd2UKoozsSsAknnWoDJlSK4lC0mpmjfDsfqNwX48= -github.com/oasdiff/yaml v0.0.9/go.mod h1:8lvhgJG4xiKPj3HN5lDow4jZHPlx1i7dIwzkdAo6oAM= -github.com/oasdiff/yaml3 v0.0.9 h1:rWPrKccrdUm8J0F3sGuU+fuh9+1K/RdJlWF7O/9yw2g= -github.com/oasdiff/yaml3 v0.0.9/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= -github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= -github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= +github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= +github.com/oapi-codegen/runtime v1.5.0 h1:aiil4QnH+eiWYSO60eaYZ4aur7sJH3rz6BvT5EBFnxc= +github.com/oapi-codegen/runtime v1.5.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/oasdiff/yaml v0.1.1 h1:6nHx+pn9gBRM6YpBlFZFQGCCd1nuvqOBtTD3KKTgGxY= +github.com/oasdiff/yaml v0.1.1/go.mod h1:EYJNoyktvWMJ0Hmhx+6qTaqMOsalUaRGT8Sj1hNcegU= +github.com/oasdiff/yaml3 v0.0.14 h1:aLJee3hxBK2H5wdXd9iPcIXb93Nty1Ge0pT171eHtkw= +github.com/oasdiff/yaml3 v0.0.14/go.mod h1:csto2xfDjYccdUn/yw/bPjj/cYTdp6HtFA0J4TWG+gg= +github.com/oklog/run v1.2.0 h1:O8x3yXwah4A73hJdlrwo/2X6J62gE5qTMusH0dvz60E= +github.com/oklog/run v1.2.0/go.mod h1:mgDbKRSwPhJfesJ4PntqFUbKQRZ50NgmZTSPlFA0YFk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= -github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -296,8 +286,10 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:Om github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shamaton/msgpack/v3 v3.1.0 h1:jsk0vEAqVvvS9+fTZ5/EcQ9tz860c9pWxJ4Iwecz8gU= -github.com/shamaton/msgpack/v3 v3.1.0/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/shamaton/msgpack/v3 v3.2.0 h1:1q2Ms+MWmuRju+PuDMSFDB7p7621npeX4zprJN5Zck8= +github.com/shamaton/msgpack/v3 v3.2.0/go.mod h1:sgBYvEiyz8JR1NC3yGRoPVME9xXovpnh3l/plW1nfRo= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= @@ -307,8 +299,8 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= -github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -325,22 +317,18 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= -github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= -github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI= -github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw= -github.com/vbauerster/mpb/v8 v8.12.0 h1:+gneY3ifzc88tKDzOtfG8k8gfngCx615S2ZmFM4liWg= -github.com/vbauerster/mpb/v8 v8.12.0/go.mod h1:V02YIuMVo301Y1VE9VtZlD8s84OMsk+EKN6mwvf/588= -github.com/woodsbury/decimal128 v1.4.0 h1:xJATj7lLu4f2oObouMt2tgGiElE5gO6mSWUjQsBgUlc= -github.com/woodsbury/decimal128 v1.4.0/go.mod h1:BP46FUrVjVhdTbKT+XuQh2xfQaGki9LMIRJSFuh6THU= +github.com/valyala/fasthttp v1.72.0 h1:R7kYdoWhn1ye1fVpP+cDHDJwYm3NkwLliwgzJ/Abg7M= +github.com/valyala/fasthttp v1.72.0/go.mod h1:zsbLTYqcpIktdQytlVBwIjY9La5d6bs990nBxWg8efk= +github.com/vbauerster/mpb/v8 v8.12.1 h1:pyj3yQ2ZGQJgUXm4h17QpR+eERaNz5OQ1ftPSEE/sMM= +github.com/vbauerster/mpb/v8 v8.12.1/go.mod h1:XLXRfStkw/6i5k0aQltijDHT1Z93fD1DVwmIdcFUp6k= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -349,35 +337,37 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= -go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0/go.mod h1:D7J12YRapIekYyPWgGPlA/23pRmpSEZC5xJC/TTLI9U= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0 h1:hqxVTu/GtBF+vJ8d1fzW7fRxZFvgoDjWcxwwCaFDYpU= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY= -gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= +gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -386,12 +376,12 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -399,23 +389,23 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.276.0 h1:nVArUtfLEihtW+b0DdcqRGK1xoEm2+ltAihyztq7MKY= -google.golang.org/api v0.276.0/go.mod h1:Fnag/EWUPIcJXuIkP1pjoTgS5vdxlk3eeemL7Do6bvw= -google.golang.org/genproto v0.0.0-20260414002931-afd174a4e478 h1:aLsVTW0lZ8+IY5u/ERjZSCvAmhuR7slKzyha3YikDNA= -google.golang.org/genproto v0.0.0-20260414002931-afd174a4e478/go.mod h1:YJAzKjfHIUHb9T+bfu8L7mthAp7VVXQBUs1PLdBWS7M= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/api v0.288.0 h1:glhO/J88obKP5I269W3hB73dvBKrjU56ZfmNlNXpgTU= +google.golang.org/api v0.288.0/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= +google.golang.org/genproto v0.0.0-20260713224248-f5fc221cf8c4 h1:PWVFocb6Y93aqxY7m9ArHu5zQiLkIrXBLr+ajQxRHH0= +google.golang.org/genproto v0.0.0-20260713224248-f5fc221cf8c4/go.mod h1:v47/Kzm6KzB/2uGxuDjGww+7Dk7ny/VFIaIw4WxEtu0= +google.golang.org/genproto/googleapis/api v0.0.0-20260713224248-f5fc221cf8c4 h1:lI0NbdWVmT6lOJJNDd7vyeTdfxP/7ouCLSJUKNNXa0k= +google.golang.org/genproto/googleapis/api v0.0.0-20260713224248-f5fc221cf8c4/go.mod h1:WRrQ7/7N19PypuT0fxLOL5Lq0waoiRri4FbtHDEKrGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260713224248-f5fc221cf8c4 h1:7RtFDizMtT9eZzHzKxifoMGfcDBBy+LYZlgfg24ZmOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260713224248-f5fc221cf8c4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= -gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work b/go.work index 269df952..fb0d2bbe 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.26.4 +go 1.26.5 use ( . diff --git a/go.work.sum b/go.work.sum index df6e708c..be41eb9a 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,251 +1,445 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.82.0/go.mod h1:vlKccHJGuFBFufnAnuB08dfEH9Y3H7dzDzRECFdC2TA= cloud.google.com/go/accessapproval v1.10.0 h1:AkUX4ayuxplE7ZSbya9bBvXFoRu8E9bJYmulNVcex6I= cloud.google.com/go/accessapproval v1.10.0/go.mod h1:7bmInw17bQX+ZPi7YmReC3xKymDrMmxXaUnaI6zQOqI= +cloud.google.com/go/accessapproval v1.13.0/go.mod h1:7bmInw17bQX+ZPi7YmReC3xKymDrMmxXaUnaI6zQOqI= cloud.google.com/go/accesscontextmanager v1.11.0 h1:41YvtDKbyIVwhpzDUJ7TI8Ca3BRACvT4cPQSeg+lXsM= cloud.google.com/go/accesscontextmanager v1.11.0/go.mod h1:VO15iVnsM0FO9Dt8hSFPgkuHRZjq6LEYZq1szJ27U2k= +cloud.google.com/go/accesscontextmanager v1.14.0/go.mod h1:VO15iVnsM0FO9Dt8hSFPgkuHRZjq6LEYZq1szJ27U2k= cloud.google.com/go/aiplatform v1.123.0 h1:Gh/SjaxI5wu/jeYQPPn8m+iGECBxlvsoyCWv8MqJ6xs= cloud.google.com/go/aiplatform v1.123.0/go.mod h1:yWTZiCunYDnyxeWWD14tDo6+BMlvAUCC5VxuxhvbrVI= +cloud.google.com/go/aiplatform v1.125.0/go.mod h1:yWTZiCunYDnyxeWWD14tDo6+BMlvAUCC5VxuxhvbrVI= cloud.google.com/go/analytics v0.32.0 h1:dGhOv5gmWi0sq/5oQ9Ojfx/R8daCAjv5CuvxVU0Zs8g= cloud.google.com/go/analytics v0.32.0/go.mod h1:V9Qef2N0y8GDqQ9FTlmM2XpDEMYonZJRPSUNGZlPCcc= +cloud.google.com/go/analytics v0.36.0/go.mod h1:q/KfbXopU5Ad7LThQDrcx/B6A6kAQNhQU7zX4gD+JLQ= cloud.google.com/go/apigateway v1.9.0 h1:7wOtsGpsfgbUNMfw1D4CHpz6oLchn4vWxoVi2uAjW2Y= cloud.google.com/go/apigateway v1.9.0/go.mod h1:f3Sk8Tdh1Ty5HR7kgbWB6Yu1M82LM+nIr5DTMZnLZWk= +cloud.google.com/go/apigateway v1.12.0/go.mod h1:f3Sk8Tdh1Ty5HR7kgbWB6Yu1M82LM+nIr5DTMZnLZWk= cloud.google.com/go/apigeeconnect v1.9.0 h1:7Dicdaf2uRZnxYxgcgyVY4IIs0rlj81TNnd5Z+Gt92M= cloud.google.com/go/apigeeconnect v1.9.0/go.mod h1:mYJekCKZHc2ia5yZX5lwtexTn9CzsOfb6+sh/2hi42Q= +cloud.google.com/go/apigeeconnect v1.12.0/go.mod h1:mYJekCKZHc2ia5yZX5lwtexTn9CzsOfb6+sh/2hi42Q= cloud.google.com/go/apigeeregistry v0.12.0 h1:7D4E+Gf1wV5GAo9VsmWnwGX2MFprjjDYMe9ItT4xmg8= cloud.google.com/go/apigeeregistry v0.12.0/go.mod h1:o+j6eA8hYhTWX5gEqMMBVDWY+/QQFrYe/YJBsO19pn0= +cloud.google.com/go/apigeeregistry v1.0.0/go.mod h1:o+j6eA8hYhTWX5gEqMMBVDWY+/QQFrYe/YJBsO19pn0= cloud.google.com/go/appengine v1.11.0 h1:BGf1Ig76ZB5m11sJGLkcMrx9LZ8YkYWQVk9vD+kV8KA= cloud.google.com/go/appengine v1.11.0/go.mod h1:JMjrVFg+YgfksZCWbtA3TgbKbPfZZtapB9cGL/5WVnM= +cloud.google.com/go/appengine v1.14.0/go.mod h1:JMjrVFg+YgfksZCWbtA3TgbKbPfZZtapB9cGL/5WVnM= cloud.google.com/go/area120 v0.12.0 h1:huHekUFvDRlDLM1asSGoKvGp639Hki6WYu0t84gqKTU= cloud.google.com/go/area120 v0.12.0/go.mod h1:jD1fw9W4xxIZMY68g7PpbCPleoeGddFs5jPcdhfg3+Y= +cloud.google.com/go/area120 v0.15.0/go.mod h1:jD1fw9W4xxIZMY68g7PpbCPleoeGddFs5jPcdhfg3+Y= cloud.google.com/go/artifactregistry v1.22.0 h1:yL1iX0kxord8xVi1C+wvWA8zy07x8XyPIv34/uOnYq8= cloud.google.com/go/artifactregistry v1.22.0/go.mod h1:aMmdtqKVmbuxCCb/NGDJYZHsK6AtqlcyvD05ACzs1n8= +cloud.google.com/go/artifactregistry v1.25.0/go.mod h1:aMmdtqKVmbuxCCb/NGDJYZHsK6AtqlcyvD05ACzs1n8= cloud.google.com/go/asset v1.24.0 h1:guLe35IT8wGzMj1HSPCjuF+KYoJRf7uy6AKhXhE4unY= cloud.google.com/go/asset v1.24.0/go.mod h1:+HaDReZQAh/0syAf0uTMeUrMfXikr+KKyDtCdvf7j4M= +cloud.google.com/go/asset v1.27.0/go.mod h1:+HaDReZQAh/0syAf0uTMeUrMfXikr+KKyDtCdvf7j4M= cloud.google.com/go/assuredworkloads v1.15.0 h1:K21uUQ+KzM8QakqBE378lbwkk3VdlqTK31vfUx7Tq/8= cloud.google.com/go/assuredworkloads v1.15.0/go.mod h1:zBnVYn0E+sDW/mhEmcg1R8+8tguXrtBgmfGY0q34kss= +cloud.google.com/go/assuredworkloads v1.18.0/go.mod h1:zBnVYn0E+sDW/mhEmcg1R8+8tguXrtBgmfGY0q34kss= +cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= cloud.google.com/go/automl v1.17.0 h1:9X05fKqAcLbsJ3hFohvcgwmaPnhrUbHt6ph28CWcow8= cloud.google.com/go/automl v1.17.0/go.mod h1:OkHxjbVDblDafhwuP8yEkz1xcUJhgcbhbsieCW7GaiI= +cloud.google.com/go/automl v1.20.0/go.mod h1:OkHxjbVDblDafhwuP8yEkz1xcUJhgcbhbsieCW7GaiI= cloud.google.com/go/baremetalsolution v1.6.0 h1:JYf8RUjYvfLQ9FGCA/PhkmENbhLZkW8yAXTeB6hvots= cloud.google.com/go/baremetalsolution v1.6.0/go.mod h1:o+stutiS8t+HmjNIG92Gkn8H9+5/q27d6lQp7e9GWdg= +cloud.google.com/go/baremetalsolution v1.9.0/go.mod h1:o+stutiS8t+HmjNIG92Gkn8H9+5/q27d6lQp7e9GWdg= cloud.google.com/go/batch v1.16.0 h1:iop/7lKI28WHJ2mEFhkt0HNpX3MXy7r++zqL0BRy1+k= cloud.google.com/go/batch v1.16.0/go.mod h1:dpWfhLmLQZqsTBAFYjZA3pS04fCY5ttTenZcWmSeILw= +cloud.google.com/go/batch v1.19.0/go.mod h1:dpWfhLmLQZqsTBAFYjZA3pS04fCY5ttTenZcWmSeILw= cloud.google.com/go/beyondcorp v1.4.0 h1:UfJgOIKgAoFCtSrYFj/DwRmQxEsC1qvP3RjNgdXYNE8= cloud.google.com/go/beyondcorp v1.4.0/go.mod h1:vujdO0wfsBV2y1egrJxGtwKZr5P5V6bIHKWp1phWHBY= +cloud.google.com/go/beyondcorp v1.7.0/go.mod h1:vujdO0wfsBV2y1egrJxGtwKZr5P5V6bIHKWp1phWHBY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.76.0 h1:wnfVSXN6GEMlsAoHWdhzTC8NMsptOx2hsqPiI+lTs3I= cloud.google.com/go/bigquery v1.76.0/go.mod h1:J4wuqka/1hEpdJxH2oBrUR0vjTD+r7drGkpcA3yqERM= +cloud.google.com/go/bigquery v1.78.0/go.mod h1:NreOOkdlH/8ji6liI8wpSRvys9C6NSqfNx/M1VKQjBc= cloud.google.com/go/bigtable v1.46.0 h1:Bd6vITx01s6gsdEPvjIGJL/oMMdKvraGI34UiixR2gk= cloud.google.com/go/bigtable v1.46.0/go.mod h1:GUM6PdkG3rrDse9kugqvX5+ktwo3ldfLtLi1VFn5Wj4= +cloud.google.com/go/bigtable v1.50.0/go.mod h1:RTannV5mvoJM8KscLTfRYMPo84u9/j+C3PSyYJGf5Ic= cloud.google.com/go/billing v1.22.0 h1:yU4GwnMlr67TAV7apg+Ca2i7Lu1rJnWGgmLS6d+qTyM= cloud.google.com/go/billing v1.22.0/go.mod h1:nQVTVqWyw8YK0QScVV1pXr9KrvjvM7cabIQr6tLJMys= +cloud.google.com/go/billing v1.26.0/go.mod h1:axqDO1uHegh7u5qngkTfqN1djAeLGsWAFAblERgmgEk= cloud.google.com/go/binaryauthorization v1.12.0 h1:dWY+Vq1Y/aIz01okcLU5S4herPtkU/8r9KGsrwzqe7s= cloud.google.com/go/binaryauthorization v1.12.0/go.mod h1:+0CndCJPtcHuVCNok+qQskWvbP5Sp5m6eGL8Vpu5mss= +cloud.google.com/go/binaryauthorization v1.15.0/go.mod h1:+0CndCJPtcHuVCNok+qQskWvbP5Sp5m6eGL8Vpu5mss= cloud.google.com/go/certificatemanager v1.11.0 h1:PATHWGLy/kNDiUlGlM5FCoA+aZVKcELbXai40TsFGEU= cloud.google.com/go/certificatemanager v1.11.0/go.mod h1:QOA8qRoM6/Ik03+srLnBykenGTy0fk78dnPcx5ZWOW8= +cloud.google.com/go/certificatemanager v1.14.0/go.mod h1:QOA8qRoM6/Ik03+srLnBykenGTy0fk78dnPcx5ZWOW8= cloud.google.com/go/channel v1.23.0 h1:mWw74pQ14Kl5c4Akkh/7vnkv9VXJIXMs3O7Qs8TDTuU= cloud.google.com/go/channel v1.23.0/go.mod h1:04T5Wjq+mHlvEUNzExydnBW1vO64q3Q2Wsblp/dpBxY= +cloud.google.com/go/channel v1.26.0/go.mod h1:04T5Wjq+mHlvEUNzExydnBW1vO64q3Q2Wsblp/dpBxY= cloud.google.com/go/cloudbuild v1.27.0 h1:CNvJxVXqDj3cVSXQ2OnVNEa5w7Oo+PZvp41e6A99QAQ= cloud.google.com/go/cloudbuild v1.27.0/go.mod h1:rg52xEmndQQPiC9NV/8sCaVtKxHMU9D9MeU+oE9VGKA= +cloud.google.com/go/cloudbuild v1.31.0/go.mod h1:QeHawskCCsONQoWJAUeV/qOq4Jablq5n0x8hamsQi7Y= cloud.google.com/go/clouddms v1.10.0 h1:fbXBTIzk58x9KTHEGTBw9v7R2PkdiZjz+iu5h+iJQb0= cloud.google.com/go/clouddms v1.10.0/go.mod h1:aMgrOZ+/EKF/PL+h1sDbS+7fAIYV5rTwD+G/apCeHQk= +cloud.google.com/go/clouddms v1.13.0/go.mod h1:aMgrOZ+/EKF/PL+h1sDbS+7fAIYV5rTwD+G/apCeHQk= cloud.google.com/go/cloudtasks v1.15.0 h1:hVjZUqEbqrTl9GPLcGl1jEyd48YXOn8F4Y5HA7OYYEA= cloud.google.com/go/cloudtasks v1.15.0/go.mod h1:3KeCxwtGEyaySL7CR3lMmEa2I4mq1ynXdgmfNiO4RYE= +cloud.google.com/go/cloudtasks v1.18.0/go.mod h1:3KeCxwtGEyaySL7CR3lMmEa2I4mq1ynXdgmfNiO4RYE= cloud.google.com/go/compute v1.58.0 h1:WQamD8d9Vxu+m5IJBxgUzqsbHxXwx/T7Ol4V5GyVCZE= cloud.google.com/go/compute v1.58.0/go.mod h1:5/1KiFDIdFPcx/Fw6pFQpcQaBGc6MGtgSfczeLFBj7o= cloud.google.com/go/contactcenterinsights v1.19.0 h1:3BGjbFCfSnRLkIhhJ4eRinKONZXRDohDxC6A1nZ0EWU= cloud.google.com/go/contactcenterinsights v1.19.0/go.mod h1:2Crd36H59Lwkt4gWrLgmnbnF59IIZIa3XYt1gtNqJkQ= +cloud.google.com/go/contactcenterinsights v1.22.0/go.mod h1:2Crd36H59Lwkt4gWrLgmnbnF59IIZIa3XYt1gtNqJkQ= cloud.google.com/go/container v1.48.0 h1:OIz8pl1oDPYq0+X1eWS3WLQXBWa2UKQtieq1MTP6nl4= cloud.google.com/go/container v1.48.0/go.mod h1:9pBrzW8z1Ytu/c8a3AD8QISILiRsNqYV7lJ1xInrdhU= +cloud.google.com/go/container v1.53.0/go.mod h1:SBOylKhlKYCBFs/8kz2yqRdUW5ctVNHs82JKOTjrB9s= cloud.google.com/go/containeranalysis v0.16.0 h1:v+Q3ZBOf5LdmQ9uiAcxqwTZfGCBpQ6ZniG0iZK398js= cloud.google.com/go/containeranalysis v0.16.0/go.mod h1:4bj+b/9SrnbGoxOVRaPwr1DlKxeRPHFWDZE3We/l8ng= +cloud.google.com/go/containeranalysis v0.19.0/go.mod h1:Zq0XHzUIa0oTa7H6aSR8HWqeJnoRI9syUcYJzfozjZQ= cloud.google.com/go/datacatalog v1.28.0 h1:hkwiX19v+qpZtVu8wpSVbb50P90Td3LmbGZYG6Dcivk= cloud.google.com/go/datacatalog v1.28.0/go.mod h1:MP8V3kNuESnwMk4mB6zdWmw/4KQ5xZ8dyUNVsggqN5I= +cloud.google.com/go/datacatalog v1.32.0/go.mod h1:DE272tynQUwheJeQAyVfV+nO8yrdkuDyOgH2LtOrkWM= cloud.google.com/go/dataflow v0.13.0 h1:ImEtopgIS7kQBQh+tnj/zJDS0YIwcJSIcfWRkTf1hEA= cloud.google.com/go/dataflow v0.13.0/go.mod h1:BWhSrIGmsMfuYj3J+nJ2Tw7tplRR6r28kvRiqCD3WlQ= +cloud.google.com/go/dataflow v0.16.0/go.mod h1:BWhSrIGmsMfuYj3J+nJ2Tw7tplRR6r28kvRiqCD3WlQ= cloud.google.com/go/dataform v0.16.0 h1:G4hlmGO8BFxM2NJ+9KDUOT2eIw+JE4cR24AjJ3RvIiI= cloud.google.com/go/dataform v0.16.0/go.mod h1:i1a0zkS751kvrY1IIPpUQZ77H5doxx7cs0AP3hnXTMk= +cloud.google.com/go/dataform v1.1.0/go.mod h1:ITyvb6cr2uQ5l4IEdHPrKmCVn6naTVoFbs0OqJJPtis= cloud.google.com/go/datafusion v1.10.0 h1:T+8c2aBThPJY33/yH0G9eH/PWqjB7GcouTl/4V9sD9k= cloud.google.com/go/datafusion v1.10.0/go.mod h1:MQdANs3I/4gitzY+mTBx27rrQyMiUg8uc2Z4TPLWWfc= +cloud.google.com/go/datafusion v1.13.0/go.mod h1:MQdANs3I/4gitzY+mTBx27rrQyMiUg8uc2Z4TPLWWfc= cloud.google.com/go/datalabeling v0.11.0 h1:CqtTYmgkEuSzHSJu6In30Rw3CmH12Vj30LQ8ZXf/pDM= cloud.google.com/go/datalabeling v0.11.0/go.mod h1:DYjvP4RhQ0332YgO22APYlBjCebb+SCaS0e2KApDq/Q= +cloud.google.com/go/datalabeling v0.14.0/go.mod h1:DYjvP4RhQ0332YgO22APYlBjCebb+SCaS0e2KApDq/Q= cloud.google.com/go/dataplex v1.31.0 h1:F3RgMT5RMW0nAtn9XuVdw83HcbPaSwzVVhgYQnoe7Zo= cloud.google.com/go/dataplex v1.31.0/go.mod h1:sOazL+Bs/PTxiMHQ5yBboBvEW9qPrpGogx3+RAgfIt8= +cloud.google.com/go/dataplex v1.35.0/go.mod h1:B7AFwXU1u3sp7FVQ3IFYnQguGTycJS2mF1voE0lLe1o= cloud.google.com/go/dataproc/v2 v2.18.0 h1:pcCfmceUwcFZsdw1vIx6x4tFfdaqLBZCdTFZdkse+ZQ= cloud.google.com/go/dataproc/v2 v2.18.0/go.mod h1:oARVSa38kAHvSuG+cozsrY2sE6UajGuvOOf9vS+ADHI= +cloud.google.com/go/dataproc/v2 v2.24.0/go.mod h1:sjjMzfmK3Ne/gTyA/H+uV0yIDix8+sXAJO44OgAo1q8= cloud.google.com/go/dataqna v0.11.0 h1:4NEQ+z6Qw9nl6FhNSQMiExiFKXyYeIhDsYp8mGQ8C84= cloud.google.com/go/dataqna v0.11.0/go.mod h1:XiVVFTOEJLBSvm3ILbyjXngGQYpjb/66MSksqz/56fs= +cloud.google.com/go/dataqna v0.13.0/go.mod h1:XiVVFTOEJLBSvm3ILbyjXngGQYpjb/66MSksqz/56fs= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.22.0 h1:FOyx2Ag6ibD2wFkz9S8EiNrmBugia8pQOfpyJxi2yqA= cloud.google.com/go/datastore v1.22.0/go.mod h1:aopSX+Whx0lHspWWBj+AjWt68/zjYsPfDe3LjWtqZg8= +cloud.google.com/go/datastore v1.24.0/go.mod h1:cEkLhU6Ti/gauQ7DFrUrG8bQjiMIxi++b5ePiThi5So= cloud.google.com/go/datastream v1.17.0 h1:VA0bz7qFVDDoUrTMm77q6cdhkrpeAqdKXvSoLh+Qq1E= cloud.google.com/go/datastream v1.17.0/go.mod h1:uoWTtfP20W8MXuV2DPcl5zqnVsxQ9QEmmBHX858oYTQ= +cloud.google.com/go/datastream v1.20.0/go.mod h1:uoWTtfP20W8MXuV2DPcl5zqnVsxQ9QEmmBHX858oYTQ= cloud.google.com/go/deploy v1.29.0 h1:fnhmOmf4XktCwEAQPJsBeggImukEApR4DJU9jzLyrjI= cloud.google.com/go/deploy v1.29.0/go.mod h1:lUG7maG/NkoTXmQ8G1mtcVymnbizfDJh6ER7vljVa/U= +cloud.google.com/go/deploy v1.32.0/go.mod h1:lUG7maG/NkoTXmQ8G1mtcVymnbizfDJh6ER7vljVa/U= cloud.google.com/go/dialogflow v1.79.0 h1:k3GOUD4Viu3IR/SW65y4NaFyFdsnAOvxoI+j3DzMq8U= cloud.google.com/go/dialogflow v1.79.0/go.mod h1:UtuiGOq9gAlTz9u4Vt+q1syMrx9ANQzTk+lC3WDdSOw= +cloud.google.com/go/dialogflow v1.83.0/go.mod h1:Rr0/YdrUAVQ+CPgt1yq8VpJmVSChfhz+uodykrBP1d0= cloud.google.com/go/dlp v1.31.0 h1:O0hRNTcMhUNrvHmIAUpk3GY6sOS5TUWmLHJdtH7YOog= cloud.google.com/go/dlp v1.31.0/go.mod h1:+haQd/n0QTv5BK7wZnCk2qctd5sfKL50jjh9E6N0d/Q= +cloud.google.com/go/dlp v1.36.0/go.mod h1:UW92dBhxvqkSKLct+Ril7Y9B4CanS5VLuDwlTGVA9VQ= cloud.google.com/go/documentai v1.45.0 h1:IY+gbb7B/J9XbPMTmmM8M4klPXjk4ZRFtpKbT5wuGMc= cloud.google.com/go/documentai v1.45.0/go.mod h1:mGjfbNf0cqCHKgxMZZV7frbfoF9T2hKkU1h88QyOy3c= +cloud.google.com/go/documentai v1.48.0/go.mod h1:mGjfbNf0cqCHKgxMZZV7frbfoF9T2hKkU1h88QyOy3c= cloud.google.com/go/domains v0.12.0 h1:3nohkVMXkqjaBAZIc9l7VfhwxtG/JYe36csp0mBVb/c= cloud.google.com/go/domains v0.12.0/go.mod h1:BjoSVNc+LVwoHMnE2fxTQNzGLSWWb6f3a8VAN6+VjVk= +cloud.google.com/go/domains v0.15.0/go.mod h1:BjoSVNc+LVwoHMnE2fxTQNzGLSWWb6f3a8VAN6+VjVk= cloud.google.com/go/edgecontainer v1.6.0 h1:1Zfa7DBtxc5u5vqurw9XEJ/UhghHXLH+aHLCSpyXTgs= cloud.google.com/go/edgecontainer v1.6.0/go.mod h1:mZmgXuMGTGI6RUUTXsOZa+F2rFF21v0JPnuX7LQEqBE= +cloud.google.com/go/edgecontainer v1.9.0/go.mod h1:mZmgXuMGTGI6RUUTXsOZa+F2rFF21v0JPnuX7LQEqBE= cloud.google.com/go/errorreporting v0.6.0 h1:52yZlQc72U1ygLfLUrulWshCxUjRVXsjv9agJQ5Lk2w= cloud.google.com/go/errorreporting v0.6.0/go.mod h1:POGEbtuDfvr8gjl9Je4fZGyAHOa5oF1FmlV9TeB3hvw= +cloud.google.com/go/errorreporting v0.9.0/go.mod h1:V7ojx7z76JITDZNGyDNkIIa9nNEkQzF6Yj+VHl2YF84= cloud.google.com/go/essentialcontacts v1.9.0 h1:ZqIG4XB1IaOJac+IgH6vriKBoJ9KmLLYToz6ohNRcFI= cloud.google.com/go/essentialcontacts v1.9.0/go.mod h1:W8fTL17jP6vmsPHQaCT5rOjWGohEssuqDUroxnjST0A= +cloud.google.com/go/essentialcontacts v1.12.0/go.mod h1:W8fTL17jP6vmsPHQaCT5rOjWGohEssuqDUroxnjST0A= cloud.google.com/go/eventarc v1.20.0 h1:5oJY8mvuM2a+zJ5V4M9FlyF27VOjuOEderzpPBWTyyQ= cloud.google.com/go/eventarc v1.20.0/go.mod h1:tIJL0hoWtZXVa5MjcAep/4xB+AXz4AbqQV14ogX5VwU= +cloud.google.com/go/eventarc v1.24.0/go.mod h1:pFJA4y1jNwTT1oq7BlV04G3oRr0PjQB4lQjJxmpFZdE= cloud.google.com/go/filestore v1.12.0 h1:9gyvG3IbbP2TW2Vx5lnxfjk/54bmWuw0aUZ0J3tDOHY= cloud.google.com/go/filestore v1.12.0/go.mod h1:oD+PvCWu4HqfEdNv65yk2XaLIiP7h4AuAH9Ua5YBRTM= +cloud.google.com/go/filestore v1.15.0/go.mod h1:oD+PvCWu4HqfEdNv65yk2XaLIiP7h4AuAH9Ua5YBRTM= cloud.google.com/go/firestore v1.21.0 h1:BhopUsx7kh6NFx77ccRsHhrtkbJUmDAxNY3uapWdjcM= cloud.google.com/go/firestore v1.21.0/go.mod h1:1xH6HNcnkf/gGyR8udd6pFO4Z7GWJSwLKQMx/u6UrP4= +cloud.google.com/go/firestore v1.23.0/go.mod h1:2EWfSUj+iqHckyH7uSjLI+lDgSAWJs9jL/uP45PnuQQ= cloud.google.com/go/functions v1.21.0 h1:16ViG3LpWYNgiEf1Ke6HD3x6kklvAjM22BxASxklXmo= cloud.google.com/go/functions v1.21.0/go.mod h1:t40GeqBAQNuqKlHCxmV/pxhyYJnImLcvRa3GBv4tAy0= +cloud.google.com/go/functions v1.24.0/go.mod h1:t40GeqBAQNuqKlHCxmV/pxhyYJnImLcvRa3GBv4tAy0= cloud.google.com/go/gkebackup v1.10.0 h1:FSFo87mslXBuzi0HoQd0uIsujFZAYpLwGIHlwNwY3E0= cloud.google.com/go/gkebackup v1.10.0/go.mod h1:D2MDbHW4V/uKCmS9TnT8hNKX2tPkE/pWp9nSm0TQ9hY= +cloud.google.com/go/gkebackup v1.13.0/go.mod h1:D2MDbHW4V/uKCmS9TnT8hNKX2tPkE/pWp9nSm0TQ9hY= cloud.google.com/go/gkeconnect v0.14.0 h1:1/OdUqvqpOUe22Id39LWMvIU7SEkGNIM3d/8jypvgJ8= cloud.google.com/go/gkeconnect v0.14.0/go.mod h1:5iWSBQzMIRLwUHUWVhxxcNK45ZPE8ntyBgE0MkavlqQ= +cloud.google.com/go/gkeconnect v1.0.0/go.mod h1:5iWSBQzMIRLwUHUWVhxxcNK45ZPE8ntyBgE0MkavlqQ= cloud.google.com/go/gkehub v0.18.0 h1:GW/XtnNhHG5lqghiRKD6Vi4Z5Uo2tb8bgA3WgBt1ZMY= cloud.google.com/go/gkehub v0.18.0/go.mod h1:xKePlMrI8LpKErzKMWdH/yQv+GDV60ypCNfTTdT+BN0= +cloud.google.com/go/gkehub v0.21.0/go.mod h1:xKePlMrI8LpKErzKMWdH/yQv+GDV60ypCNfTTdT+BN0= cloud.google.com/go/gkemulticloud v1.8.0 h1:Kad9qd7D4wRv/IYbLF2Np2N1wsgPO1GWpgjrVs/4wUA= cloud.google.com/go/gkemulticloud v1.8.0/go.mod h1:OtfHtgqOgDrXfcdFw8eUkCUI154Q51vvdqZYZV4c4qM= +cloud.google.com/go/gkemulticloud v1.11.0/go.mod h1:OtfHtgqOgDrXfcdFw8eUkCUI154Q51vvdqZYZV4c4qM= cloud.google.com/go/gsuiteaddons v1.9.0 h1:Mikqc/0DDiH1sr0lJXEAusEUPPdjtUvkqNasOkwkTuQ= cloud.google.com/go/gsuiteaddons v1.9.0/go.mod h1:rm/XT7wmwOFGn7jmWtVV65QmZCakzTbHLSojIC4Hskg= +cloud.google.com/go/gsuiteaddons v1.12.0/go.mod h1:rm/XT7wmwOFGn7jmWtVV65QmZCakzTbHLSojIC4Hskg= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= cloud.google.com/go/iap v1.14.0 h1:/VpfHrG9hHSkbNdI8MpgS0zlJpERdmJfuR4JmhD8jQA= cloud.google.com/go/iap v1.14.0/go.mod h1:b+r+yjrss2WmAEzNrQQjlEdD5E9B8c47mOF7XnqT+z0= +cloud.google.com/go/iap v1.17.0/go.mod h1:b+r+yjrss2WmAEzNrQQjlEdD5E9B8c47mOF7XnqT+z0= cloud.google.com/go/ids v1.7.0 h1:0mCN95Cemgur3U6n6TuX3phJ3HsGysG1JzykcOG1dbI= cloud.google.com/go/ids v1.7.0/go.mod h1:uCSFrXfCnRUKBl5PdE/ZqBNp1+vKSKPWpdYGa61WjpQ= +cloud.google.com/go/ids v1.10.0/go.mod h1:uCSFrXfCnRUKBl5PdE/ZqBNp1+vKSKPWpdYGa61WjpQ= cloud.google.com/go/iot v1.10.0 h1:lZtlw2YAXOOb8JKjC3jybu3/lnQwIZ8lQhgJhJ30hvE= cloud.google.com/go/iot v1.10.0/go.mod h1:62W4n2fe/Ct66NWJEfCB5suZ3XsL5Atx+MxFjScr+9s= +cloud.google.com/go/iot v1.13.0/go.mod h1:62W4n2fe/Ct66NWJEfCB5suZ3XsL5Atx+MxFjScr+9s= +cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58= cloud.google.com/go/kms v1.28.0 h1:TeBvmmVF3EtrXXC8hiBo0uV8ntX8QBbZeEQFnVpE70k= cloud.google.com/go/kms v1.28.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U= +cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U= cloud.google.com/go/language v1.16.0 h1:F/r+Fw6ZEkuw3kLz/hnkqtAv+U4PyciLzxpp9SsWKh8= cloud.google.com/go/language v1.16.0/go.mod h1:xSeiVB4UiA9wYmFy2GWjf1Mb1K3uR1Yi/80qoqTxH04= +cloud.google.com/go/language v1.18.0/go.mod h1:xSeiVB4UiA9wYmFy2GWjf1Mb1K3uR1Yi/80qoqTxH04= cloud.google.com/go/lifesciences v0.12.0 h1:du9NOdooXpqvQLEeUedIc5v6yVvH6qRr7xxz+9iI5II= cloud.google.com/go/lifesciences v0.12.0/go.mod h1:FwS+QkqPdVWl4SmKUCFozFvsTVWTLH13HCKcwR/MR9U= +cloud.google.com/go/lifesciences v0.15.0/go.mod h1:FwS+QkqPdVWl4SmKUCFozFvsTVWTLH13HCKcwR/MR9U= +cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= +cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= cloud.google.com/go/managedidentities v1.9.0 h1:304LxYj+TrAvjNFsnN+1I02+/GIGdC+7cHrnwEVqv3I= cloud.google.com/go/managedidentities v1.9.0/go.mod h1:rm72jf/v//0NG73VQNZM1JlV2E95uhJymmSXlgi6hMA= +cloud.google.com/go/managedidentities v1.12.0/go.mod h1:rm72jf/v//0NG73VQNZM1JlV2E95uhJymmSXlgi6hMA= cloud.google.com/go/maps v1.32.0 h1:TnqBkMtgPFl1SHKtlF7iArQHtYJL52V+OErIplesmXM= cloud.google.com/go/maps v1.32.0/go.mod h1:HH1V8tduMn+b9oRMCdl3vok98uvHco/wElZXyJQ/9kU= +cloud.google.com/go/maps v1.36.0/go.mod h1:Ly0sd/0G1MgKuWpGc2vCBjNZ+fc8iRHzcBWJqrw7Xao= cloud.google.com/go/mediatranslation v0.11.0 h1:w1yErnkEUAxt+1SDeXUi3ntGdJCYdh5nGMdRuh7m/3s= cloud.google.com/go/mediatranslation v0.11.0/go.mod h1:kjZrowuigFr+Bf1HM1TCtp1a3E3kfG1ovPK5VEuaNAQ= +cloud.google.com/go/mediatranslation v0.13.0/go.mod h1:kjZrowuigFr+Bf1HM1TCtp1a3E3kfG1ovPK5VEuaNAQ= cloud.google.com/go/memcache v1.13.0 h1:aeiNy8t+G5LebFlsNziEOkxNFAE9vl192KQY0A/ZNaE= cloud.google.com/go/memcache v1.13.0/go.mod h1:y/rXhJiieCF742K958dY29fSfM+Y3wh2thRmWspU2Dg= +cloud.google.com/go/memcache v1.16.0/go.mod h1:y/rXhJiieCF742K958dY29fSfM+Y3wh2thRmWspU2Dg= cloud.google.com/go/metastore v1.16.0 h1:msqpQlbkzJ99PvdrmxX75FQQiELjOshGrxprjKiGKug= cloud.google.com/go/metastore v1.16.0/go.mod h1:JGTjGdQ627m2ptDo86XsIKqzzZCk+GG41VEFD7ENsqs= +cloud.google.com/go/metastore v1.19.0/go.mod h1:JGTjGdQ627m2ptDo86XsIKqzzZCk+GG41VEFD7ENsqs= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= cloud.google.com/go/networkconnectivity v1.23.0 h1:KWR8jIWQ7cu8fp/oeedp4Rbd1G8yOlz66uY0Bo5stJM= cloud.google.com/go/networkconnectivity v1.23.0/go.mod h1:Uhzfk7NbiY6RNqV9XFvPWRji58+MkTYsTRfQ3EPtrGg= +cloud.google.com/go/networkconnectivity v1.26.0/go.mod h1:Uhzfk7NbiY6RNqV9XFvPWRji58+MkTYsTRfQ3EPtrGg= cloud.google.com/go/networkmanagement v1.25.0 h1:PBGV/4TV9SX5asGZAF7QiJq1sI1XfVk2cLynBNQhihs= cloud.google.com/go/networkmanagement v1.25.0/go.mod h1:2YogSU3sD7LvtmWntUAuGARbFQmy3A0En3LrJr69jkU= +cloud.google.com/go/networkmanagement v1.29.0/go.mod h1:lk9xX5YTlDyEc6zTp5ARD6MxfDwOJ6qw+MP6yU2Mluw= cloud.google.com/go/networksecurity v0.13.0 h1:TEOg4586EFhLypoPjE8LKdpOzwpOb0Jf/ZDs2JfRal4= cloud.google.com/go/networksecurity v0.13.0/go.mod h1:LMn10eRVf4K85PMF33yRoKAra7VhCOetxFcLDMh9A74= +cloud.google.com/go/networksecurity v0.18.0/go.mod h1:mcXDEKYoT2E3oKO6nh9vpz25DfmT8FYOC90Ua+Z/D8E= cloud.google.com/go/notebooks v1.14.0 h1:MBNcFjcuuQRC0a5qEy6G6RNhZR3GqTp4ax3XgwY9wzc= cloud.google.com/go/notebooks v1.14.0/go.mod h1:NScGIhfQCqLRIlVaUVbm595F6dhqiTl5XS1KaKgitKM= +cloud.google.com/go/notebooks v1.17.0/go.mod h1:NScGIhfQCqLRIlVaUVbm595F6dhqiTl5XS1KaKgitKM= cloud.google.com/go/optimization v1.9.0 h1:tHK6kYfZmzS3yvu40/vspI0C+NvMYD0hu7P/ZMb6cFg= cloud.google.com/go/optimization v1.9.0/go.mod h1:qCWskZMcynh0GBsUrCP6oPwwnUhbwg5UcXvVM9hzOD8= +cloud.google.com/go/optimization v1.11.0/go.mod h1:qCWskZMcynh0GBsUrCP6oPwwnUhbwg5UcXvVM9hzOD8= cloud.google.com/go/orchestration v1.13.0 h1:Zc7B3dhsb5HNbPc/62BRopMaPS34EQESkS2lwYeOOKg= cloud.google.com/go/orchestration v1.13.0/go.mod h1:H7MFVP8Z/dtml39nf43sWYPL/2o7J4tdSZAlJrBuqnQ= +cloud.google.com/go/orchestration v1.16.0/go.mod h1:H7MFVP8Z/dtml39nf43sWYPL/2o7J4tdSZAlJrBuqnQ= cloud.google.com/go/orgpolicy v1.17.0 h1:zpP13I+e3yOISCdGVf0ibXODieQzyr3wFVp1nM/o5Zo= cloud.google.com/go/orgpolicy v1.17.0/go.mod h1:9LHqEGx5P5dhansdKTNIEXpM+QbebAIOs66+HUID4aQ= +cloud.google.com/go/orgpolicy v1.20.0/go.mod h1:9LHqEGx5P5dhansdKTNIEXpM+QbebAIOs66+HUID4aQ= cloud.google.com/go/osconfig v1.18.0 h1:p9CgUYAOUXduHBOP7IlxujnqxJd+R//RvAa5itKokrk= cloud.google.com/go/osconfig v1.18.0/go.mod h1:BofnHqjjvu6lZQv/hqo2+rLCUiY4O6A9UYwwvVrSBjk= +cloud.google.com/go/osconfig v1.21.0/go.mod h1:BofnHqjjvu6lZQv/hqo2+rLCUiY4O6A9UYwwvVrSBjk= cloud.google.com/go/oslogin v1.16.0 h1:s/aH9HVBJgLWwqplshyEr6ppIug9MMboE8yTE1bvpVw= cloud.google.com/go/oslogin v1.16.0/go.mod h1:3Oa36T3781Mv+yCSVYlfasi7auHjfPFqvNOd1q92umc= +cloud.google.com/go/oslogin v1.18.0/go.mod h1:3Oa36T3781Mv+yCSVYlfasi7auHjfPFqvNOd1q92umc= cloud.google.com/go/phishingprotection v0.11.0 h1:OK26yBWBr1Ldd2E0UPSVIZfnjb88lCB0knlC9mJzxhM= cloud.google.com/go/phishingprotection v0.11.0/go.mod h1:2gyYqwNjePPEocXDkDve3EuJPaRqN/E7fp28K3arR0k= +cloud.google.com/go/phishingprotection v0.13.0/go.mod h1:2gyYqwNjePPEocXDkDve3EuJPaRqN/E7fp28K3arR0k= cloud.google.com/go/policytroubleshooter v1.13.0 h1:qct52TKyLYT51Xf5Atwv/elgNjJPvHI/gbkGEJz71eo= cloud.google.com/go/policytroubleshooter v1.13.0/go.mod h1:yNuROjN6h+2/TE2JOvBBJMjYIjC6j0UYHq8f2kVHlA4= +cloud.google.com/go/policytroubleshooter v1.15.0/go.mod h1:yNuROjN6h+2/TE2JOvBBJMjYIjC6j0UYHq8f2kVHlA4= cloud.google.com/go/privatecatalog v0.12.0 h1:nJi3YDgERB/Zs5Pmzs8G1fGDIxPMPnKGSOwVJK0yJeg= cloud.google.com/go/privatecatalog v0.12.0/go.mod h1:av2b5Rv+oG5ORxUqGlCAYO9s4pXjgc6q2qO9nkTcqT8= +cloud.google.com/go/privatecatalog v0.15.0/go.mod h1:av2b5Rv+oG5ORxUqGlCAYO9s4pXjgc6q2qO9nkTcqT8= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk= cloud.google.com/go/pubsub v1.50.2 h1:54Up97HnThdP4H8jjWJSSQ/mnYG2EKon7ZSNETRq0tM= cloud.google.com/go/pubsub v1.50.2/go.mod h1:jyCWeZdGFqd4mitSsBERnJcpqaHBsxQoPkNvjj4sp0w= +cloud.google.com/go/pubsub v1.50.4/go.mod h1:CBCG3lNP243mGNB8kTILs/Pd/gTV9a00kM1KjOtdxEk= +cloud.google.com/go/pubsub/v2 v2.4.0/go.mod h1:2lS/XQKq5qtOMs6kHBK+WX1ytUC36kLl2ig3zqsGUx8= cloud.google.com/go/pubsub/v2 v2.5.1 h1:+TwXJr78P9RrMV3S8lKHIhJo2E99jI7ta65e+ujJjts= cloud.google.com/go/pubsub/v2 v2.5.1/go.mod h1:Pd+qeabMX+576vQJhTN7TelE4k6kJh15dLU/ptOQ/UA= +cloud.google.com/go/pubsub/v2 v2.6.0/go.mod h1:4anqvV/w8Pcgu2tO0qr2XgsF3GXHowzryfQ5gOnVmWY= cloud.google.com/go/pubsublite v1.8.2 h1:jLQozsEVr+c6tOU13vDugtnaBSUy/PD5zK6mhm+uF1Y= cloud.google.com/go/pubsublite v1.8.2/go.mod h1:4r8GSa9NznExjuLPEJlF1VjOPOpgf3IT6k8x/YgaOPI= +cloud.google.com/go/pubsublite v1.9.0/go.mod h1:a2QASlHAcTR3sFs8Tz8TTZcyymyaTr67V3DFG3FkteI= cloud.google.com/go/recaptchaenterprise/v2 v2.23.0 h1:Xg76iWTekGXtmlDKVbujV6NVGPKm6Gn7hzr1Berd5ho= cloud.google.com/go/recaptchaenterprise/v2 v2.23.0/go.mod h1:+ntF70/j7qBa6G/pwmYA0mkBcDeTCXV6WDqUL7GObfs= +cloud.google.com/go/recaptchaenterprise/v2 v2.26.0/go.mod h1:+ntF70/j7qBa6G/pwmYA0mkBcDeTCXV6WDqUL7GObfs= cloud.google.com/go/recommendationengine v0.11.0 h1:1kifT41qy/r/XNITOgL2wOGA2EY0QV3gTgfXqvSjYRw= cloud.google.com/go/recommendationengine v0.11.0/go.mod h1:UP9cN46tDpZ/N57eDYIWeIRHjMOchtiIyjWjV0Dvr3k= +cloud.google.com/go/recommendationengine v0.14.0/go.mod h1:UP9cN46tDpZ/N57eDYIWeIRHjMOchtiIyjWjV0Dvr3k= cloud.google.com/go/recommender v1.15.0 h1:I82Iu70NcyHnTswn4IlJVnzLsp5zC9mCrMBDY0SP6bA= cloud.google.com/go/recommender v1.15.0/go.mod h1:INRBLfBQJCrgPqjBVFht4OjaFq/WhB/c5V1sqBOdX8g= +cloud.google.com/go/recommender v1.19.0/go.mod h1:LRh+1HJjLx2kDE3S65AIlG/lvwA0llEFWYPD/QtgoaU= cloud.google.com/go/redis v1.20.0 h1:pQYJxWxYLDx0r/W75ju5+vP5b5ePpZqpm4NWx04OYTQ= cloud.google.com/go/redis v1.20.0/go.mod h1:EUlUT24BAL6LsE1f/N9Bg3LhRCfH+LzwLGbst3KuZRw= +cloud.google.com/go/redis v1.23.0/go.mod h1:EUlUT24BAL6LsE1f/N9Bg3LhRCfH+LzwLGbst3KuZRw= cloud.google.com/go/resourcemanager v1.12.0 h1:sknKs2H0QW9uYPhnR7lD73HQ6v+qb4XXlxoCq0rbj9g= cloud.google.com/go/resourcemanager v1.12.0/go.mod h1:ve0VNxPoDU6XxDuEMCjkineb0YzXQXx3mOWwnNckGDE= +cloud.google.com/go/resourcemanager v1.15.0/go.mod h1:ve0VNxPoDU6XxDuEMCjkineb0YzXQXx3mOWwnNckGDE= cloud.google.com/go/resourcesettings v1.8.3 h1:13HOFU7v4cEvIHXSAQbinF4wp2Baybbq7q9FMctg1Ek= cloud.google.com/go/resourcesettings v1.8.3/go.mod h1:BzgfXFHIWOOmHe6ZV9+r3OWfpHJgnqXy8jqwx4zTMLw= cloud.google.com/go/retail v1.28.0 h1:TZdueueRw5KO0sec7m1TQIWHHIFvf2GhqOiWtCn4rPY= cloud.google.com/go/retail v1.28.0/go.mod h1:sfq/cT+gfSLuURf/mdVAw5n0pav3hxSP1rT8RfL7Qxk= +cloud.google.com/go/retail v1.31.0/go.mod h1:sfq/cT+gfSLuURf/mdVAw5n0pav3hxSP1rT8RfL7Qxk= cloud.google.com/go/run v1.18.0 h1:D4u23qVmL5H4nq0wjf8hvnkPqF/8Y3pN8QpvUG/LNys= cloud.google.com/go/run v1.18.0/go.mod h1:Z5wHbyFirI8XU48EPs5XJf/qmVm1SXZEhuS8EvZOuQU= +cloud.google.com/go/run v1.21.0/go.mod h1:Z5wHbyFirI8XU48EPs5XJf/qmVm1SXZEhuS8EvZOuQU= cloud.google.com/go/scheduler v1.13.0 h1:1yxpZj9nJZ5+ZAeLaB9AfvfZ3QN/uUktV5n4jst7Meo= cloud.google.com/go/scheduler v1.13.0/go.mod h1:0hsZg0MZJADyke1lutI0FHAYJR8Dtm8oIivXkmpACkA= +cloud.google.com/go/scheduler v1.16.0/go.mod h1:0hsZg0MZJADyke1lutI0FHAYJR8Dtm8oIivXkmpACkA= +cloud.google.com/go/secretmanager v1.16.0/go.mod h1://C/e4I8D26SDTz1f3TQcddhcmiC3rMEl0S1Cakvs3Q= cloud.google.com/go/secretmanager v1.18.0 h1:VA/ynUUapUF3+xrm0R1dMx8i21p2jfRAWFpokYPncKU= cloud.google.com/go/secretmanager v1.18.0/go.mod h1:9OmSuOeiiUicANglrbdKWSnT3gYkRcXuUQDk7dDW0zU= +cloud.google.com/go/secretmanager v1.20.0/go.mod h1:9OmSuOeiiUicANglrbdKWSnT3gYkRcXuUQDk7dDW0zU= cloud.google.com/go/security v1.21.0 h1:kNdALc/zRzp7JAoVangZS4/72PkjBBigol0DyVw21gc= cloud.google.com/go/security v1.21.0/go.mod h1:XaB3p0SE7v2bBitsLBb1hM6R8/oI/k/IujpXFJalFK0= +cloud.google.com/go/security v1.25.0/go.mod h1:xKPO7XBfUtgjfzPJeznEhI0gp/ZRJt/ZbWtuMYMeUDk= cloud.google.com/go/securitycenter v1.41.0 h1:UnBuTkbp9VS/yGzgNYi6K+m3MLJdGkt3pPOXRnARGIE= cloud.google.com/go/securitycenter v1.41.0/go.mod h1:7BMMbSTAddVfiE+HrC8tKS6SuRkyK7FRPlkpAZBRV3U= +cloud.google.com/go/securitycenter v1.44.0/go.mod h1:7BMMbSTAddVfiE+HrC8tKS6SuRkyK7FRPlkpAZBRV3U= cloud.google.com/go/servicedirectory v1.14.0 h1:cZGu+3RdGcCITK0bYqJ6Bz2sAyIJKjOT75gCGOCsmIc= cloud.google.com/go/servicedirectory v1.14.0/go.mod h1:CtgjXS1idj3s9Q6tB68021Rzk8Q6decV6+ldXC1BoBk= +cloud.google.com/go/servicedirectory v1.17.0/go.mod h1:CtgjXS1idj3s9Q6tB68021Rzk8Q6decV6+ldXC1BoBk= cloud.google.com/go/shell v1.10.0 h1:EH/c5tP+vIwzWKSW0U41C+39Pi5cEt529rrh1x1Eml4= cloud.google.com/go/shell v1.10.0/go.mod h1:TivWrVriy6xQ0wBjNJJridJgODZz8zXUEW2u48kynzY= +cloud.google.com/go/shell v1.12.0/go.mod h1:TivWrVriy6xQ0wBjNJJridJgODZz8zXUEW2u48kynzY= cloud.google.com/go/spanner v1.89.0 h1:r3h5Z5RA8JRPf3HCvA6ujNhREIMhPY+MrDL9mkY8jS0= cloud.google.com/go/spanner v1.89.0/go.mod h1:okNuxnp1wdPaVoM5M28Al2irKZLkHhZ2Z+DW6/ZJWGw= +cloud.google.com/go/spanner v1.92.0/go.mod h1:rCDPfWXNX0h+t484r+crCEaaMKbJfoWkHRDKU3H3+oY= cloud.google.com/go/speech v1.32.0 h1:GajYWX7B+BnHuXBxUBBaSk95uiY5AHxOo8TC63dYQz8= cloud.google.com/go/speech v1.32.0/go.mod h1:shnf33sZbGnQQZyek1fdLOR5rRKV6D3jsNqpqyijvj8= +cloud.google.com/go/speech v1.35.0/go.mod h1:shnf33sZbGnQQZyek1fdLOR5rRKV6D3jsNqpqyijvj8= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= cloud.google.com/go/storagetransfer v1.15.0 h1:ScTtw+rEawshRNfxusOrf0DqFCz/r2IcqnBooB2mBhw= cloud.google.com/go/storagetransfer v1.15.0/go.mod h1:AbGutEym/KNasoiDpSj/CYbigp5yhgosSgwlhGvQNs4= +cloud.google.com/go/storagetransfer v1.18.0/go.mod h1:AbGutEym/KNasoiDpSj/CYbigp5yhgosSgwlhGvQNs4= cloud.google.com/go/talent v1.10.0 h1:/BDAuMr5LwZFvIoeYGBGHQiM0bs5cX+cMHM2BK9kVtw= cloud.google.com/go/talent v1.10.0/go.mod h1:GSwli9V25WQdzeuJDJWH9TlQmA8lPFn7yKsxowdxW9Y= +cloud.google.com/go/talent v1.13.0/go.mod h1:GSwli9V25WQdzeuJDJWH9TlQmA8lPFn7yKsxowdxW9Y= cloud.google.com/go/texttospeech v1.18.0 h1:0pI/aUIR7/hpf6T1IhRb9Hbr1nrKddnPeFL1v9cQbH0= cloud.google.com/go/texttospeech v1.18.0/go.mod h1:p/UVJILAo/S5vsJaWZVdDRzNzA7wXIA+hTACvpMeOBk= +cloud.google.com/go/texttospeech v1.21.0/go.mod h1:p/UVJILAo/S5vsJaWZVdDRzNzA7wXIA+hTACvpMeOBk= cloud.google.com/go/tpu v1.10.0 h1:wYKfCRb1jG4JadajlM6Wr0dCN+3PiidLOYPuJn/4Yo8= cloud.google.com/go/tpu v1.10.0/go.mod h1:F5gT5BL22Dhsr05JLHdMjAjj+wcTn3Xtuu4jvq9yFug= +cloud.google.com/go/tpu v1.13.0/go.mod h1:F5gT5BL22Dhsr05JLHdMjAjj+wcTn3Xtuu4jvq9yFug= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= cloud.google.com/go/translate v1.14.0 h1:d0MD1Zj1N62knqL5uapNuYorjPAww86tOPrpnAYYdWM= cloud.google.com/go/translate v1.14.0/go.mod h1:XnU3rbMmUBZFThzPWJOSNy8V1N9ONnjdH/vWyPppcXU= +cloud.google.com/go/translate v1.17.0/go.mod h1:3mErnHTQBu9yeLiL35K0HBBuaM6Vk2fD/vyWFz790VU= cloud.google.com/go/video v1.29.0 h1:hNj9gsZAg3p47TeE+hlwxIQyaX20zoQVg2oLTd535Z8= cloud.google.com/go/video v1.29.0/go.mod h1:KxDL728ZzH+FJwtEb9XkiLTETW5bI37hTWbJiRYeXkk= +cloud.google.com/go/video v1.32.0/go.mod h1:KxDL728ZzH+FJwtEb9XkiLTETW5bI37hTWbJiRYeXkk= cloud.google.com/go/videointelligence v1.14.0 h1:Eh9I5RuROBy4zIG5QmPENhwBKK74hgOBpeSUVUKtjJ8= cloud.google.com/go/videointelligence v1.14.0/go.mod h1:mmX1JpIWzwozaigrdRNjikZc3aFLNHFKh+OFwAdfiW4= +cloud.google.com/go/videointelligence v1.16.0/go.mod h1:mmX1JpIWzwozaigrdRNjikZc3aFLNHFKh+OFwAdfiW4= cloud.google.com/go/vision/v2 v2.11.0 h1:6MdlpZsPOUpvdx54XsahTxWUMb81NZC3cDgU6izCJsw= cloud.google.com/go/vision/v2 v2.11.0/go.mod h1:ODlLCajJOq4t8thoi1uVvbnfIfix73HsYWhZuIveagQ= +cloud.google.com/go/vision/v2 v2.14.0/go.mod h1:ODlLCajJOq4t8thoi1uVvbnfIfix73HsYWhZuIveagQ= cloud.google.com/go/vmmigration v1.12.0 h1:2/OSRMTfyFx16G7YGTGjURFhobLJXPsx68wbTi3No6Y= cloud.google.com/go/vmmigration v1.12.0/go.mod h1:MP6mQ21ru1usBeCbl805Ioz0Fy+yf3qK2kUkhZ69QQY= +cloud.google.com/go/vmmigration v1.15.0/go.mod h1:MP6mQ21ru1usBeCbl805Ioz0Fy+yf3qK2kUkhZ69QQY= cloud.google.com/go/vmwareengine v1.5.0 h1:BlQwexZuVDgHKtNc51BxO6MM45UpNgg4eWeVEpRp3EU= cloud.google.com/go/vmwareengine v1.5.0/go.mod h1:e66l90IZhm1yQfYZv+YCWjSNSklQZCRmuEvKL8n3Ua0= +cloud.google.com/go/vmwareengine v1.8.0/go.mod h1:e66l90IZhm1yQfYZv+YCWjSNSklQZCRmuEvKL8n3Ua0= cloud.google.com/go/vpcaccess v1.10.0 h1:gY/MOPAWnw9QnPT1VSwdKtJZ9I276rP/+ZwrGDN7G0A= cloud.google.com/go/vpcaccess v1.10.0/go.mod h1:4Uus6E/9FYUtIrwBE1wJ1RosKwb02H6kEd9puJ02TL8= +cloud.google.com/go/vpcaccess v1.13.0/go.mod h1:4Uus6E/9FYUtIrwBE1wJ1RosKwb02H6kEd9puJ02TL8= cloud.google.com/go/webrisk v1.13.0 h1:ANgImW/EBfqPEQtYckW5+vWTgfBb2VvG6YImZCk6NUk= cloud.google.com/go/webrisk v1.13.0/go.mod h1:VIQw8smiaMOlget/xOk6niTkNJTiQc5skEmCuAksxJc= +cloud.google.com/go/webrisk v1.16.0/go.mod h1:VIQw8smiaMOlget/xOk6niTkNJTiQc5skEmCuAksxJc= cloud.google.com/go/websecurityscanner v1.9.0 h1:0QUAHT973x5ZJWQT9oFgUgGRmdcvtbANXmCx54SdqnQ= cloud.google.com/go/websecurityscanner v1.9.0/go.mod h1:cZSc9HqoFdccL1mqZtPIInOd4R8PBGwI20wdnrz6AO8= +cloud.google.com/go/websecurityscanner v1.12.0/go.mod h1:cZSc9HqoFdccL1mqZtPIInOd4R8PBGwI20wdnrz6AO8= cloud.google.com/go/workflows v1.16.0 h1:Ml3EHP/Bmnh8HwOlrJcCXHl3iiRPFCgbp7A1Cg6TGvw= cloud.google.com/go/workflows v1.16.0/go.mod h1:TWsrDGgsJy7xAJ07byzHhKKehEWItJG3BivEHVhGH5g= +cloud.google.com/go/workflows v1.19.0/go.mod h1:TWsrDGgsJy7xAJ07byzHhKKehEWItJG3BivEHVhGH5g= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/Azure/azure-amqp-common-go/v3 v3.2.3 h1:uDF62mbd9bypXWi19V1bN5NZEO84JqgmI5G73ibAmrk= github.com/Azure/azure-amqp-common-go/v3 v3.2.3/go.mod h1:7rPmbSfszeovxGfc5fSAXE4ehlXQZHpMja2OtxC2Tas= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.10.0 h1:kE5kpeiSqu4jcCQ/sWuyggMXJ/pT6oQ99+8hwPmyeJ0= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.10.0/go.mod h1:IAN3Z0DMtehoxoQQnfqg1891z1P7GNoDryKtFcAyMBI= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0/go.mod h1:Y2b/1clN4zsAoUd/pgNAQHjLDnTis/6ROkUfyob6psM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= github.com/Azure/go-amqp v1.5.0 h1:GRiQK1VhrNFbyx5VlmI6BsA1FCp27W5rb9kxOZScnTo= github.com/Azure/go-amqp v1.5.0/go.mod h1:vZAogwdrkbyK3Mla8m/CxSc/aKdnTZ4IbPxl51Y5WZE= +github.com/Azure/go-amqp v1.5.1/go.mod h1:vZAogwdrkbyK3Mla8m/CxSc/aKdnTZ4IbPxl51Y5WZE= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 h1:sR+/8Yb4slttB4vD+b9btVEnWgL3Q00OBTzVT8B9C0c= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v6 v6.2.0 h1:EpcZ6SR9n28BUGtNJSvlBqf90IpjeFr36Tizxhn/oME= github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4= github.com/GoogleCloudPlatform/cloudsql-proxy v1.37.10 h1:0jDrC5r/G+L/p715lTXEYRQ6sET0lzPxwTQlMTy9XfQ= github.com/GoogleCloudPlatform/cloudsql-proxy v1.37.10/go.mod h1:gij9WLu9mdiAFCM2EB+fwnbrVvc7cLr/klV1eEcFwbQ= +github.com/GoogleCloudPlatform/cloudsql-proxy v1.37.13/go.mod h1:6IMmxkLFo8kvCZbG0egv30L6YSgeZwxjH0WVRMGAacQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0 h1:5eCqTd9rTwMlE62z0xFdzPJ+3pji75hJrwq1jrCjo5w= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0/go.mod h1:4BcvJy7WxY8X2eX49z2VO1ByhO+CcQK8lKPCH/QlZvo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.31.0/go.mod h1:VLoD5cAsRQXsAFXpOZrrTGzbuMsntlspIZno4xor5Zg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.58.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0 h1:mQdVn6c25/S2MHfJTWGSK3NwGoI/w9Ad7tzyLWbjAQI= github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.54.0/go.mod h1:8W5IW/jylevlBQKSWkh5ZMP2oy7yT9Pnfug6Y6W/9D8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.55.0/go.mod h1:8W5IW/jylevlBQKSWkh5ZMP2oy7yT9Pnfug6Y6W/9D8= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3 h1:Qbeh12Vq6BxURXT1qZBRHsDxeURB8ztcL6f3EXSGeHk= github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= @@ -254,29 +448,62 @@ github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 h1:KkH3I3sJuOLP github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06/go.mod h1:7erjKLwalezA0k99cWs5L11HWOAPNjdUZ6RxH1BXbbM= github.com/XSAM/otelsql v0.40.0 h1:8jaiQ6KcoEXF46fBmPEqb+pp29w2xjWfuXjZXTXBjaA= github.com/XSAM/otelsql v0.40.0/go.mod h1:/7F+1XKt3/sTlYtwKtkHQ5Gzoom+EerXmD1VdnTqfB4= +github.com/XSAM/otelsql v0.41.0/go.mod h1:NMQT0PiKoFILp9QgjQz+D5mvW+9mT0suR7OejqrtMaM= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= +github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= +github.com/aws/aws-sdk-go-v2/config v1.32.20/go.mod h1:PuwEpciweIXGULWeOeSTXtSbH4CW9mWdWrhdCKQI1sM= +github.com/aws/aws-sdk-go-v2/credentials v1.19.19/go.mod h1:7y63L1kGzeoDlJaQ3Z578KrnmfBut96JjvJUzGwR+YE= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.20.26 h1:khdgzmb6QKweEAnjBhg/Ikcn0VguyOyg0gMSVyK8ddI= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.20.26/go.mod h1:P5lKM3+laQ9v0KAOLhxOkClj4UbBwXJ2QcQc2sKSOYo= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.20.35/go.mod h1:ypTMB9nZhpqfMeRVesGj4dEknIg0YS+aXGtLMidw/Ek= github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.8.26 h1:tHmqvPzeKaCIlD4kdQVP+7v5GdHHtu9olhvweho5zr8= github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.8.26/go.mod h1:qEScmjwld3lw08e6CIWbPIgfcCKBdw2htqqBtSOSINQ= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.8.35/go.mod h1:SomvXQRUKYBML53k4LqIgszKJKz8TdUwi/Zwig7JhfU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25/go.mod h1:9FDWUothyr5RCRAHc45XOiVCzUR8n/IhCYX+uVqw6vk= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.14 h1:gKXU53GYsPuYgkdTdMHh6vNdcbIgoxFQLQGjg+iRG+k= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.14/go.mod h1:jyoemRAktfCyZR9bTb5gT3kn/Vj2KwYDm0Pev5TsmEQ= +github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.20/go.mod h1:wtCkeFPPKHdxFPrZGkdT5tKR4boa3GvW54sYdGNWPHg= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25/go.mod h1:G6kntsA2GorAxDPbap6xgB2F+amSLUF8GJTi7PUoX44= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25/go.mod h1:cKf+D+NMDK1LndD7BowHbBZPgR9V0/5HubH0PFWvA+c= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26/go.mod h1:dY4MRzXEizrD4hqtpKvWVGPX7QleSGGVY+EBolo1RmM= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.53.2 h1:+/HEQj1fQGr17AQ0fAKpefDHw2hxQ3f0q96hY39J8Ao= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.53.2/go.mod h1:bz4cZH7uK5fLxQbj7hL4MFDL+pjReC9en/nM2Wfwxsk= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.56.2/go.mod h1:dLREOeW66eVaaGIOi2ZlLHDgkR3nuJ02rd00j0YSlBE= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.32.6 h1:m8Odxvyy7nirivpiI0VLwqd3lUkVRgeKPQgdJ9YhvcQ= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.32.6/go.mod h1:r2DJVcbGPv7oJGoPICCQJ+4ci5oSGjdXtdscnJIQBfk= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.32.13/go.mod h1:D5up2/CMSP4sF8ESBWla6gJvIMySJi8dYYAaED4oTCc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10/go.mod h1:a57l7Hwh+FWI+we50g5NPJHYUKeJKfXbc4w8SyXu8Ig= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.14 h1:3exo28cClRTVnxdj/LULxkESZSSv74RUIjZ7tfHXfWQ= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.14/go.mod h1:yLon9pByjyB6JZq5IAmwnjE3ObIhD0QibfRWH7tUhLU= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.20/go.mod h1:ihZMtPTKoX/ugQRHbui6zNdSgVYN1KY2Dgwb2d3hXlc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25/go.mod h1:0yAbjPfd64gG7mj85RW+fMEYdfBgCRZw8g/oWcL1pjc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM= +github.com/aws/aws-sdk-go-v2/service/kms v1.50.3/go.mod h1:/iSgiUor15ZuxFGQSTf3lA2FmKxFsQoc2tADOarQBSw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.40.2 h1:p0tPbc1uXSAYs9ACiVB9WxlV6AY5TBVNadXdvGrtOHA= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.40.2/go.mod h1:c6Vg0BRiU7v0MVhHupw90RyL120QBwAMLbDCzptGeMk= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4/go.mod h1:cxiXDhEzIq7Xx1BtmC4lGBK3SwAZ79+EUWiKawYHo14= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.1/go.mod h1:vUtyoSj0OPji3kjIVSc/GlKuWEiL33f/WFxl6dmpy/A= github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4= github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks= +github.com/aws/aws-sdk-go-v2/service/sns v1.39.14/go.mod h1:NKVY7DER6VXHkt2I/ycmHakALNboi3Rqwt4eEf/1Cnk= github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc= github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I= +github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24/go.mod h1:Ql9ziDutk8ERAN9HMaYANCW3lop451ppebkxEJMLCTM= github.com/aws/aws-sdk-go-v2/service/ssm v1.67.4 h1:pOwUUY5FzKUsxtxGR6qsczZP7MuZMVlMbAOPQOcmJlo= github.com/aws/aws-sdk-go-v2/service/ssm v1.67.4/go.mod h1:+nlWvcgDPQ56mChEBzTC0puAMck+4onOFaHg5cE+Lgg= +github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3/go.mod h1:rcRkKbUJ2437WuXdq9fbj+MjTudYWzY9Ct8kiBbN8a8= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.19/go.mod h1:3gt5WJArFooNmyLONS+h/R4J+o86II8du38IgCwj9dE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2/go.mod h1:hU6fqB3OJA6/ePheD47LQnxvjYk6br6PtQxs+Q9ojvk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.3/go.mod h1:ULe4HCzfKPiR6R3HEurE3b1upEkuk8AkMrOKtaOxKO8= +github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= @@ -288,26 +515,50 @@ github.com/calypr/syfon/apigen v0.1.4/go.mod h1:2oqNkseovHqbU8kYu4nU+w5VrwhzAj6K github.com/calypr/syfon/client v0.1.5/go.mod h1:LA+07PI2ztRlLND0lxOSjgkBm62CT6+02Cc/iBjVPaY= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0Hw= github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -320,53 +571,144 @@ github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1 github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gofiber/schema v1.8.0/go.mod h1:lmbXPQ8hvzXSLkdS2DS7pb4kpunC2Roh7Sj3HMjGfzA= +github.com/gofiber/utils/v2 v2.1.1/go.mod h1:DdOgEVwQTi8cou/AKWPqhXOR4fHGRVhA/rEWL3IXG7Q= +github.com/gofiber/utils/v2 v2.1.2/go.mod h1:DdOgEVwQTi8cou/AKWPqhXOR4fHGRVhA/rEWL3IXG7Q= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 h1:EcQR3gusLHN46TAD+G+EbaaqJArt5vHhNpXAa12PQf4= github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/gomarkdown/markdown v0.0.0-20240328165702-4d01890c35c0/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td8= github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d h1:c93kUJDtVAXFEhsCh5jSxyOJmFHuzcihnslQiX8Urwo= github.com/kataras/blocks v0.0.7 h1:cF3RDY/vxnSRezc7vLFlQFTYXG/yAr1o7WImJuZbzC4= github.com/kataras/blocks v0.0.7/go.mod h1:UJIU97CluDo0f+zEjbnbkeMRlvYORtmc1304EeyXf4I= +github.com/kataras/blocks v0.0.8/go.mod h1:9Jm5zx6BB+06NwA+OhTbHW1xkMOYxahnqTN5DveZ2Yg= github.com/kataras/golog v0.1.9 h1:vLvSDpP7kihFGKFAvBSofYo7qZNULYSHOH2D7rPTKJk= github.com/kataras/golog v0.1.9/go.mod h1:jlpk/bOaYCyqDqH18pgDHdaJab72yBE6i0O3s30hpWY= +github.com/kataras/golog v0.1.11/go.mod h1:mAkt1vbPowFUuUGvexyQ5NFW6djEgGyxQBIARJ0AH4A= github.com/kataras/iris/v12 v12.2.6-0.20230908161203-24ba4e8933b9 h1:Vx8kDVhO2qepK8w44lBtp+RzN3ld743i+LYPzODJSpQ= github.com/kataras/iris/v12 v12.2.6-0.20230908161203-24ba4e8933b9/go.mod h1:ldkoR3iXABBeqlTibQ3MYaviA1oSlPvim6f55biwBh4= +github.com/kataras/iris/v12 v12.2.11/go.mod h1:uMAeX8OqG9vqdhyrIPv8Lajo/wXTtAF43wchP9WHt2w= github.com/kataras/pio v0.0.12 h1:o52SfVYauS3J5X08fNjlGS5arXHjW/ItLkyLcKjoH6w= github.com/kataras/pio v0.0.12/go.mod h1:ODK/8XBhhQ5WqrAhKy+9lTPS7sBf6O3KcLhc9klfRcY= +github.com/kataras/pio v0.0.13/go.mod h1:k3HNuSw+eJ8Pm2lA4lRhg3DiCjVgHlP8hmXApSej3oM= github.com/kataras/sitemap v0.0.6 h1:w71CRMMKYMJh6LR2wTgnk5hSgjVNB9KL60n5e2KHvLY= github.com/kataras/sitemap v0.0.6/go.mod h1:dW4dOCNs896OR1HmG+dMLdT7JjDk7mYBzoIRwuj5jA4= github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA= github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46 h1:veS9QfglfvqAw2e+eeNT/SbGySq8ajECXJ9e4fPoLhY= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labstack/echo/v4 v4.15.1 h1:S9keusg26gZpjMmPqB5hOEvNKnmd1lNmcHrbbH2lnFs= @@ -375,13 +717,16 @@ github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0 github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.12.0/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/lyft/protoc-gen-star/v2 v2.0.4 h1:JDlNKttNIRd68AAIychs0AqEpO8/I/WYi01OQ7Raw6Q= github.com/lyft/protoc-gen-star/v2 v2.0.4/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= +github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/moby/sys/mount v0.3.4 h1:yn5jq4STPztkkzSKpZkLcmjue+bZJ0u2AuQY1iNI1Ww= github.com/moby/sys/mount v0.3.4/go.mod h1:KcQJMbQdJHPlq5lcYT+/CjatWM4PuxKe+XLSVS4J6Os= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= @@ -396,10 +741,18 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU= github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -407,12 +760,16 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6Ng github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= +github.com/shamaton/msgpack/v3 v3.1.2/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc= +github.com/shamaton/msgpack/v3 v3.2.0/go.mod h1:sgBYvEiyz8JR1NC3yGRoPVME9xXovpnh3l/plW1nfRo= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.1/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad h1:fiWzISvDn0Csy5H0iwgAuJGQTUpVfEMJJd4nRFXogbc= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -420,38 +777,70 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tdewolff/minify/v2 v2.12.9 h1:dvn5MtmuQ/DFMwqf5j8QhEVpPX6fi3WGImhv8RUB4zA= github.com/tdewolff/minify/v2 v2.12.9/go.mod h1:qOqdlDfL+7v0/fyymB+OP497nIxJYSvX4MQWA8OoiXU= +github.com/tdewolff/minify/v2 v2.20.19/go.mod h1:ulkFoeAVWMLEyjuDz1ZIWOA31g5aWOawCFRp9R/MudM= github.com/tdewolff/parse/v2 v2.6.8 h1:mhNZXYCx//xG7Yq2e/kVLNZw4YfYmeHbhx+Zc0OvFMA= github.com/tdewolff/parse/v2 v2.6.8/go.mod h1:XHDhaU6IBgsryfdnpzUXBlT6leW/l25yrFBTEb4eIyM= +github.com/tdewolff/parse/v2 v2.7.12/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA= github.com/tdewolff/test v1.0.9/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= +github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA= github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/detectors/aws/ec2 v1.38.0 h1:gSqtaXUzONG+J3PutTnKpLVLpKLsX+FztJwxSFQ15PU= go.opentelemetry.io/contrib/detectors/aws/ec2 v1.38.0/go.mod h1:AqLDNPbKVFwdXy2/Xu2EYElVHO7ghhbEhKCCWymjpMI= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/contrib/propagators/aws v1.38.0 h1:eRZ7asSbLc5dH7+TBzL6hFKb1dabz0IV51uUUwYRZts= go.opentelemetry.io/contrib/propagators/aws v1.38.0/go.mod h1:wXqc9NTGcXapBExHBDVLEZlByu6quiQL8w7Tjgv8TCg= +go.opentelemetry.io/contrib/propagators/aws v1.42.0/go.mod h1:Jzw9hZHtxdpCN7x8S17UH59X/EiFivp6VXLs9bdM1OQ= +go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q= +go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -463,58 +852,444 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.272.0/go.mod h1:wKjowi5LNJc5qarNvDCvNQBn3rVK8nSy6jg2SwRwzIA= +google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= +google.golang.org/api v0.280.0/go.mod h1:oGKmPZRDoD3vdkf6MA7F4VNkR1rxCiuaPSkhsf3EolU= +google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:RRHjglSYABVCWpQ7USCpdfhcd9t4PkajvVwyynZizTc= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:1dCETSCY2YKZNXQE3h4fun3TYwF5p8jejRKZgfWAgAY= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800/go.mod h1:FPk7EXUKMtImne7AmknoYjT4QXqKIzzRbeQIXzLk6fQ= google.golang.org/genproto/googleapis/bytestream v0.0.0-20260319201613-d00831a3d3e7 h1:6jJ8xNMxB3mmH3HsxEMPjJU+WeFOiwBJ+cLm60OvSZs= google.golang.org/genproto/googleapis/bytestream v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:6TABGosqSqU2l1+fJ3jdvOYPPVryeKybxYF0cCZkTBE= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20260630182238-925bb5da69e7/go.mod h1:6TABGosqSqU2l1+fJ3jdvOYPPVryeKybxYF0cCZkTBE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401001100-f93e5f3e9f0f/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds= google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/api/docs/swagger.go b/internal/api/docs/swagger.go index acc36332..986af9d0 100644 --- a/internal/api/docs/swagger.go +++ b/internal/api/docs/swagger.go @@ -104,82 +104,6 @@ func handleInternalOpenAPISpec(w http.ResponseWriter, r *http.Request) { } } -func findOpenAPISpecPath() (string, bool) { - candidates := []string{ - "apigen/openapi/openapi.yaml", - filepath.Join(filepath.Dir(os.Args[0]), "apigen", "openapi", "openapi.yaml"), - } - - if _, thisFile, _, ok := runtime.Caller(0); ok { - repoRoot := filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..", "..")) - candidates = append(candidates, filepath.Join(repoRoot, "apigen", "openapi", "openapi.yaml")) - } - - for _, path := range candidates { - if _, err := os.Stat(path); err == nil { - return path, true - } - } - return "", false -} - -func findLFSOpenAPISpecPath() (string, bool) { - candidates := []string{ - "apigen/openapi/lfs.openapi.yaml", - filepath.Join(filepath.Dir(os.Args[0]), "apigen", "openapi", "lfs.openapi.yaml"), - } - - if _, thisFile, _, ok := runtime.Caller(0); ok { - repoRoot := filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..", "..")) - candidates = append(candidates, filepath.Join(repoRoot, "apigen", "openapi", "lfs.openapi.yaml")) - } - - for _, path := range candidates { - if _, err := os.Stat(path); err == nil { - return path, true - } - } - return "", false -} - -func findCompatOpenAPISpecPath() (string, bool) { - candidates := []string{ - "apigen/openapi/compat.openapi.yaml", - filepath.Join(filepath.Dir(os.Args[0]), "apigen", "openapi", "compat.openapi.yaml"), - } - - if _, thisFile, _, ok := runtime.Caller(0); ok { - repoRoot := filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..", "..")) - candidates = append(candidates, filepath.Join(repoRoot, "apigen", "openapi", "compat.openapi.yaml")) - } - - for _, path := range candidates { - if _, err := os.Stat(path); err == nil { - return path, true - } - } - return "", false -} - -func findBucketOpenAPISpecPath() (string, bool) { - candidates := []string{ - "apigen/openapi/bucket.openapi.yaml", - filepath.Join(filepath.Dir(os.Args[0]), "apigen", "openapi", "bucket.openapi.yaml"), - } - - if _, thisFile, _, ok := runtime.Caller(0); ok { - repoRoot := filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..", "..")) - candidates = append(candidates, filepath.Join(repoRoot, "apigen", "openapi", "bucket.openapi.yaml")) - } - - for _, path := range candidates { - if _, err := os.Stat(path); err == nil { - return path, true - } - } - return "", false -} - func findNamedOpenAPISpecPath(fileName string) (string, bool) { candidates := []string{ filepath.Join("apigen", "openapi", fileName), @@ -237,18 +161,6 @@ func buildMergedOpenAPISpec() ([]byte, error) { return out, nil } -func loadSpecYAML(path string) (map[string]interface{}, error) { - raw, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var doc map[string]interface{} - if err := yaml.Unmarshal(raw, &doc); err != nil { - return nil, err - } - return doc, nil -} - func loadSpecYAMLByName(fileName string) (map[string]interface{}, error) { raw, err := loadSpecBytesByName(fileName) if err != nil { diff --git a/internal/api/drsapi/mutations.go b/internal/api/drsapi/mutations.go index 657a1f6c..db31be4d 100644 --- a/internal/api/drsapi/mutations.go +++ b/internal/api/drsapi/mutations.go @@ -134,7 +134,10 @@ func handleBulkDeleteObjectsFiber(om *core.ObjectManager) fiber.Handler { ids = append(ids, id) } - if err := om.BulkDeleteObjects(c.Context(), ids); err != nil { + opts := core.DeleteOptions{ + DeleteStorageData: body.DeleteStorageData != nil && *body.DeleteStorageData, + } + if err := om.BulkDeleteObjectsWithOptions(c.Context(), ids, opts); err != nil { return apiutil.HandleError(c, err) } return c.SendStatus(fiber.StatusNoContent) diff --git a/internal/api/internaldrs/bucket_helpers.go b/internal/api/internaldrs/bucket_helpers.go index 688ba24c..9d93e32f 100644 --- a/internal/api/internaldrs/bucket_helpers.go +++ b/internal/api/internaldrs/bucket_helpers.go @@ -46,16 +46,6 @@ func serviceResourceAllowed(ctx context.Context, resource, service string, metho return authz.HasAnyServiceMethodAccess(ctx, []string{resource}, service, methods...) } -func allowedBucketsForScopes(ctx context.Context, scopes []models.BucketScope, methods ...string) map[string]bool { - allowed := make(map[string]bool) - for _, scope := range scopes { - if bucketScopeAllowed(ctx, scope, methods...) { - allowed[scope.Bucket] = true - } - } - return allowed -} - func bucketsAllowedByNames(ctx context.Context, scopes []models.BucketScope, bucket string, methods ...string) bool { for _, scope := range scopes { if scope.Bucket != bucket { diff --git a/internal/api/internaldrs/bucket_policy_test.go b/internal/api/internaldrs/bucket_policy_test.go index 1db28fe7..94449f08 100644 --- a/internal/api/internaldrs/bucket_policy_test.go +++ b/internal/api/internaldrs/bucket_policy_test.go @@ -42,10 +42,6 @@ func TestBucketPolicyHelpers(t *testing.T) { resource: {"read": true}, }) - allowed := allowedBucketsForScopes(ctx, []models.BucketScope{scope}, "read") - if !allowed["bucket-a"] { - t.Fatal("expected bucket to be allowed") - } if !bucketsAllowedByNames(ctx, []models.BucketScope{scope}, "bucket-a", "read") { t.Fatal("expected bucket name match to be allowed") } diff --git a/internal/api/internaldrs/bucket_routes_test.go b/internal/api/internaldrs/bucket_routes_test.go index 1a75ab45..fea0a263 100644 --- a/internal/api/internaldrs/bucket_routes_test.go +++ b/internal/api/internaldrs/bucket_routes_test.go @@ -199,6 +199,66 @@ func TestHandleInternalPutBucket_RejectsInvalidGeneratedPayloads(t *testing.T) { } } +func TestHandleInternalPutBucket_ReusesExistingPhysicalBucketCredential(t *testing.T) { + mockDB := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cbdscollab_1c102e76761b": { + CredentialID: "cbdscollab_1c102e76761b", + Bucket: "cbdscollab", + Provider: "s3", + Region: "us-east-1", + AccessKey: "old-ak", + SecretKey: "old-sk", + Endpoint: "https://old-endpoint.example.org", + }, + }, + } + provider := "s3" + region := "us-east-1" + accessKey := "new-ak" + secretKey := "new-sk" + endpoint := "https://fortera-object.ohsu.edu" + path := "s3://cbdscollab/Lab_Projects/Embedding_Rotation" + putBody, _ := json.Marshal(bucketapi.PutBucketRequest{ + Bucket: "cbdscollab", + Provider: &provider, + Region: ®ion, + AccessKey: &accessKey, + SecretKey: &secretKey, + Endpoint: &endpoint, + Organization: "Ellrott_Lab", + ProjectId: "embedding_rotation", + Path: &path, + }) + req := httptest.NewRequest(http.MethodPut, "/data/buckets", bytes.NewBuffer(putBody)) + req = req.WithContext(dataTestAuthContext(req.Context(), "gen3", true, map[string]map[string]bool{ + "/programs/Ellrott_Lab/projects/embedding_rotation": {"create": true, "update": true}, + })) + + rr := doInternalDRSTestRequest(req, core.NewObjectManager(mockDB, &testutils.MockUrlManager{})) + if rr.Code != http.StatusCreated { + t.Fatalf("expected 201, got %d body=%s", rr.Code, rr.Body.String()) + } + + updated, ok := mockDB.Credentials["cbdscollab_1c102e76761b"] + if !ok { + t.Fatalf("expected existing credential id to be preserved, got %+v", mockDB.Credentials) + } + if updated.AccessKey != accessKey || updated.SecretKey != secretKey || updated.Endpoint != endpoint { + t.Fatalf("expected existing credential to be updated, got %+v", updated) + } + if _, exists := mockDB.Credentials[common.DeriveCredentialID("cbdscollab", provider, region, endpoint, accessKey)]; exists { + t.Fatalf("expected no replacement credential to be created, got %+v", mockDB.Credentials) + } + scope, ok := mockDB.BucketScopes["Ellrott_Lab|embedding_rotation"] + if !ok { + t.Fatalf("expected bucket scope to be created") + } + if scope.CredentialID != "cbdscollab_1c102e76761b" || scope.Bucket != "cbdscollab" || scope.PathPrefix != "Lab_Projects/Embedding_Rotation" { + t.Fatalf("unexpected scope saved: %+v", scope) + } +} + func TestRegisterInternalRoutes_Smoke(t *testing.T) { app := fiber.New() om := core.NewObjectManager(&testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}, Credentials: map[string]models.S3Credential{"b1": {Bucket: "b1"}}}, &testutils.MockUrlManager{}) @@ -316,6 +376,37 @@ func TestHandleInternalListBucketScopes_FiltersUnauthorizedScopesOnSharedBucket( } } +func TestHandleInternalListBucketScopes_RendersRootScopeAsBucketURL(t *testing.T) { + mockDB := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "gdcdata": {CredentialID: "gdcdata", Bucket: "gdcdata", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "gdc|": {Organization: "gdc", ProjectID: "", CredentialID: "gdcdata", Bucket: "gdcdata", PathPrefix: ""}, + }, + } + req := httptest.NewRequest(http.MethodGet, "/data/buckets/gdcdata/scopes", nil) + req = req.WithContext(dataTestAuthContext(req.Context(), "gen3", true, map[string]map[string]bool{ + "/programs/gdc": {"read": true}, + })) + + rr := doInternalDRSTestRequest(req, core.NewObjectManager(mockDB, &testutils.MockUrlManager{})) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + + var resp []bucketapi.BucketScopeResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp) != 1 { + t.Fatalf("expected 1 scope, got %d: %+v", len(resp), resp) + } + if resp[0].Path == nil || *resp[0].Path != "s3://gdcdata" { + t.Fatalf("expected root scope path s3://gdcdata, got %+v", resp[0].Path) + } +} + func TestHandleInternalListBucketScopes_RequiresGen3Auth(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/data/buckets/bucket-a/scopes", nil) req = req.WithContext(dataTestAuthContext(req.Context(), "gen3", false, nil)) diff --git a/internal/api/internaldrs/buckets.go b/internal/api/internaldrs/buckets.go index 7aa600a5..01b4d08f 100644 --- a/internal/api/internaldrs/buckets.go +++ b/internal/api/internaldrs/buckets.go @@ -93,15 +93,15 @@ func handleInternalPutBucketFiber(c fiber.Ctx, om *core.ObjectManager) error { return apiutil.Reject(c, fiber.StatusBadRequest, err.Error()) } - hasCredentialMaterial := accessKey != "" || secretKey != "" || endpoint != "" || region != "" || rawProvider != "" credentialID := "" var existingCred *models.S3Credential var credErr error - if !hasCredentialMaterial { - existingCred, credErr = om.GetS3Credential(c.Context(), req.Bucket) - if credErr == nil && existingCred != nil { - credentialID = existingCred.CredentialID - } + existingCred, credErr = om.GetS3Credential(c.Context(), req.Bucket) + if credErr != nil && !isCredentialNotFoundError(credErr) { + return apiutil.HandleError(c, credErr) + } + if credErr == nil && existingCred != nil { + credentialID = existingCred.CredentialID } if credentialID == "" { credentialID = common.DeriveCredentialID(req.Bucket, bucketProvider, region, endpoint, accessKey) @@ -110,6 +110,10 @@ func handleInternalPutBucketFiber(c fiber.Ctx, om *core.ObjectManager) error { existingCred, credErr = om.GetS3Credential(c.Context(), credentialID) } hasExistingCred := credErr == nil && existingCred != nil + if hasExistingCred && rawProvider == "" { + bucketProvider = existingCred.Provider + req.Provider = common.Ptr(bucketProvider) + } scopeOnly := hasExistingCred && accessKey == "" && secretKey == "" && @@ -174,6 +178,10 @@ func handleInternalPutBucketFiber(c fiber.Ctx, om *core.ObjectManager) error { return c.SendStatus(fiber.StatusCreated) } +func isCredentialNotFoundError(err error) bool { + return err != nil && strings.EqualFold(strings.TrimSpace(err.Error()), "credential not found") +} + func handleInternalDeleteBucketFiber(c fiber.Ctx, om *core.ObjectManager) error { credentialID := strings.TrimSpace(c.Params("bucket")) if credentialID == "" { @@ -353,13 +361,13 @@ func handleInternalListBucketScopesFiber(c fiber.Ctx, om *core.ObjectManager) er if !bucketScopeAllowed(c.Context(), scope, "read") { continue } - path := "" - if scope.PathPrefix != "" { - scheme := "s3" - if cred, err := om.GetS3Credential(c.Context(), scope.CredentialID); err == nil && cred != nil { - scheme = common.ProviderToScheme(cred.Provider) - } - path = fmt.Sprintf("%s://%s/%s", scheme, scope.Bucket, scope.PathPrefix) + scheme := "s3" + if cred, err := om.GetS3Credential(c.Context(), scope.CredentialID); err == nil && cred != nil { + scheme = common.ProviderToScheme(cred.Provider) + } + path := fmt.Sprintf("%s://%s", scheme, scope.Bucket) + if strings.TrimSpace(scope.PathPrefix) != "" { + path = fmt.Sprintf("%s/%s", path, strings.Trim(strings.TrimSpace(scope.PathPrefix), "/")) } result = append(result, bucketapi.BucketScopeResponse{ Organization: scope.Organization, diff --git a/internal/api/internaldrs/index.go b/internal/api/internaldrs/index.go index 9fd3bb65..500b5250 100644 --- a/internal/api/internaldrs/index.go +++ b/internal/api/internaldrs/index.go @@ -1,15 +1,14 @@ package internaldrs import ( + "encoding/hex" "fmt" - "net/http" - "sort" + "log" "strconv" "strings" "time" "github.com/calypr/syfon/apigen/server/internalapi" - sycommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/api/apiutil" apimiddleware "github.com/calypr/syfon/internal/api/middleware" "github.com/calypr/syfon/internal/api/routeutil" @@ -20,8 +19,9 @@ import ( ) const ( - defaultInternalListLimit = 1000 - maxInternalListLimit = 10000 + defaultInternalListLimit = 1000 + maxInternalListLimit = 10000 + maxInternalBulkMissingSHA256 = 10000 ) func RegisterInternalRoutes(router fiber.Router, om *core.ObjectManager) { @@ -38,13 +38,69 @@ func RegisterInternalRoutes(router fiber.Router, om *core.ObjectManager) { router.Post(common.RouteInternalBulkHashes, handleInternalBulkHashesFiber(om)) router.Post(common.RouteInternalBulkSHA256, handleInternalBulkSHA256ValidityFiber(om)) + router.Post(common.RouteInternalBulkSHA256Missing, handleInternalBulkMissingSHA256Fiber(om)) router.Post(common.RouteInternalBulkCreate, handleInternalBulkCreateFiber(om)) router.Post(common.RouteInternalBulkDocs, handleInternalBulkDocumentsFiber(om)) router.Post(common.RouteInternalBulkDeleteHashes, handleInternalBulkDeleteFiber(om)) + router.Post(common.RouteInternalRepairScopeAudit, handleInternalScopeRepairAuditFiber(om)) + router.Post(common.RouteInternalRepairScopeApply, handleInternalScopeRepairApplyFiber(om)) registerInternalTransferRoutes(router, om) } +func handleInternalBulkMissingSHA256Fiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + var req internalapi.BulkMissingSHA256Request + if err := c.Bind().JSON(&req); err != nil { + return c.Status(fiber.StatusBadRequest).SendString("Invalid request body") + } + if strings.TrimSpace(req.Organization) == "" || strings.TrimSpace(req.Project) == "" || len(req.Sha256) == 0 { + return c.Status(fiber.StatusBadRequest).SendString("Invalid request body: organization, project, and sha256 values are required") + } + + normalized, err := normalizeMissingSHA256(req.Sha256) + if err != nil { + return c.Status(fiber.StatusBadRequest).SendString(err.Error()) + } + if len(normalized) > maxInternalBulkMissingSHA256 { + return c.Status(fiber.StatusRequestEntityTooLarge).SendString(fmt.Sprintf("too many sha256 values: maximum is %d", maxInternalBulkMissingSHA256)) + } + + missing, err := om.ListMissingScopedSHA256(c.Context(), req.Organization, req.Project, normalized) + if err != nil { + return apiutil.HandleError(c, err) + } + return c.JSON(internalapi.BulkMissingSHA256Response{Checked: int32(len(normalized)), MissingSha256: missing}) + } +} + +func normalizeMissingSHA256(values []string) ([]string, error) { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, raw := range values { + value := strings.TrimSpace(raw) + value = strings.TrimPrefix(strings.ToLower(value), "sha256:") + if value == "" { + continue + } + if len(value) != 64 { + return nil, fmt.Errorf("invalid sha256 checksum %q", raw) + } + if _, err := hex.DecodeString(value); err != nil { + return nil, fmt.Errorf("invalid sha256 checksum %q", raw) + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + if len(out) == 0 { + return nil, fmt.Errorf("Invalid request body: sha256 values are required") + } + return out, nil +} + func handleInternalGetFiber(om *core.ObjectManager) fiber.Handler { return func(c fiber.Ctx) error { id := c.Params("id") @@ -61,13 +117,7 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { hash := c.Query("hash") hashType := c.Query("hash_type") objectURL := strings.TrimSpace(c.Query("url")) - hasPathQuery := c.Request().URI().QueryArgs().Has("path") - pathQuery := c.Query("path") - if hash != "" { - if hasPathQuery { - return c.Status(fiber.StatusBadRequest).SendString("path is only supported for exact organization/project listings") - } hashType, hash = common.ParseHashQuery(hash, hashType) filterOrg := strings.TrimSpace(c.Query("organization")) filterProject := strings.TrimSpace(c.Query("project")) @@ -83,6 +133,10 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { if err != nil { return apiutil.HandleError(c, err) } + objs, err = om.PrepareScopedObjects(c.Context(), objs, filterOrg, filterProject, "read") + if err != nil { + return apiutil.HandleError(c, err) + } records := make([]internalapi.InternalRecord, 0, len(objs)) for _, o := range objs { records = append(records, core.InternalObjectToInternalRecord(o)) @@ -97,59 +151,41 @@ func handleInternalListFiber(om *core.ObjectManager) fiber.Handler { if !hasScope { filterOrg, filterProject = "", "" } - if hasPathQuery && (filterOrg == "" || filterProject == "") { - return c.Status(fiber.StatusBadRequest).SendString("organization and project are required when path is set") - } - if hasPathQuery && objectURL != "" { - return c.Status(fiber.StatusBadRequest).SendString("path is only supported for exact organization/project listings") - } - if _, _, err := common.NormalizeBrowsePath(pathQuery); err != nil { - return c.Status(fiber.StatusBadRequest).SendString(err.Error()) - } - limit, start, offset, err := parseInternalListPaginationFiber(c) if err != nil { return c.Status(fiber.StatusBadRequest).SendString(err.Error()) } - if hasPathQuery { - ids, directories, err := om.ListObjectIDsPageByPath(c.Context(), filterOrg, filterProject, pathQuery, "read", start, limit, offset) + + requestStart := time.Now() + listStart := time.Now() + var objs []models.InternalObject + if objectURL != "" { + var ids []string + ids, err = om.ListObjectIDsPageByURL(c.Context(), objectURL, filterOrg, filterProject, "read", start, limit, offset) if err != nil { return apiutil.HandleError(c, err) } - objs, err := om.GetBulkObjects(c.Context(), ids, "read") + bulkStart := time.Now() + objs, err = om.GetBulkObjects(c.Context(), ids, "read") if err != nil { return apiutil.HandleError(c, err) } - records := make([]internalapi.InternalRecord, 0, len(objs)) - for _, obj := range objs { - records = append(records, core.InternalObjectToInternalRecord(obj)) - } - respDirectories := make([]internalapi.IndexDirectory, 0, len(directories)) - for _, directory := range directories { - respDirectories = append(respDirectories, internalapi.IndexDirectory{ - Name: directory.Name, - Path: directory.Path, - }) + bulkDuration := time.Since(bulkStart) + prepareStart := time.Now() + objs, err = om.PrepareScopedObjects(c.Context(), objs, filterOrg, filterProject, "read") + if err != nil { + return apiutil.HandleError(c, err) } - return c.JSON(internalapi.ListRecordsResponse{ - Directories: &respDirectories, - Records: &records, - }) - } - - var ids []string - if objectURL != "" { - ids, err = om.ListObjectIDsPageByURL(c.Context(), objectURL, filterOrg, filterProject, "read", start, limit, offset) + listDuration := time.Since(listStart) + prepareDuration := time.Since(prepareStart) + log.Printf("INFO: syfon_internal_index_list organization=%s project=%s url_filter=%t start_after=%t limit=%d offset=%d ids=%d records=%d list_ids_ms=%d bulk_objects_ms=%d prepare_scoped_ms=%d duration_ms=%d", filterOrg, filterProject, true, strings.TrimSpace(start) != "", limit, offset, len(ids), len(objs), listDuration.Milliseconds(), bulkDuration.Milliseconds(), prepareDuration.Milliseconds(), time.Since(requestStart).Milliseconds()) } else { - ids, err = om.ListObjectIDsPageByScope(c.Context(), filterOrg, filterProject, "read", start, limit, offset) - } - if err != nil { - return apiutil.HandleError(c, err) - } - - objs, err := om.GetBulkObjects(c.Context(), ids, "read") - if err != nil { - return apiutil.HandleError(c, err) + objs, err = om.ListPreparedObjectsPageByScope(c.Context(), filterOrg, filterProject, "read", start, limit, offset) + if err != nil { + return apiutil.HandleError(c, err) + } + listDuration := time.Since(listStart) + log.Printf("INFO: syfon_internal_index_list organization=%s project=%s url_filter=%t start_after=%t limit=%d offset=%d records=%d list_prepared_ms=%d duration_ms=%d", filterOrg, filterProject, false, strings.TrimSpace(start) != "", limit, offset, len(objs), listDuration.Milliseconds(), time.Since(requestStart).Milliseconds()) } records := make([]internalapi.InternalRecord, 0, len(objs)) for _, obj := range objs { @@ -321,7 +357,6 @@ func handleInternalBulkDocumentsFiber(om *core.ObjectManager) fiber.Handler { } if obj, err := req.AsBulkDocumentsRequest1(); err == nil { ids = append(ids, common.DerefStringSlice(obj.Ids)...) - ids = append(ids, common.DerefStringSlice(obj.Dids)...) } if len(ids) == 0 { return c.Status(fiber.StatusBadRequest).SendString("Invalid request body: ids are required") @@ -369,23 +404,30 @@ func handleInternalBulkDeleteFiber(om *core.ObjectManager) fiber.Handler { func handleInternalUpdateFiber(c fiber.Ctx, om *core.ObjectManager) error { id := c.Params("id") - var req models.InternalObject - if err := c.Bind().JSON(&req); err != nil { - return c.Status(fiber.StatusBadRequest).SendString("Invalid request body") + var req internalapi.InternalRecord + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return c.Status(fiber.StatusBadRequest).SendString("Invalid request body: " + err.Error()) + } + if strings.TrimSpace(req.Did) == "" { + req.Did = id + } + update, err := core.InternalRecordToInternalObject(req, time.Now().UTC()) + if err != nil { + return c.Status(fiber.StatusBadRequest).SendString("Invalid request body: " + err.Error()) } existing, err := om.GetObject(c.Context(), id, "update") if err != nil { return apiutil.HandleError(c, err) } - merged, err := core.MergeInternalObjectUpdate(*existing, req, id, time.Now().UTC()) + merged, err := core.MergeInternalObjectUpdate(*existing, update, id, time.Now().UTC()) if err != nil { return apiutil.HandleError(c, err) } if err := om.ReplaceObjects(c.Context(), []models.InternalObject{merged}); err != nil { return apiutil.HandleError(c, err) } - return c.JSON(merged) + return c.JSON(core.InternalObjectToInternalRecordResponse(merged)) } func parseScopeQueryParts(organization, program, project string) (string, string, bool, error) { @@ -403,38 +445,6 @@ func parseScopeQueryParts(organization, program, project string) (string, string return "", "", false, nil } -func parseScopeQuery(r *http.Request) (string, string, bool, error) { - return parseScopeQueryParts( - r.URL.Query().Get("organization"), - r.URL.Query().Get("program"), - r.URL.Query().Get("project"), - ) -} - -func paginateInternalListIDsFiber(c fiber.Ctx, ids []string) ([]string, error) { - limit, start, offset, err := parseInternalListPaginationFiber(c) - if err != nil { - return nil, err - } - if limit == 0 || len(ids) == 0 { - return []string{}, nil - } - if start != "" { - offset = sort.SearchStrings(ids, start) - for offset < len(ids) && ids[offset] <= start { - offset++ - } - } - if offset >= len(ids) { - return []string{}, nil - } - end := offset + limit - if end > len(ids) { - end = len(ids) - } - return ids[offset:end], nil -} - func parseInternalListPaginationFiber(c fiber.Ctx) (int, string, int, error) { limit := defaultInternalListLimit rawLimit := strings.TrimSpace(c.Query("limit")) @@ -518,8 +528,3 @@ func normalizeNonEmptyBulkHashes(hashes []string) []string { } return normalized } - -func objectAuthzMatchesScope(obj models.InternalObject, org, project string) bool { - authzMap := sycommon.ControlledAccessToAuthzMap(core.ObjectAccessResources(&obj)) - return sycommon.AuthzMapMatchesScope(authzMap, org, project) -} diff --git a/internal/api/internaldrs/index_additional_test.go b/internal/api/internaldrs/index_additional_test.go index 48a23383..4f7bda81 100644 --- a/internal/api/internaldrs/index_additional_test.go +++ b/internal/api/internaldrs/index_additional_test.go @@ -81,18 +81,18 @@ func TestHandleInternalBulkDocuments_InvalidBodyAndMissingIDs(t *testing.T) { } } -func TestHandleInternalList_PathBrowseValidation(t *testing.T) { +func TestHandleInternalList_IgnoresLegacyPathValidation(t *testing.T) { om := core.NewObjectManager(&testutils.MockDatabase{}, &testutils.MockUrlManager{}) req := httptest.NewRequest(http.MethodGet, "/index?path=nested", nil) rr := doInternalDRSTestRequest(req, om) - if rr.Code != http.StatusBadRequest { - t.Fatalf("expected 400 when path is set without exact scope, got %d body=%s", rr.Code, rr.Body.String()) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200 when legacy path query is ignored, got %d body=%s", rr.Code, rr.Body.String()) } req = httptest.NewRequest(http.MethodGet, "/index?organization=org&project=proj&path=../nested", nil) rr = doInternalDRSTestRequest(req, om) - if rr.Code != http.StatusBadRequest { - t.Fatalf("expected 400 for invalid path, got %d body=%s", rr.Code, rr.Body.String()) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200 when invalid legacy path query is ignored, got %d body=%s", rr.Code, rr.Body.String()) } } diff --git a/internal/api/internaldrs/index_test.go b/internal/api/internaldrs/index_test.go index ad0616d1..bfb6de0f 100644 --- a/internal/api/internaldrs/index_test.go +++ b/internal/api/internaldrs/index_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "slices" "strings" "testing" "time" @@ -17,6 +18,7 @@ import ( internalauth "github.com/calypr/syfon/internal/auth" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/db" "github.com/calypr/syfon/internal/models" "github.com/calypr/syfon/internal/testutils" "github.com/gofiber/fiber/v3" @@ -30,30 +32,6 @@ func indexTestAuthContext(base context.Context, mode string, authHeader bool, pr return internalauth.WithSession(base, session) } -func TestParseScopeQuery(t *testing.T) { - t.Run("organization and project", func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/?organization=prog&project=study", nil) - org, project, ok, err := parseScopeQuery(req) - if err != nil { - t.Fatalf("parseScopeQuery returned error: %v", err) - } - if !ok || org != "prog" || project != "study" { - t.Fatalf("unexpected scope parse result: ok=%v org=%q project=%q", ok, org, project) - } - }) - - t.Run("project without organization fails", func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/?project=study", nil) - _, _, ok, err := parseScopeQuery(req) - if err == nil { - t.Fatal("expected validation error") - } - if ok { - t.Fatal("expected hasScope=false on validation error") - } - }) -} - func TestHandleInternalList_ScopeFilteringByReadPrivilege(t *testing.T) { now := time.Now().UTC() mockDB := &testutils.MockDatabase{ @@ -136,6 +114,275 @@ func TestHandleInternalList_ExactScopeListingDoesNotDependOnBrowseRows(t *testin } } +func TestHandleInternalList_CanonicalizesProjectChecksumDuplicates(t *testing.T) { + database := db.NewInMemoryDB() + om := core.NewObjectManager(database, &testutils.MockUrlManager{}) + now := time.Now().UTC() + later := now.Add(time.Minute) + sha := "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + oldURL := "s3://ellrottlab/old/" + sha + newURL := "s3://EllrottLab/new/" + sha + + for _, obj := range []models.InternalObject{ + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-1", + Name: stringPtr("older.tsv"), + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: sha}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: oldURL}, + }}, + }, + }, + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-2", + Name: stringPtr("newer.tsv"), + CreatedTime: later, + UpdatedTime: &later, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: sha}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: newURL}, + }}, + }, + }, + } { + if err := om.RegisterObjects(context.Background(), []models.InternalObject{obj}); err != nil { + t.Fatalf("RegisterObjects failed: %v", err) + } + } + + req := httptest.NewRequest(http.MethodGet, "/index?organization=org&project=p1", nil) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var payload struct { + Records []internalapi.InternalRecord `json:"records"` + } + if err := json.NewDecoder(rr.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload.Records) != 1 { + t.Fatalf("expected 1 canonical record, got %d", len(payload.Records)) + } + if payload.Records[0].Did != "did-1" { + t.Fatalf("expected canonical did-1, got %q", payload.Records[0].Did) + } + if payload.Records[0].Name == nil || *payload.Records[0].Name != "newer.tsv" { + t.Fatalf("expected latest name newer.tsv, got %+v", payload.Records[0].Name) + } + if payload.Records[0].NameAliases == nil || !slices.Equal(*payload.Records[0].NameAliases, []string{"older.tsv"}) { + t.Fatalf("unexpected name aliases: %#v", payload.Records[0].NameAliases) + } + if payload.Records[0].AccessMethods == nil || len(*payload.Records[0].AccessMethods) != 2 { + t.Fatalf("expected 2 merged access methods, got %#v", payload.Records[0].AccessMethods) + } + gotURLs := make([]string, 0, len(*payload.Records[0].AccessMethods)) + for _, method := range *payload.Records[0].AccessMethods { + if method.AccessUrl != nil { + gotURLs = append(gotURLs, method.AccessUrl.Url) + } + } + slices.Sort(gotURLs) + if !slices.Equal(gotURLs, []string{newURL, oldURL}) { + t.Fatalf("unexpected merged access method urls: %#v", gotURLs) + } +} + +func TestHandleInternalList_FillsLimitAfterCanonicalizingDuplicates(t *testing.T) { + database := db.NewInMemoryDB() + om := core.NewObjectManager(database, &testutils.MockUrlManager{}) + now := time.Now().UTC() + later := now.Add(time.Minute) + duplicateSHA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + uniqueSHA := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + + for _, obj := range []models.InternalObject{ + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-1", + Name: stringPtr("older.tsv"), + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: duplicateSHA}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/old/" + duplicateSHA}, + }}, + }, + }, + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-2", + Name: stringPtr("newer.tsv"), + CreatedTime: later, + UpdatedTime: &later, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: duplicateSHA}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/new/" + duplicateSHA}, + }}, + }, + }, + { + Authorizations: map[string][]string{"org": {"p1"}}, + DrsObject: drs.DrsObject{ + Id: "did-3", + Name: stringPtr("unique.tsv"), + CreatedTime: later, + UpdatedTime: &later, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: uniqueSHA}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/unique/" + uniqueSHA}, + }}, + }, + }, + } { + if err := om.RegisterObjects(context.Background(), []models.InternalObject{obj}); err != nil { + t.Fatalf("RegisterObjects failed: %v", err) + } + } + + req := httptest.NewRequest(http.MethodGet, "/index?organization=org&project=p1&limit=2", nil) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var payload struct { + Records []internalapi.InternalRecord `json:"records"` + } + if err := json.NewDecoder(rr.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload.Records) != 2 { + t.Fatalf("expected filled canonical page of 2 records, got %d: %+v", len(payload.Records), payload.Records) + } + if payload.Records[0].Did != "did-1" || payload.Records[1].Did != "did-3" { + t.Fatalf("unexpected canonical page ids: %+v", []string{payload.Records[0].Did, payload.Records[1].Did}) + } + + req = httptest.NewRequest(http.MethodGet, "/index?organization=org&project=p1&limit=1&start=did-1", nil) + rr = doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + payload.Records = nil + if err := json.NewDecoder(rr.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload.Records) != 1 || payload.Records[0].Did != "did-3" { + t.Fatalf("expected pagination to skip duplicate sibling and return did-3, got %+v", payload.Records) + } +} + +func TestHandleInternalList_MergesSiblingAccessMethodsFromLegacyDuplicateRows(t *testing.T) { + database := db.NewInMemoryDB() + om := core.NewObjectManager(database, &testutils.MockUrlManager{}) + now := time.Now().UTC() + later := now.Add(time.Minute) + sha := "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + oldURL := "s3://ellrottlab/fa4ee697-f689-5291-a7cc-8ebe2f3ea8b9/" + sha + newURL := "s3://EllrottLab/" + sha + controlled := []string{"/organization/org/project/p1"} + + for _, obj := range []models.InternalObject{ + { + DrsObject: drs.DrsObject{ + Id: "did-legacy-1", + Name: stringPtr("legacy.tsv"), + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: sha}}, + ControlledAccess: &controlled, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: oldURL}, + }}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "did-legacy-2", + Name: stringPtr("canonical.tsv"), + CreatedTime: later, + UpdatedTime: &later, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: sha}}, + ControlledAccess: &controlled, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: newURL}, + }}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + } { + candidate := obj + if err := database.CreateObject(context.Background(), &candidate); err != nil { + t.Fatalf("CreateObject failed: %v", err) + } + } + + req := httptest.NewRequest(http.MethodGet, "/index?organization=org&project=p1", nil) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var payload struct { + Records []internalapi.InternalRecord `json:"records"` + } + if err := json.NewDecoder(rr.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload.Records) != 1 { + t.Fatalf("expected 1 canonical record, got %d", len(payload.Records)) + } + if payload.Records[0].AccessMethods == nil || len(*payload.Records[0].AccessMethods) != 2 { + t.Fatalf("expected 2 merged access methods, got %#v", payload.Records[0].AccessMethods) + } + gotURLs := make([]string, 0, len(*payload.Records[0].AccessMethods)) + for _, method := range *payload.Records[0].AccessMethods { + if method.AccessUrl != nil { + gotURLs = append(gotURLs, method.AccessUrl.Url) + } + } + slices.Sort(gotURLs) + if !slices.Equal(gotURLs, []string{newURL, oldURL}) { + t.Fatalf("unexpected merged access method urls: %#v", gotURLs) + } +} + func TestHandleInternalList_PaginatesIDs(t *testing.T) { now := time.Now().UTC() mockDB := &testutils.MockDatabase{ @@ -267,31 +514,6 @@ func TestHandleInternalList_PagePaginatesIDs(t *testing.T) { } } -func TestPaginateInternalListIDsFiberDefaultLimit(t *testing.T) { - ids := make([]string, defaultInternalListLimit+5) - for i := range ids { - ids[i] = "obj" - } - app := fiber.New() - app.Get("/", func(c fiber.Ctx) error { - paged, err := paginateInternalListIDsFiber(c, ids) - if err != nil { - return err - } - if len(paged) != defaultInternalListLimit { - t.Fatalf("expected default limit %d, got %d", defaultInternalListLimit, len(paged)) - } - return c.SendStatus(fiber.StatusOK) - }) - resp, err := app.Test(httptest.NewRequest(http.MethodGet, "/", nil)) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } -} - func TestHandleInternalList_LimitIsCappedAtTenThousand(t *testing.T) { now := time.Now().UTC() total := maxInternalListLimit + 1 @@ -333,7 +555,7 @@ func TestHandleInternalList_LimitIsCappedAtTenThousand(t *testing.T) { } } -func TestHandleInternalList_PathBrowseReturnsImmediateDirectoriesAndFiles(t *testing.T) { +func TestHandleInternalList_IgnoresLegacyPathQuery(t *testing.T) { now := time.Now().UTC() mockDB := &testutils.MockDatabase{ Objects: map[string]*drs.DrsObject{ @@ -366,11 +588,8 @@ func TestHandleInternalList_PathBrowseReturnsImmediateDirectoriesAndFiles(t *tes if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { t.Fatalf("decode response: %v", err) } - if payload.Directories == nil || len(*payload.Directories) != 1 || (*payload.Directories)[0].Name != "deep" || (*payload.Directories)[0].Path != "nested/deep" { - t.Fatalf("unexpected directories: %+v", payload.Directories) - } - if payload.Records == nil || len(*payload.Records) != 1 || (*payload.Records)[0].Did != "obj-a" { - t.Fatalf("unexpected paged records: %+v", payload.Records) + if payload.Records == nil || len(*payload.Records) != 1 { + t.Fatalf("expected one paged record, got %+v", payload.Records) } req = httptest.NewRequest(http.MethodGet, "/index?organization=org-a&project=proj-a&path=nested&limit=1&start=obj-a", nil) @@ -382,14 +601,12 @@ func TestHandleInternalList_PathBrowseReturnsImmediateDirectoriesAndFiles(t *tes body, _ := io.ReadAll(resp.Body) t.Fatalf("expected 200, got %d body=%s", resp.StatusCode, string(body)) } + payload = internalapi.ListRecordsResponse{} if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - t.Fatalf("decode start response: %v", err) + t.Fatalf("decode paged response: %v", err) } - if payload.Directories == nil || len(*payload.Directories) != 1 { - t.Fatalf("expected directories on paged response, got %+v", payload.Directories) - } - if payload.Records == nil || len(*payload.Records) != 1 || (*payload.Records)[0].Did != "obj-d" { - t.Fatalf("expected obj-d after start cursor, got %+v", payload.Records) + if payload.Records == nil || len(*payload.Records) != 1 { + t.Fatalf("expected one paged record with start cursor, got %+v", payload.Records) } } @@ -574,6 +791,59 @@ func TestHandleInternalBulkSHA256Validity(t *testing.T) { } } +func TestHandleInternalBulkMissingSHA256(t *testing.T) { + present := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + missing := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + now := time.Now().UTC() + mockDB := &testutils.MockDatabase{Objects: map[string]*drs.DrsObject{ + "obj-sha": { + Id: "obj-sha", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: present}}, + }, + }, ObjectAuthz: map[string]map[string][]string{ + "obj-sha": {"org": {"project"}}, + }} + om := core.NewObjectManager(mockDB, &testutils.MockUrlManager{}) + app := fiber.New() + app.Post("/index/bulk/sha256/missing", handleInternalBulkMissingSHA256Fiber(om)) + req := httptest.NewRequest(http.MethodPost, "/index/bulk/sha256/missing", strings.NewReader(`{"organization":"org","project":"project","sha256":["SHA256:`+present+`","`+missing+`","`+missing+`"]}`)) + req.Header.Set("Content-Type", "application/json") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 200, got %d body=%s", resp.StatusCode, string(body)) + } + var payload internalapi.BulkMissingSHA256Response + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if payload.Checked != 2 || !slices.Equal(payload.MissingSha256, []string{missing}) { + t.Fatalf("unexpected response: %+v", payload) + } +} + +func TestHandleInternalBulkMissingSHA256RejectsInvalidChecksum(t *testing.T) { + om := core.NewObjectManager(&testutils.MockDatabase{}, &testutils.MockUrlManager{}) + app := fiber.New() + app.Post("/index/bulk/sha256/missing", handleInternalBulkMissingSHA256Fiber(om)) + req := httptest.NewRequest(http.MethodPost, "/index/bulk/sha256/missing", strings.NewReader(`{"organization":"org","project":"project","sha256":["not-a-sha256"]}`)) + req.Header.Set("Content-Type", "application/json") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400, got %d", resp.StatusCode) + } +} + func TestHandleInternalCreate_PersistsControlledAccess(t *testing.T) { mockDB := &testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}} reqBody := `{"records":[{"did":"obj-1","size":42,"controlled_access":["https://calypr.org/program/test/project/p1"],"access_methods":[{"type":"s3","access_url":{"url":"s3://bucket/path/obj-1"}}]}]}` @@ -623,6 +893,43 @@ func TestHandleInternalBulkCreate_PersistsControlledAccess(t *testing.T) { } } +func TestHandleInternalBulkCreate_OrganizationProjectAddsCanonicalControlledAccess(t *testing.T) { + mockDB := &testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}} + reqBody := `{"records":[{"did":"obj-bulk-2","organization":"test","project":"p2","size":7,"access_methods":[{"type":"s3","access_url":{"url":"s3://bucket/path/obj-bulk-2"}}]}]}` + req := httptest.NewRequest(http.MethodPost, "/index/bulk", strings.NewReader(reqBody)) + req.Header.Set("Content-Type", "application/json") + om := core.NewObjectManager(mockDB, &testutils.MockUrlManager{}) + rr := doInternalDRSTestRequest(req, om) + + if rr.Code != http.StatusCreated { + t.Fatalf("expected 201, got %d body=%s", rr.Code, rr.Body.String()) + } + if got := mockDB.ObjectAuthz["obj-bulk-2"]; len(got["test"]) != 1 || got["test"][0] != "p2" { + t.Fatalf("expected canonical project authz, got %v", got) + } +} + +func TestHandleInternalUpdate_OrganizationProjectAddsCanonicalControlledAccess(t *testing.T) { + now := time.Now().UTC() + mockDB := &testutils.MockDatabase{ + Objects: map[string]*drs.DrsObject{ + "obj-update": {Id: "obj-update", CreatedTime: now, UpdatedTime: &now}, + }, + } + reqBody := `{"did":"obj-update","organization":"test","project":"p3"}` + req := httptest.NewRequest(http.MethodPut, "/index/obj-update", strings.NewReader(reqBody)) + req.Header.Set("Content-Type", "application/json") + om := core.NewObjectManager(mockDB, &testutils.MockUrlManager{}) + rr := doInternalDRSTestRequest(req, om) + + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + if got := mockDB.ObjectAuthz["obj-update"]; len(got["test"]) != 1 || got["test"][0] != "p3" { + t.Fatalf("expected canonical project authz after update, got %v", got) + } +} + func TestHandleInternalBulkCreate_AllowsCreateAccessForAnyControlledAccessScope(t *testing.T) { ctx := indexTestAuthContext(context.Background(), "gen3", true, map[string]map[string]bool{ "/programs/test/projects/p1": {"create": true}, diff --git a/internal/api/internaldrs/inspect.go b/internal/api/internaldrs/inspect.go index f8ab4104..a11b5716 100644 --- a/internal/api/internaldrs/inspect.go +++ b/internal/api/internaldrs/inspect.go @@ -1,20 +1,58 @@ package internaldrs import ( + "log" "strings" + "time" + "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/api/apiutil" apimiddleware "github.com/calypr/syfon/internal/api/middleware" + "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/models" "github.com/gofiber/fiber/v3" ) type internalInspectObjectRequest struct { + ID string `json:"id,omitempty"` + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + Key string `json:"key,omitempty"` + Scheme string `json:"scheme,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` + ExpectedName string `json:"expected_name,omitempty"` +} + +type internalInspectObjectsBulkRequest struct { + Items []internalInspectObjectRequest `json:"items"` +} + +type internalInspectProjectBucketRequest struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + IncludeHead bool `json:"include_head,omitempty"` + Mode string `json:"mode,omitempty"` + PathPrefix string `json:"path_prefix,omitempty"` +} + +type internalInspectProjectRecordsRequest struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + PathPrefix string `json:"path_prefix,omitempty"` +} + +type internalInspectProjectScopesRequest struct { Organization string `json:"organization,omitempty"` Project string `json:"project,omitempty"` - Key string `json:"key,omitempty"` - Scheme string `json:"scheme,omitempty"` - ObjectURL string `json:"object_url,omitempty"` +} + +type internalDeleteProjectBucketObjectsRequest struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + ObjectURLs []string `json:"object_urls"` } type internalInspectObjectResponse struct { @@ -29,6 +67,107 @@ type internalInspectObjectResponse struct { LastModTime string `json:"last_modified,omitempty"` } +type internalInspectObjectBulkResponse struct { + Items []internalInspectObjectBulkItem `json:"items"` +} + +type internalInspectProjectBucketResponse struct { + Summary *internalInspectProjectBucketSummary `json:"summary,omitempty"` + Items []internalInspectProjectBucketItem `json:"items"` +} + +type internalInspectProjectRecordsResponse struct { + Items []internalInspectProjectRecordItem `json:"items"` +} + +type internalInspectProjectScopesResponse struct { + Items []internalInspectProjectScopeItem `json:"items"` +} + +type internalDeleteProjectBucketObjectsResponse struct { + Items []internalDeleteProjectBucketObjectsItem `json:"items"` +} + +type internalInspectObjectBulkItem struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + Provider string `json:"provider,omitempty"` + Bucket string `json:"bucket,omitempty"` + Key string `json:"key,omitempty"` + Path string `json:"path,omitempty"` + Exists bool `json:"exists"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + MetaSHA256 string `json:"meta_sha256,omitempty"` + ETag string `json:"etag,omitempty"` + LastModTime string `json:"last_modified,omitempty"` + ValidationStatus string `json:"validation_status"` + SizeMatch *bool `json:"size_match,omitempty"` + NameMatch *bool `json:"name_match,omitempty"` + SHA256Match *bool `json:"sha256_match,omitempty"` + ValidationMismatches []string `json:"validation_mismatches,omitempty"` +} + +type internalInspectProjectBucketSummary struct { + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + Exists bool `json:"exists"` + ObjectCount int `json:"object_count"` + TotalBytes int64 `json:"total_bytes"` + ComputedAt string `json:"computed_at"` + Mode string `json:"mode"` +} + +type internalInspectProjectBucketItem struct { + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + MetaSHA256 string `json:"meta_sha256,omitempty"` + ETag string `json:"etag,omitempty"` + LastModTime string `json:"last_modified,omitempty"` + InventoryComplete bool `json:"inventory_complete,omitempty"` +} + +type internalInspectProjectRecordItem struct { + ObjectID string `json:"object_id"` + Name string `json:"name,omitempty"` + Checksum string `json:"checksum"` + Organization string `json:"organization"` + Project string `json:"project"` + Size int64 `json:"size"` + CreatedTime string `json:"created_time,omitempty"` + UpdatedTime string `json:"updated_time,omitempty"` + AccessURLs []string `json:"access_urls"` + AccessMethods []internalProjectAccessMethod `json:"access_methods"` +} + +type internalProjectAccessMethod struct { + AccessID string `json:"access_id,omitempty"` + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Headers []string `json:"headers,omitempty"` +} + +type internalInspectProjectScopeItem struct { + Bucket string `json:"bucket"` + Organization string `json:"organization"` + ProjectID string `json:"project_id,omitempty"` + Path string `json:"path,omitempty"` +} + +type internalDeleteProjectBucketObjectsItem struct { + ObjectURL string `json:"object_url"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} + func handleInternalInspectObjectFiber(om *core.ObjectManager) fiber.Handler { return func(c fiber.Ctx) error { if apimiddleware.MissingGen3AuthHeader(c.Context()) { @@ -39,11 +178,14 @@ func handleInternalInspectObjectFiber(om *core.ObjectManager) fiber.Handler { return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) } resp, err := om.InspectStorageObject(c.Context(), core.InspectStorageRequest{ - Organization: strings.TrimSpace(req.Organization), - Project: strings.TrimSpace(req.Project), - Key: strings.TrimSpace(req.Key), - Scheme: strings.TrimSpace(req.Scheme), - ObjectURL: strings.TrimSpace(req.ObjectURL), + ID: strings.TrimSpace(req.ID), + Organization: strings.TrimSpace(req.Organization), + Project: strings.TrimSpace(req.Project), + Key: strings.TrimSpace(req.Key), + Scheme: strings.TrimSpace(req.Scheme), + ObjectURL: strings.TrimSpace(req.ObjectURL), + ExpectedSizeBytes: req.ExpectedSizeBytes, + ExpectedSHA256: strings.TrimSpace(req.ExpectedSHA256), }) if err != nil { return handleInspectStorageError(c, err) @@ -65,6 +207,264 @@ func handleInternalInspectObjectFiber(om *core.ObjectManager) fiber.Handler { } } +func handleInternalInspectObjectBulkFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectObjectsBulkRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + if len(req.Items) == 0 { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: items are required") + } + items := make([]core.InspectStorageRequest, 0, len(req.Items)) + for _, item := range req.Items { + items = append(items, core.InspectStorageRequest{ + ID: strings.TrimSpace(item.ID), + Organization: strings.TrimSpace(item.Organization), + Project: strings.TrimSpace(item.Project), + Key: strings.TrimSpace(item.Key), + Scheme: strings.TrimSpace(item.Scheme), + ObjectURL: strings.TrimSpace(item.ObjectURL), + ExpectedSizeBytes: item.ExpectedSizeBytes, + ExpectedSHA256: strings.TrimSpace(item.ExpectedSHA256), + }) + } + results := om.InspectStorageObjects(c.Context(), items) + out := internalInspectObjectBulkResponse{Items: make([]internalInspectObjectBulkItem, 0, len(results))} + for _, result := range results { + out.Items = append(out.Items, bulkInspectItemFromCore(result)) + } + return c.JSON(out) + } +} + +func handleInternalInspectObjectBulkListFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + started := time.Now() + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectObjectsBulkRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + if len(req.Items) == 0 { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: items are required") + } + items := make([]core.StorageListValidationRequest, 0, len(req.Items)) + for _, item := range req.Items { + items = append(items, core.StorageListValidationRequest{ + ID: strings.TrimSpace(item.ID), + ObjectURL: strings.TrimSpace(item.ObjectURL), + ExpectedSizeBytes: item.ExpectedSizeBytes, + ExpectedName: strings.TrimSpace(item.ExpectedName), + }) + } + results := om.ListValidateStorageObjects(c.Context(), items) + out := internalInspectObjectBulkResponse{Items: make([]internalInspectObjectBulkItem, 0, len(results))} + for _, result := range results { + out.Items = append(out.Items, bulkListInspectItemFromCore(result)) + } + log.Printf("INFO: syfon_inspect_bulk_list_handler items=%d results=%d duration_ms=%d", len(items), len(out.Items), time.Since(started).Milliseconds()) + return c.JSON(out) + } +} + +func handleInternalInspectProjectBucketFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + started := time.Now() + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectProjectBucketRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + result, err := om.InspectProjectStorage(c.Context(), strings.TrimSpace(req.Organization), strings.TrimSpace(req.Project), core.ProjectStorageInspectOptions{ + Mode: core.ProjectStorageInspectMode(strings.TrimSpace(req.Mode)), + IncludeHead: req.IncludeHead, + PathPrefix: strings.TrimSpace(req.PathPrefix), + }) + if err != nil { + log.Printf("INFO: syfon_project_bucket_handler organization=%s project=%s mode=%s path_prefix=%q include_head=%t duration_ms=%d error=%q", req.Organization, req.Project, req.Mode, req.PathPrefix, req.IncludeHead, time.Since(started).Milliseconds(), err.Error()) + return handleInspectStorageError(c, err) + } + out := projectBucketInventoryResponseFromCore(result) + exists := false + objectCount := 0 + totalBytes := int64(0) + mode := strings.TrimSpace(req.Mode) + if out.Summary != nil { + exists = out.Summary.Exists + objectCount = out.Summary.ObjectCount + totalBytes = out.Summary.TotalBytes + mode = out.Summary.Mode + } + log.Printf("INFO: syfon_project_bucket_handler organization=%s project=%s mode=%s path_prefix=%q include_head=%t exists=%t object_count=%d returned_items=%d total_bytes=%d duration_ms=%d", req.Organization, req.Project, mode, req.PathPrefix, req.IncludeHead, exists, objectCount, len(out.Items), totalBytes, time.Since(started).Milliseconds()) + return c.JSON(out) + } +} + +func handleInternalInspectProjectBucketInventoryFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + started := time.Now() + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectProjectBucketRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + result, err := om.InspectProjectStorage(c.Context(), strings.TrimSpace(req.Organization), strings.TrimSpace(req.Project), core.ProjectStorageInspectOptions{ + Mode: core.ProjectStorageInspectItems, + PathPrefix: strings.TrimSpace(req.PathPrefix), + }) + if err != nil { + log.Printf("INFO: syfon_project_bucket_inventory_handler organization=%s project=%s path_prefix=%q duration_ms=%d error=%q", req.Organization, req.Project, req.PathPrefix, time.Since(started).Milliseconds(), err.Error()) + return handleInspectStorageError(c, err) + } + out := projectBucketInventoryResponseFromCore(result) + objectCount := 0 + totalBytes := int64(0) + bucket := "" + prefix := "" + if out.Summary != nil { + objectCount = out.Summary.ObjectCount + totalBytes = out.Summary.TotalBytes + bucket = out.Summary.Bucket + prefix = out.Summary.Prefix + } + log.Printf("INFO: syfon_project_bucket_inventory_handler organization=%s project=%s path_prefix=%q bucket=%s prefix=%q object_count=%d returned_items=%d total_bytes=%d duration_ms=%d", req.Organization, req.Project, req.PathPrefix, bucket, prefix, objectCount, len(out.Items), totalBytes, time.Since(started).Milliseconds()) + return c.JSON(out) + } +} + +func handleInternalInspectProjectRecordsFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectProjectRecordsRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + organization := strings.TrimSpace(req.Organization) + project := strings.TrimSpace(req.Project) + if organization == "" || project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + objects, err := om.ListObjectsByScope(c.Context(), organization, project, "read") + if err != nil { + return apiutil.HandleError(c, err) + } + pathPrefix := strings.Trim(strings.TrimSpace(req.PathPrefix), "/") + pathPrefixes := []string{} + if pathPrefix != "" { + pathPrefixes = append(pathPrefixes, pathPrefix) + if resolvedPrefix, err := om.ResolveProjectStoragePathPrefix(c.Context(), organization, project, pathPrefix); err == nil && strings.TrimSpace(resolvedPrefix) != "" && !strings.EqualFold(strings.TrimSpace(resolvedPrefix), pathPrefix) { + pathPrefixes = append(pathPrefixes, strings.TrimSpace(resolvedPrefix)) + } + } + out := internalInspectProjectRecordsResponse{Items: make([]internalInspectProjectRecordItem, 0, len(objects))} + for _, obj := range objects { + record, ok := projectRecordAuditItemFromObject(obj, organization, project) + if !ok { + continue + } + if len(pathPrefixes) > 0 && !projectRecordMatchesAnyPathPrefix(record, pathPrefixes...) { + continue + } + out.Items = append(out.Items, record) + } + return c.JSON(out) + } +} + +func handleInternalInspectProjectScopesFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalInspectProjectScopesRequest + switch c.Method() { + case fiber.MethodGet: + req.Organization = c.Query("organization") + req.Project = c.Query("project") + default: + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + } + organization := strings.TrimSpace(req.Organization) + project := strings.TrimSpace(req.Project) + if organization == "" || project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + scopes, err := om.ListBucketScopes(c.Context()) + if err != nil { + return apiutil.HandleError(c, err) + } + out := internalInspectProjectScopesResponse{Items: make([]internalInspectProjectScopeItem, 0)} + for _, scope := range scopes { + if !strings.EqualFold(strings.TrimSpace(scope.Organization), organization) { + continue + } + scopeProject := strings.TrimSpace(scope.ProjectID) + if scopeProject != "" && !strings.EqualFold(scopeProject, project) { + continue + } + if !bucketScopeAllowed(c.Context(), scope, "read") { + continue + } + row := internalInspectProjectScopeItem{ + Bucket: strings.TrimSpace(scope.Bucket), + Organization: organization, + ProjectID: scopeProject, + } + scheme := "s3" + if cred, err := om.GetS3Credential(c.Context(), scope.CredentialID); err == nil && cred != nil { + scheme = common.ProviderToScheme(cred.Provider) + } + row.Path = scheme + "://" + strings.TrimSpace(scope.Bucket) + if scope.PathPrefix != "" { + row.Path += "/" + strings.Trim(strings.TrimSpace(scope.PathPrefix), "/") + } + out.Items = append(out.Items, row) + } + return c.JSON(out) + } +} + +func handleInternalDeleteProjectBucketObjectsFiber(om *core.ObjectManager) fiber.Handler { + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req internalDeleteProjectBucketObjectsRequest + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + if len(req.ObjectURLs) == 0 { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: object_urls are required") + } + results := om.DeleteProjectStorageObjects(c.Context(), strings.TrimSpace(req.Organization), strings.TrimSpace(req.Project), req.ObjectURLs) + out := internalDeleteProjectBucketObjectsResponse{ + Items: make([]internalDeleteProjectBucketObjectsItem, 0, len(results)), + } + for _, result := range results { + out.Items = append(out.Items, internalDeleteProjectBucketObjectsItem{ + ObjectURL: result.ObjectURL, + Status: result.Status, + Error: result.Error, + }) + } + return c.JSON(out) + } +} + func handleInspectStorageError(c fiber.Ctx, err error) error { if inspectErr, ok := err.(*core.StorageInspectError); ok { switch inspectErr.Kind { @@ -72,9 +472,206 @@ func handleInspectStorageError(c fiber.Ctx, err error) error { return apiutil.Reject(c, fiber.StatusBadRequest, inspectErr.Error()) case core.StorageInspectScopeNotFound, core.StorageInspectCredentialMissing, core.StorageInspectObjectNotFound: return apiutil.Reject(c, fiber.StatusNotFound, inspectErr.Error()) + case core.StorageInspectBucketUnavailable: + return apiutil.Reject(c, fiber.StatusConflict, inspectErr.Error()) case core.StorageInspectPermissionDenied: return apiutil.Reject(c, fiber.StatusForbidden, inspectErr.Error()) } } return apiutil.HandleError(c, err) } + +func bulkInspectItemFromCore(result core.StorageProbeResult) internalInspectObjectBulkItem { + out := internalInspectObjectBulkItem{ + ID: result.ID, + ObjectURL: result.ObjectURL, + Provider: result.Provider, + Bucket: result.Bucket, + Key: result.Key, + Path: result.Path, + Exists: result.Exists, + Status: string(result.Status), + Error: result.Error, + ErrorKind: result.ErrorKind, + SizeBytes: result.SizeBytes, + MetaSHA256: result.MetaSHA256, + ETag: result.ETag, + ValidationStatus: string(result.ValidationStatus), + SizeMatch: result.SizeMatch, + SHA256Match: result.SHA256Match, + ValidationMismatches: append([]string(nil), result.ValidationMismatches...), + } + if !result.LastModTime.IsZero() { + out.LastModTime = result.LastModTime.Format(time.RFC3339) + } + return out +} + +func bulkListInspectItemFromCore(result core.StorageListValidationResult) internalInspectObjectBulkItem { + out := internalInspectObjectBulkItem{ + ID: result.ID, + ObjectURL: result.ObjectURL, + Provider: result.Provider, + Bucket: result.Bucket, + Key: result.Key, + Path: result.Path, + Exists: result.Exists, + Status: string(result.Status), + Error: result.Error, + ErrorKind: result.ErrorKind, + SizeBytes: result.SizeBytes, + ETag: result.ETag, + ValidationStatus: string(result.ValidationStatus), + SizeMatch: result.SizeMatch, + NameMatch: result.NameMatch, + ValidationMismatches: append([]string(nil), result.ValidationMismatches...), + } + if !result.LastModTime.IsZero() { + out.LastModTime = result.LastModTime.Format(time.RFC3339) + } + return out +} + +func projectBucketSummaryFromCore(summary core.ProjectStorageSummary) *internalInspectProjectBucketSummary { + out := &internalInspectProjectBucketSummary{ + Provider: summary.Provider, + Bucket: summary.Bucket, + Prefix: summary.Prefix, + ObjectURL: summary.ObjectURL, + Exists: summary.Exists, + ObjectCount: summary.ObjectCount, + TotalBytes: summary.TotalBytes, + Mode: string(summary.Mode), + } + if !summary.ComputedAt.IsZero() { + out.ComputedAt = summary.ComputedAt.Format(time.RFC3339) + } + return out +} + +func projectBucketInventoryResponseFromCore(result *core.ProjectStorageInspectResult) internalInspectProjectBucketResponse { + if result == nil { + return internalInspectProjectBucketResponse{Items: []internalInspectProjectBucketItem{}} + } + out := internalInspectProjectBucketResponse{ + Summary: projectBucketSummaryFromCore(result.Summary), + Items: make([]internalInspectProjectBucketItem, 0, len(result.Items)), + } + for _, item := range result.Items { + row := internalInspectProjectBucketItem{ + ObjectURL: item.ObjectURL, + Provider: item.Provider, + Bucket: item.Bucket, + Key: item.Key, + Path: item.Path, + SizeBytes: item.SizeBytes, + MetaSHA256: item.MetaSHA256, + ETag: item.ETag, + InventoryComplete: true, + } + if !item.LastModTime.IsZero() { + row.LastModTime = item.LastModTime.Format(time.RFC3339) + } + out.Items = append(out.Items, row) + } + return out +} + +func projectRecordAuditItemFromObject(obj models.InternalObject, organization, project string) (internalInspectProjectRecordItem, bool) { + checksum := primarySHA256Checksum(obj.Checksums) + if checksum == "" { + return internalInspectProjectRecordItem{}, false + } + accessURLs := make([]string, 0) + accessMethods := make([]internalProjectAccessMethod, 0) + if obj.AccessMethods != nil { + for _, method := range *obj.AccessMethods { + row := internalProjectAccessMethod{ + Type: strings.TrimSpace(string(method.Type)), + Headers: []string{}, + } + if method.AccessId != nil { + row.AccessID = strings.TrimSpace(*method.AccessId) + } + if method.AccessUrl != nil { + row.URL = strings.TrimSpace(method.AccessUrl.Url) + if row.URL != "" { + accessURLs = append(accessURLs, row.URL) + } + if method.AccessUrl.Headers != nil { + row.Headers = append([]string(nil), (*method.AccessUrl.Headers)...) + } + } + accessMethods = append(accessMethods, row) + } + } + item := internalInspectProjectRecordItem{ + ObjectID: strings.TrimSpace(obj.Id), + Checksum: checksum, + Organization: organization, + Project: project, + Size: obj.Size, + AccessURLs: accessURLs, + AccessMethods: accessMethods, + } + if obj.Name != nil { + item.Name = strings.TrimSpace(*obj.Name) + } + if !obj.CreatedTime.IsZero() { + item.CreatedTime = obj.CreatedTime.Format(time.RFC3339Nano) + } + if obj.UpdatedTime != nil && !obj.UpdatedTime.IsZero() { + item.UpdatedTime = obj.UpdatedTime.Format(time.RFC3339Nano) + } + return item, true +} + +func projectRecordMatchesAnyPathPrefix(record internalInspectProjectRecordItem, pathPrefixes ...string) bool { + for _, prefix := range pathPrefixes { + if projectRecordMatchesPathPrefix(record, prefix) { + return true + } + } + return false +} + +func projectRecordMatchesPathPrefix(record internalInspectProjectRecordItem, pathPrefix string) bool { + normalizedPrefix := strings.Trim(strings.TrimSpace(pathPrefix), "/") + if normalizedPrefix == "" { + return true + } + prefixWithSlash := normalizedPrefix + "/" + for _, raw := range record.AccessURLs { + parsedBucket, parsedKey, ok := common.ParseS3URL(strings.TrimSpace(raw)) + if !ok || strings.TrimSpace(parsedBucket) == "" { + continue + } + key := strings.Trim(strings.TrimSpace(parsedKey), "/") + if key == normalizedPrefix || strings.HasPrefix(key, prefixWithSlash) { + return true + } + } + for _, method := range record.AccessMethods { + parsedBucket, parsedKey, ok := common.ParseS3URL(strings.TrimSpace(method.URL)) + if !ok || strings.TrimSpace(parsedBucket) == "" { + continue + } + key := strings.Trim(strings.TrimSpace(parsedKey), "/") + if key == normalizedPrefix || strings.HasPrefix(key, prefixWithSlash) { + return true + } + } + return false +} + +func primarySHA256Checksum(checksums []drs.Checksum) string { + for _, checksum := range checksums { + if strings.EqualFold(strings.TrimSpace(checksum.Type), "sha256") { + value := strings.TrimSpace(strings.TrimPrefix(checksum.Checksum, "sha256:")) + if value != "" { + return value + } + } + } + return "" +} diff --git a/internal/api/internaldrs/inspect_routes_test.go b/internal/api/internaldrs/inspect_routes_test.go index 68d32b17..b58d8d0f 100644 --- a/internal/api/internaldrs/inspect_routes_test.go +++ b/internal/api/internaldrs/inspect_routes_test.go @@ -4,12 +4,15 @@ import ( "bytes" "context" "encoding/json" + "fmt" + "io" "net/http" "net/http/httptest" "strings" "testing" "time" + "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/core" "github.com/calypr/syfon/internal/models" "github.com/calypr/syfon/internal/testutils" @@ -113,3 +116,509 @@ func TestHandleInternalInspectObjectMalformedURL(t *testing.T) { t.Fatalf("expected 400, got %d body=%s", rr.Code, rr.Body.String()) } } + +func TestHandleInternalInspectProjectRecords(t *testing.T) { + body, _ := json.Marshal(internalInspectProjectRecordsRequest{Organization: "syfon", Project: "e2e"}) + name := "example.bin" + created := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + updated := time.Date(2026, 7, 1, 12, 30, 0, 0, time.UTC) + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + Objects: map[string]*drs.DrsObject{ + "obj-1": { + Id: "obj-1", + Name: &name, + Checksums: []drs.Checksum{ + {Type: "sha256", Checksum: "abc123"}, + }, + CreatedTime: created, + UpdatedTime: &updated, + Size: 17, + AccessMethods: &[]drs.AccessMethod{ + { + Type: "s3", + AccessId: ptr("acc-1"), + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket-a/prefix/example.bin"}, + }, + }, + }, + }, + ObjectAuthz: map[string]map[string][]string{ + "obj-1": {"syfon": {"e2e"}}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-records", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{"/organization/syfon/project/e2e": {"read": true}}) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectProjectRecordsResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Items) != 1 { + t.Fatalf("expected 1 item, got %d", len(resp.Items)) + } + item := resp.Items[0] + if item.ObjectID != "obj-1" || item.Checksum != "abc123" { + t.Fatalf("unexpected item identity: %+v", item) + } + if item.Name != "example.bin" { + t.Fatalf("expected project record name, got %q", item.Name) + } + if item.Organization != "syfon" || item.Project != "e2e" { + t.Fatalf("unexpected scope: %+v", item) + } + if len(item.AccessURLs) != 1 || item.AccessURLs[0] != "s3://bucket-a/prefix/example.bin" { + t.Fatalf("unexpected access urls: %+v", item.AccessURLs) + } +} + +func TestProjectRecordMatchesAnyPathPrefixAvoidsFalsePrefixMatches(t *testing.T) { + record := internalInspectProjectRecordItem{ + AccessURLs: []string{"s3://bucket-a/project-root/CONFIG/a.json"}, + AccessMethods: []internalProjectAccessMethod{ + {URL: "s3://bucket-a/project-root/CONFIG/a.json"}, + }, + } + if !projectRecordMatchesAnyPathPrefix(record, "CONFIG", "project-root/CONFIG") { + t.Fatalf("expected CONFIG path prefix to match CONFIG/a.json") + } + if projectRecordMatchesAnyPathPrefix(record, "CONFIGURATION", "project-root/CONFIGURATION") { + t.Fatalf("expected CONFIGURATION path prefix not to match CONFIG/a.json") + } +} + +func TestHandleInternalInspectProjectScopes(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|": {Organization: "syfon", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "program-root"}, + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + authz := map[string]map[string]bool{ + "/organization/syfon": {"read": true}, + "/organization/syfon/project/e2e": {"read": true}, + } + cases := []struct { + name string + method string + path string + body io.Reader + }{ + { + name: "get query", + method: http.MethodGet, + path: "/data/inspect/project-scopes?organization=syfon&project=e2e", + }, + { + name: "post body", + method: http.MethodPost, + path: "/data/inspect/project-scopes", + body: bytes.NewBufferString(`{"organization":"syfon","project":"e2e"}`), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := withTestAuthzContext(httptest.NewRequest(tc.method, tc.path, tc.body), "gen3", authz) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectProjectScopesResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Items) != 2 { + t.Fatalf("expected 2 items, got %d", len(resp.Items)) + } + if resp.Items[0].Bucket != "bucket-a" || !strings.HasPrefix(resp.Items[0].Path, "s3://bucket-a/") { + t.Fatalf("unexpected first scope: %+v", resp.Items[0]) + } + }) + } +} + +func TestHandleInternalInspectProjectBucketModesUsePrefixList(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|": {Organization: "syfon", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "program-root"}, + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + inspectCalls := 0 + om.SetS3ObjectInspector(func(ctx context.Context, cred models.S3Credential, bucket string, key string) (*core.StorageObjectMetadata, error) { + inspectCalls++ + return nil, nil + }) + var listCalls []core.StoragePrefixListOptions + var listedPrefixes []string + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + if bucket != "bucket-a" { + t.Fatalf("unexpected list target bucket=%q prefix=%q", bucket, prefix) + } + listedPrefixes = append(listedPrefixes, prefix) + listCalls = append(listCalls, options) + if prefix == "program-root/project-root/CONFIG" { + return []core.StorageBucketObject{ + {Provider: "s3", Bucket: bucket, Key: prefix + "/a.bin", Path: "CONFIG/a.bin", SizeBytes: 10}, + {Provider: "s3", Bucket: bucket, Key: prefix + "/nested/b.bin", Path: "CONFIG/nested/b.bin", SizeBytes: 15}, + }, nil + } + if prefix != "program-root/project-root" { + t.Fatalf("unexpected list prefix=%q", prefix) + } + return []core.StorageBucketObject{ + {Provider: "s3", Bucket: bucket, Key: prefix + "/a.bin", Path: "a.bin", SizeBytes: 10}, + {Provider: "s3", Bucket: bucket, Key: prefix + "/b.bin", Path: "b.bin", SizeBytes: 15}, + }, nil + }) + + existsBody, _ := json.Marshal(internalInspectProjectBucketRequest{Organization: "syfon", Project: "e2e", Mode: "exists"}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-bucket", bytes.NewBuffer(existsBody)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var existsResp internalInspectProjectBucketResponse + if err := json.NewDecoder(rr.Body).Decode(&existsResp); err != nil { + t.Fatalf("decode exists response: %v", err) + } + if existsResp.Summary == nil || !existsResp.Summary.Exists || existsResp.Summary.Mode != "exists" { + t.Fatalf("unexpected exists summary: %+v", existsResp.Summary) + } + if len(existsResp.Items) != 0 { + t.Fatalf("exists mode should not return item rows, got %+v", existsResp.Items) + } + if len(listCalls) != 1 || listCalls[0].MaxKeys != 1 || listCalls[0].IncludeHead { + t.Fatalf("exists mode should use MaxKeys=1 without HEAD, got %+v", listCalls) + } + + summaryBody, _ := json.Marshal(internalInspectProjectBucketRequest{Organization: "syfon", Project: "e2e", Mode: "summary"}) + req = withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-bucket", bytes.NewBuffer(summaryBody)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr = doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var summaryResp internalInspectProjectBucketResponse + if err := json.NewDecoder(rr.Body).Decode(&summaryResp); err != nil { + t.Fatalf("decode summary response: %v", err) + } + if summaryResp.Summary == nil || summaryResp.Summary.Mode != "summary" || summaryResp.Summary.ObjectCount != 2 || summaryResp.Summary.TotalBytes != 25 { + t.Fatalf("unexpected summary response: %+v", summaryResp.Summary) + } + if len(summaryResp.Items) != 0 { + t.Fatalf("summary mode should not return item rows, got %+v", summaryResp.Items) + } + if len(listCalls) != 2 || listCalls[1].MaxKeys != 0 || listCalls[1].IncludeHead { + t.Fatalf("summary mode should list recursively without HEAD, got %+v", listCalls) + } + + itemsBody, _ := json.Marshal(internalInspectProjectBucketRequest{Organization: "syfon", Project: "e2e", Mode: "items", PathPrefix: "CONFIG"}) + req = withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-bucket", bytes.NewBuffer(itemsBody)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr = doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var itemsResp internalInspectProjectBucketResponse + if err := json.NewDecoder(rr.Body).Decode(&itemsResp); err != nil { + t.Fatalf("decode items response: %v", err) + } + if itemsResp.Summary == nil || itemsResp.Summary.Mode != "items" || itemsResp.Summary.Prefix != "program-root/project-root/CONFIG" { + t.Fatalf("unexpected items summary: %+v", itemsResp.Summary) + } + if len(itemsResp.Items) != 2 { + t.Fatalf("items mode should return recursive rows, got %+v", itemsResp.Items) + } + if len(listedPrefixes) != 3 || listedPrefixes[2] != "program-root/project-root/CONFIG" { + t.Fatalf("expected path_prefix to scope recursive LIST, got %+v", listedPrefixes) + } + if inspectCalls != 0 { + t.Fatalf("expected project-bucket modes not to HEAD objects, got %d inspector calls", inspectCalls) + } +} + +func TestHandleInternalInspectProjectBucketInventoryListsProjectScope(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|": {Organization: "syfon", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "program-root"}, + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + inspectCalls := 0 + om.SetS3ObjectInspector(func(ctx context.Context, cred models.S3Credential, bucket string, key string) (*core.StorageObjectMetadata, error) { + inspectCalls++ + return nil, nil + }) + var listCalls []core.StoragePrefixListOptions + var listedPrefixes []string + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + if bucket != "bucket-a" { + t.Fatalf("unexpected list target bucket=%q prefix=%q", bucket, prefix) + } + listedPrefixes = append(listedPrefixes, prefix) + listCalls = append(listCalls, options) + return []core.StorageBucketObject{ + {Provider: "s3", Bucket: bucket, Key: prefix + "/a.bin", Path: "CONFIG/a.bin", SizeBytes: 10}, + {Provider: "s3", Bucket: bucket, Key: prefix + "/nested/b.bin", Path: "CONFIG/nested/b.bin", SizeBytes: 15}, + }, nil + }) + + body, _ := json.Marshal(internalInspectProjectBucketRequest{Organization: "syfon", Project: "e2e", PathPrefix: "CONFIG"}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/project-bucket/inventory", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectProjectBucketResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.Summary == nil || resp.Summary.Mode != "items" || resp.Summary.Prefix != "program-root/project-root/CONFIG" { + t.Fatalf("unexpected inventory summary: %+v", resp.Summary) + } + if resp.Summary.ObjectCount != 2 || resp.Summary.TotalBytes != 25 || len(resp.Items) != 2 { + t.Fatalf("unexpected inventory response summary=%+v items=%+v", resp.Summary, resp.Items) + } + for _, item := range resp.Items { + if !item.InventoryComplete { + t.Fatalf("expected successful inventory item to be marked complete, got %+v", item) + } + } + if len(listCalls) != 1 || listCalls[0].IncludeHead || listCalls[0].MaxKeys != 0 || listCalls[0].ExactPrefix { + t.Fatalf("expected one recursive LIST without HEAD, got %+v", listCalls) + } + if len(listedPrefixes) != 1 || listedPrefixes[0] != "program-root/project-root/CONFIG" { + t.Fatalf("expected path_prefix to scope recursive inventory, got %+v", listedPrefixes) + } + if inspectCalls != 0 { + t.Fatalf("expected inventory route not to HEAD objects, got %d inspector calls", inspectCalls) + } +} + +func TestHandleInternalInspectObjectBulkListValidatesExactKeyWithoutHead(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + inspectCalls := 0 + om.SetS3ObjectInspector(func(ctx context.Context, cred models.S3Credential, bucket string, key string) (*core.StorageObjectMetadata, error) { + inspectCalls++ + return nil, nil + }) + var listedPrefixes []string + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + if bucket != "bucket-a" || !options.ExactPrefix || options.MaxKeys != 1 || options.IncludeHead { + t.Fatalf("unexpected bulk-list options bucket=%q options=%+v", bucket, options) + } + listedPrefixes = append(listedPrefixes, prefix) + switch prefix { + case "project-root/file.bin": + return []core.StorageBucketObject{{Provider: "s3", Bucket: bucket, Key: "project-root/file.bin", Path: "file.bin", SizeBytes: 17, ETag: "etag-1", LastModTime: time.Date(2026, 7, 1, 1, 2, 3, 0, time.UTC)}}, nil + case "project-root/dir": + return []core.StorageBucketObject{{Provider: "s3", Bucket: bucket, Key: "project-root/dir/child.bin", Path: "dir/child.bin", SizeBytes: 19}}, nil + default: + return nil, nil + } + }) + expectedSize := int64(17) + body, _ := json.Marshal(internalInspectObjectsBulkRequest{Items: []internalInspectObjectRequest{ + {ID: "present", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &expectedSize, ExpectedName: "file.bin"}, + {ID: "prefix-child", ObjectURL: "s3://bucket-a/project-root/dir", ExpectedSizeBytes: &expectedSize, ExpectedName: "dir"}, + }}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/bulk-list", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectObjectBulkResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Items) != 2 { + t.Fatalf("expected 2 items, got %+v", resp.Items) + } + if !resp.Items[0].Exists || resp.Items[0].Status != "present" || resp.Items[0].ValidationStatus != "matched" { + t.Fatalf("unexpected present item: %+v", resp.Items[0]) + } + if resp.Items[1].Exists || resp.Items[1].Status != "not_found" { + t.Fatalf("expected prefix child not to count as exact key, got %+v", resp.Items[1]) + } + if len(listedPrefixes) != 2 { + t.Fatalf("expected two LIST calls, got %+v", listedPrefixes) + } + if inspectCalls != 0 { + t.Fatalf("expected bulk-list not to HEAD objects, got %d inspector calls", inspectCalls) + } +} + +func TestHandleInternalInspectObjectBulkListDeduplicatesExactTargets(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + listCalls := 0 + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + listCalls++ + if bucket != "bucket-a" || prefix != "project-root/file.bin" || !options.ExactPrefix || options.MaxKeys != 1 { + t.Fatalf("unexpected bulk-list request bucket=%q prefix=%q options=%+v", bucket, prefix, options) + } + return []core.StorageBucketObject{{Provider: "s3", Bucket: bucket, Key: "project-root/file.bin", Path: "file.bin", SizeBytes: 17}}, nil + }) + expectedSize := int64(17) + body, _ := json.Marshal(internalInspectObjectsBulkRequest{Items: []internalInspectObjectRequest{ + {ID: "one", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &expectedSize, ExpectedName: "file.bin"}, + {ID: "two", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &expectedSize, ExpectedName: "file.bin"}, + }}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/bulk-list", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectObjectBulkResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if listCalls != 1 { + t.Fatalf("expected duplicate exact keys to share one LIST call, got %d", listCalls) + } + if len(resp.Items) != 2 || resp.Items[0].ID != "one" || resp.Items[1].ID != "two" { + t.Fatalf("expected one response per original request, got %+v", resp.Items) + } +} + +func TestHandleInternalInspectObjectBulkListSharesRemoteEvidenceAcrossValidationExpectations(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + listCalls := 0 + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + listCalls++ + return []core.StorageBucketObject{{Provider: "s3", Bucket: bucket, Key: "project-root/file.bin", Path: "file.bin", SizeBytes: 17}}, nil + }) + size17 := int64(17) + size99 := int64(99) + body, _ := json.Marshal(internalInspectObjectsBulkRequest{Items: []internalInspectObjectRequest{ + {ID: "matched", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &size17, ExpectedName: "file.bin"}, + {ID: "mismatched", ObjectURL: "s3://bucket-a/project-root/file.bin", ExpectedSizeBytes: &size99, ExpectedName: "file.bin"}, + }}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/bulk-list", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectObjectBulkResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if listCalls != 1 { + t.Fatalf("expected one shared LIST call, got %d", listCalls) + } + if len(resp.Items) != 2 || resp.Items[0].ValidationStatus != "matched" || resp.Items[1].ValidationStatus != "mismatched" { + t.Fatalf("expected per-request validation after shared evidence, got %+v", resp.Items) + } +} + +func TestHandleInternalInspectObjectBulkListCoalescesDensePrefixes(t *testing.T) { + db := &testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "cred-1": {CredentialID: "cred-1", Bucket: "bucket-a", Provider: "s3"}, + }, + BucketScopes: map[string]models.BucketScope{ + "syfon|e2e": {Organization: "syfon", ProjectID: "e2e", Bucket: "bucket-a", CredentialID: "cred-1", PathPrefix: "project-root"}, + }, + } + om := core.NewObjectManager(db, &testutils.MockUrlManager{}) + listCalls := 0 + om.SetS3PrefixListerWithOptions(func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options core.StoragePrefixListOptions) ([]core.StorageBucketObject, error) { + listCalls++ + if bucket != "bucket-a" || prefix != "project-root/dense/" || !options.ExactPrefix || options.MaxKeys != 0 || options.IncludeHead { + t.Fatalf("unexpected coalesced LIST request bucket=%q prefix=%q options=%+v", bucket, prefix, options) + } + out := make([]core.StorageBucketObject, 0, 25) + for i := 0; i < 25; i++ { + key := fmt.Sprintf("project-root/dense/file-%02d.bin", i) + out = append(out, core.StorageBucketObject{Provider: "s3", Bucket: bucket, Key: key, Path: fmt.Sprintf("file-%02d.bin", i), SizeBytes: int64(i + 1)}) + } + return out, nil + }) + items := make([]internalInspectObjectRequest, 0, 25) + for i := 0; i < 25; i++ { + size := int64(i + 1) + items = append(items, internalInspectObjectRequest{ + ID: fmt.Sprintf("item-%02d", i), + ObjectURL: fmt.Sprintf("s3://bucket-a/project-root/dense/file-%02d.bin", i), + ExpectedSizeBytes: &size, + ExpectedName: fmt.Sprintf("file-%02d.bin", i), + }) + } + body, _ := json.Marshal(internalInspectObjectsBulkRequest{Items: items}) + req := withTestAuthzContext(httptest.NewRequest(http.MethodPost, "/data/inspect/bulk-list", bytes.NewBuffer(body)), "gen3", map[string]map[string]bool{ + "/organization/syfon/project/e2e": {"read": true}, + }) + rr := doInternalDRSTestRequest(req, om) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp internalInspectObjectBulkResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if listCalls != 1 { + t.Fatalf("expected dense prefix to coalesce into one LIST call, got %d", listCalls) + } + if len(resp.Items) != 25 { + t.Fatalf("expected 25 items, got %d", len(resp.Items)) + } +} diff --git a/internal/api/internaldrs/repair_cleanup.go b/internal/api/internaldrs/repair_cleanup.go new file mode 100644 index 00000000..75fa6f98 --- /dev/null +++ b/internal/api/internaldrs/repair_cleanup.go @@ -0,0 +1,330 @@ +package internaldrs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/calypr/syfon/apigen/client/bucketapi" + clientdrs "github.com/calypr/syfon/apigen/client/drs" + clientinternalapi "github.com/calypr/syfon/apigen/client/internalapi" + serverdrs "github.com/calypr/syfon/apigen/server/drs" + serverinternalapi "github.com/calypr/syfon/apigen/server/internalapi" + "github.com/calypr/syfon/client/request" + sycommon "github.com/calypr/syfon/common" + "github.com/calypr/syfon/internal/api/apiutil" + apimiddleware "github.com/calypr/syfon/internal/api/middleware" + "github.com/calypr/syfon/internal/authz" + intcommon "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/models" + "github.com/calypr/syfon/internal/repair" + "github.com/gofiber/fiber/v3" +) + +func authorizeStorageCleanupScope(ctx context.Context, organization, project string, methods ...string) error { + if !authz.IsAuthzEnforced(ctx) { + return nil + } + resource, err := sycommon.ResourcePath(organization, project) + if err != nil { + return err + } + if authz.HasMethodAccess(ctx, methods[0], []string{"/programs", "/data_file"}) || authz.HasAnyMethodAccess(ctx, []string{resource}, methods...) { + return nil + } + return intcommon.ErrUnauthorized +} + +func handleInternalScopeRepairAuditFiber(om *core.ObjectManager) fiber.Handler { + svc := repair.NewService(scopeRepairIndexAdapter{om: om}, scopeRepairBucketsAdapter{om: om}, &srvRequester{om: om}) + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req repair.Options + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + req.Organization = strings.TrimSpace(req.Organization) + req.Project = strings.TrimSpace(req.Project) + req.CheckStorage = true + if req.Organization == "" || req.Project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read"); err != nil { + return apiutil.HandleError(c, err) + } + report, err := svc.Audit(c.Context(), req) + if err != nil { + return apiutil.HandleError(c, err) + } + return c.JSON(report) + } +} + +func handleInternalScopeRepairApplyFiber(om *core.ObjectManager) fiber.Handler { + svc := repair.NewService(scopeRepairIndexAdapter{om: om}, scopeRepairBucketsAdapter{om: om}, &srvRequester{om: om}) + return func(c fiber.Ctx) error { + if apimiddleware.MissingGen3AuthHeader(c.Context()) { + return c.SendStatus(fiber.StatusUnauthorized) + } + var req repair.Options + if err := decodeStrictJSON(c.Body(), &req); err != nil { + return apiutil.Reject(c, fiber.StatusBadRequest, "Invalid request body: "+err.Error()) + } + req.Organization = strings.TrimSpace(req.Organization) + req.Project = strings.TrimSpace(req.Project) + req.CheckStorage = true + if req.Organization == "" || req.Project == "" { + return apiutil.Reject(c, fiber.StatusBadRequest, "organization and project are required") + } + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "read"); err != nil { + return apiutil.HandleError(c, err) + } + if err := authorizeStorageCleanupScope(c.Context(), req.Organization, req.Project, "update"); err != nil { + return apiutil.HandleError(c, err) + } + if _, err := om.CollapseProjectChecksumDuplicates(c.Context(), req.Organization, req.Project); err != nil { + return apiutil.HandleError(c, err) + } + result, err := svc.Apply(c.Context(), req) + if err != nil { + return apiutil.HandleError(c, err) + } + return c.JSON(result) + } +} + +type scopeRepairIndexAdapter struct { + om *core.ObjectManager +} + +func (a scopeRepairIndexAdapter) List(ctx context.Context, opts repair.ListRecordsOptions) (clientinternalapi.ListRecordsResponse, error) { + objects, err := a.om.ListPreparedObjectsPageByScope(ctx, strings.TrimSpace(opts.Organization), strings.TrimSpace(opts.Project), "read", strings.TrimSpace(opts.Start), opts.Limit, 0) + if err != nil { + return clientinternalapi.ListRecordsResponse{}, err + } + records := make([]clientinternalapi.InternalRecord, 0, len(objects)) + for _, obj := range objects { + records = append(records, clientRecordFromServer(core.InternalObjectToInternalRecord(obj))) + } + return clientinternalapi.ListRecordsResponse{Records: &records}, nil +} + +func (a scopeRepairIndexAdapter) Update(ctx context.Context, did string, rec clientinternalapi.InternalRecord) (clientinternalapi.InternalRecordResponse, error) { + update, err := core.InternalRecordToInternalObject(serverRecordFromClient(rec), time.Now().UTC()) + if err != nil { + return clientinternalapi.InternalRecordResponse{}, err + } + existing, err := a.om.GetObject(ctx, did, "update") + if err != nil { + return clientinternalapi.InternalRecordResponse{}, err + } + merged, err := core.MergeInternalObjectUpdate(*existing, update, did, time.Now().UTC()) + if err != nil { + return clientinternalapi.InternalRecordResponse{}, err + } + if err := a.om.ReplaceObjects(ctx, []models.InternalObject{merged}); err != nil { + return clientinternalapi.InternalRecordResponse{}, err + } + return clientRecordResponseFromServer(core.InternalObjectToInternalRecordResponse(merged)), nil +} + +type scopeRepairBucketsAdapter struct { + om *core.ObjectManager +} + +func (a scopeRepairBucketsAdapter) List(ctx context.Context) (bucketapi.BucketsResponse, error) { + creds, err := a.om.ListS3Credentials(ctx) + if err != nil { + return bucketapi.BucketsResponse{}, err + } + out := bucketapi.BucketsResponse{S3BUCKETS: map[string]bucketapi.BucketMetadata{}} + for _, cred := range creds { + out.S3BUCKETS[strings.TrimSpace(cred.Bucket)] = bucketapi.BucketMetadata{} + } + return out, nil +} + +func (a scopeRepairBucketsAdapter) ListScopes(ctx context.Context, bucket string) ([]bucketapi.BucketScopeResponse, error) { + scopes, err := a.om.ListBucketScopes(ctx) + if err != nil { + return nil, err + } + out := make([]bucketapi.BucketScopeResponse, 0) + for _, scope := range scopes { + if strings.TrimSpace(scope.Bucket) != strings.TrimSpace(bucket) { + continue + } + path := "" + if strings.TrimSpace(scope.PathPrefix) != "" { + path = fmt.Sprintf("s3://%s/%s", scope.Bucket, strings.Trim(strings.TrimSpace(scope.PathPrefix), "/")) + } else { + path = fmt.Sprintf("s3://%s", scope.Bucket) + } + out = append(out, bucketapi.BucketScopeResponse{ + Organization: scope.Organization, + ProjectId: scope.ProjectID, + Path: &path, + }) + } + return out, nil +} + +func clientRecordFromServer(rec serverinternalapi.InternalRecord) clientinternalapi.InternalRecord { + out := clientinternalapi.InternalRecord{ + Did: rec.Did, + AccessMethods: clientAccessMethodsFromServer(rec.AccessMethods), + ControlledAccess: rec.ControlledAccess, + Size: rec.Size, + CreatedTime: rec.CreatedTime, + Description: rec.Description, + Name: rec.Name, + NameAliases: rec.NameAliases, + Version: rec.Version, + UpdatedTime: rec.UpdatedTime, + Hashes: (*clientinternalapi.HashInfo)(rec.Hashes), + Organization: rec.Organization, + Project: rec.Project, + } + return out +} + +func serverRecordFromClient(rec clientinternalapi.InternalRecord) serverinternalapi.InternalRecord { + out := serverinternalapi.InternalRecord{ + Did: rec.Did, + AccessMethods: serverAccessMethodsFromClient(rec.AccessMethods), + ControlledAccess: rec.ControlledAccess, + Size: rec.Size, + CreatedTime: rec.CreatedTime, + Description: rec.Description, + Name: rec.Name, + NameAliases: rec.NameAliases, + Version: rec.Version, + UpdatedTime: rec.UpdatedTime, + Hashes: (*serverinternalapi.HashInfo)(rec.Hashes), + Organization: rec.Organization, + Project: rec.Project, + } + return out +} + +func clientRecordResponseFromServer(rec serverinternalapi.InternalRecordResponse) clientinternalapi.InternalRecordResponse { + out := clientinternalapi.InternalRecordResponse{ + Did: rec.Did, + AccessMethods: clientAccessMethodsFromServer(rec.AccessMethods), + ControlledAccess: rec.ControlledAccess, + Size: rec.Size, + CreatedTime: rec.CreatedTime, + Description: rec.Description, + Name: rec.Name, + NameAliases: rec.NameAliases, + Version: rec.Version, + UpdatedTime: rec.UpdatedTime, + Hashes: (*clientinternalapi.HashInfo)(rec.Hashes), + Organization: rec.Organization, + Project: rec.Project, + } + return out +} + +func clientAccessMethodsFromServer(in *[]serverdrs.AccessMethod) *[]clientdrs.AccessMethod { + if in == nil { + return nil + } + out := make([]clientdrs.AccessMethod, 0, len(*in)) + for _, method := range *in { + var accessURL *struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + } + if method.AccessUrl != nil { + accessURL = &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{ + Headers: method.AccessUrl.Headers, + Url: method.AccessUrl.Url, + } + } + out = append(out, clientdrs.AccessMethod{ + AccessId: method.AccessId, + AccessUrl: accessURL, + Region: method.Region, + Type: clientdrs.AccessMethodType(method.Type), + }) + } + return &out +} + +func serverAccessMethodsFromClient(in *[]clientdrs.AccessMethod) *[]serverdrs.AccessMethod { + if in == nil { + return nil + } + out := make([]serverdrs.AccessMethod, 0, len(*in)) + for _, method := range *in { + var accessURL *struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + } + if method.AccessUrl != nil { + accessURL = &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{ + Headers: method.AccessUrl.Headers, + Url: method.AccessUrl.Url, + } + } + out = append(out, serverdrs.AccessMethod{ + AccessId: method.AccessId, + AccessUrl: accessURL, + Region: method.Region, + Type: serverdrs.AccessMethodType(method.Type), + }) + } + return &out +} + +type srvRequester struct { + om *core.ObjectManager +} + +func (r *srvRequester) Do(ctx context.Context, method, path string, body, out any, opts ...request.RequestOption) error { + if path == "/data/inspect" { + data, err := json.Marshal(body) + if err != nil { + return err + } + var inspectReq core.InspectStorageRequest + if err := json.Unmarshal(data, &inspectReq); err != nil { + return err + } + meta, err := r.om.InspectStorageObject(ctx, inspectReq) + if err != nil { + var inspectErr *core.StorageInspectError + if errors.As(err, &inspectErr) { + if inspectErr.Kind == core.StorageInspectObjectNotFound { + return &request.ResponseError{ + Method: method, + URL: path, + Status: http.StatusNotFound, + Body: "storage object not found", + } + } + } + return err + } + if out != nil { + respData, _ := json.Marshal(map[string]string{"object_url": meta.ObjectURL}) + _ = json.Unmarshal(respData, out) + } + return nil + } + return fmt.Errorf("unsupported service call: %s", path) +} diff --git a/internal/api/internaldrs/repair_cleanup_test.go b/internal/api/internaldrs/repair_cleanup_test.go new file mode 100644 index 00000000..1ef62b05 --- /dev/null +++ b/internal/api/internaldrs/repair_cleanup_test.go @@ -0,0 +1,30 @@ +package internaldrs + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + sycommon "github.com/calypr/syfon/common" + "github.com/calypr/syfon/internal/core" + "github.com/calypr/syfon/internal/testutils" +) + +func TestScopeRepairApplyRejectsReadOnlyProjectUser(t *testing.T) { + resource, err := sycommon.ResourcePath("org", "proj") + if err != nil { + t.Fatalf("resource path: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/data/repair/project-scope/apply", strings.NewReader(`{"organization":"org","project":"proj"}`)) + req.Header.Set("Content-Type", "application/json") + req = req.WithContext(policyTestContext("gen3", true, map[string]map[string]bool{ + resource: {"read": true}, + })) + + rr := doInternalDRSTestRequest(req, core.NewObjectManager(&testutils.MockDatabase{}, &testutils.MockUrlManager{})) + if rr.Code != http.StatusForbidden { + t.Fatalf("expected 403 for read-only caller, got %d body=%s", rr.Code, rr.Body.String()) + } +} diff --git a/internal/api/internaldrs/transfer.go b/internal/api/internaldrs/transfer.go index 3ddaa913..72e1797c 100644 --- a/internal/api/internaldrs/transfer.go +++ b/internal/api/internaldrs/transfer.go @@ -32,6 +32,14 @@ func registerInternalTransferRoutes(router fiber.Router, om *core.ObjectManager) router.Post(common.RouteInternalUpload, handleInternalUploadBlankFiber(om)) router.Get(routeutil.FiberPath(common.RouteInternalUploadURL), handleInternalUploadURLFiber(om)) router.Post(common.RouteInternalInspectObject, handleInternalInspectObjectFiber(om)) + router.Post(common.RouteInternalInspectObjectBulk, handleInternalInspectObjectBulkFiber(om)) + router.Post(common.RouteInternalInspectObjectBulkList, handleInternalInspectObjectBulkListFiber(om)) + router.Post(common.RouteInternalInspectProjectBucket, handleInternalInspectProjectBucketFiber(om)) + router.Post(common.RouteInternalInspectProjectBucketInventory, handleInternalInspectProjectBucketInventoryFiber(om)) + router.Post(common.RouteInternalInspectProjectRecords, handleInternalInspectProjectRecordsFiber(om)) + router.Get(common.RouteInternalInspectProjectScopes, handleInternalInspectProjectScopesFiber(om)) + router.Post(common.RouteInternalInspectProjectScopes, handleInternalInspectProjectScopesFiber(om)) + router.Post(common.RouteInternalDeleteProjectBucketObjects, handleInternalDeleteProjectBucketObjectsFiber(om)) router.Post(common.RouteInternalUploadBulk, handleInternalUploadBulkFiber(om)) router.Post(common.RouteInternalMultipartInit, handleInternalMultipartInitFiber(om)) router.Post(common.RouteInternalMultipartUpload, handleInternalMultipartUploadFiber(om)) @@ -209,7 +217,7 @@ func handleInternalUploadURLFiber(om *core.ObjectManager) fiber.Handler { } else { target, err = om.ResolveCanonicalStorageTarget(c.Context(), core.CanonicalStorageTargetRequest{ Object: obj, - Key: strings.TrimSpace(common.StringVal(params.FileName)), + Key: strings.TrimSpace(common.StringVal(params.Key)), PreferChecksum: true, }) } @@ -228,7 +236,7 @@ func handleInternalUploadURLFiber(om *core.ObjectManager) fiber.Handler { } return c.JSON(internalapi.InternalSignedURL{Url: &signedURL}) } else { - if requestedKey := strings.Trim(strings.TrimSpace(firstNonEmpty(common.StringVal(params.FileName), c.Query("file_name"), c.Query("filename"))), "/"); requestedKey != "" { + if requestedKey := strings.Trim(strings.TrimSpace(firstNonEmpty(common.StringVal(params.Key), c.Query("key"))), "/"); requestedKey != "" { key = requestedKey } target, err := resolveUploadTarget(c.Context(), om, common.StringVal(params.Organization), common.StringVal(params.Project), key) @@ -258,7 +266,7 @@ func handleInternalUploadURLFiber(om *core.ObjectManager) fiber.Handler { } func uploadKeyForExistingObject(obj *models.InternalObject, params internalapi.InternalUploadURLParams) string { - if key := strings.Trim(strings.TrimSpace(common.StringVal(params.FileName)), "/"); key != "" { + if key := strings.Trim(strings.TrimSpace(common.StringVal(params.Key)), "/"); key != "" { return key } if obj != nil { @@ -296,7 +304,7 @@ func handleInternalUploadBulkFiber(om *core.ObjectManager) fiber.Handler { results := make([]internalapi.InternalUploadBulkResult, 0, len(req.Requests)) for _, item := range req.Requests { - res := internalapi.InternalUploadBulkResult{FileId: item.FileId, FileName: item.FileName} + res := internalapi.InternalUploadBulkResult{FileId: item.FileId, Key: item.Key} if item.FileId == "" { errMsg := "FileId is required" res.Error = &errMsg @@ -323,7 +331,7 @@ func handleInternalUploadBulkFiber(om *core.ObjectManager) fiber.Handler { target, err := om.ResolveCanonicalStorageTarget(c.Context(), core.CanonicalStorageTargetRequest{ Object: obj, - Key: strings.TrimSpace(common.StringVal(item.FileName)), + Key: strings.TrimSpace(common.StringVal(item.Key)), PreferChecksum: true, }) if err != nil { @@ -353,7 +361,7 @@ func handleInternalUploadBulkFiber(om *core.ObjectManager) fiber.Handler { } else { res.Url = &signedURL res.Bucket = &bucket - res.FileName = &key + res.Key = &key res.Status = http.StatusOK } results = append(results, res) @@ -380,11 +388,11 @@ func handleInternalMultipartInitFiber(om *core.ObjectManager) fiber.Handler { key := "" if req.Guid != nil { key = strings.TrimSpace(*req.Guid) - } else if req.FileName != nil { - key = strings.TrimSpace(*req.FileName) + } else if req.Key != nil { + key = strings.TrimSpace(*req.Key) } if key == "" { - return c.Status(fiber.StatusBadRequest).SendString("key/guid/file_name is required") + return c.Status(fiber.StatusBadRequest).SendString("key/guid is required") } if strings.Contains(key, "/") { diff --git a/internal/api/internaldrs/upload_routes_test.go b/internal/api/internaldrs/upload_routes_test.go index c73b9bff..1b69b798 100644 --- a/internal/api/internaldrs/upload_routes_test.go +++ b/internal/api/internaldrs/upload_routes_test.go @@ -89,7 +89,7 @@ func TestHandleInternalUploadBlank_RequiresScope(t *testing.T) { func TestHandleInternalUploadURL_MissingObjectResolvesOrganizationProjectScope(t *testing.T) { mockUM := &capturingMultipartURLManager{} req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/new-guid?organization=syfon&project=e2e&file_name=payload.bin", nil), + httptest.NewRequest(http.MethodGet, "/data/upload/new-guid?organization=syfon&project=e2e&key=payload.bin", nil), map[string]string{"file_id": "new-guid"}, ) rr := doInternalDRSTestRequest(req, core.NewObjectManager(&testutils.MockDatabase{ @@ -123,7 +123,7 @@ func TestHandleInternalMultipartInit(t *testing.T) { guid := "multipart-guid" org := "syfon" project := "e2e" - body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Guid: &guid, FileName: &fileName, Organization: &org, Project: &project}) + body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Guid: &guid, Key: &fileName, Organization: &org, Project: &project}) rr := doInternalDRSTestRequest(httptest.NewRequest(http.MethodPost, "/data/multipart/init", bytes.NewBuffer(body)), core.NewObjectManager(&testutils.MockDatabase{ Objects: map[string]*drs.DrsObject{}, Credentials: map[string]models.S3Credential{"b1": {Bucket: "b1"}}, @@ -139,7 +139,7 @@ func TestHandleInternalMultipartInit(t *testing.T) { func TestHandleInternalMultipartInit_RequiresScopeForNewUpload(t *testing.T) { fileName := "test.bam" guid := "multipart-guid" - body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Guid: &guid, FileName: &fileName}) + body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Guid: &guid, Key: &fileName}) rr := doInternalDRSTestRequest(httptest.NewRequest(http.MethodPost, "/data/multipart/init", bytes.NewBuffer(body)), core.NewObjectManager(&testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}}, &testutils.MockUrlManager{})) if rr.Code != http.StatusBadRequest { t.Fatalf("expected 400, got %d", rr.Code) @@ -197,7 +197,7 @@ func TestHandleInternalMultipartInit_PreservesRequestedKey(t *testing.T) { func TestHandleInternalMultipartInit_MintsUUIDForChecksumInput(t *testing.T) { checksum := strings.Repeat("a", 64) - body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{FileName: &checksum}) + body, _ := json.Marshal(internalapi.InternalMultipartInitRequest{Key: &checksum}) mockDB := &testutils.MockDatabase{Objects: map[string]*drs.DrsObject{}} rr := doInternalDRSTestRequest(httptest.NewRequest(http.MethodPost, "/data/multipart/init", bytes.NewBuffer(body)), core.NewObjectManager(mockDB, &testutils.MockUrlManager{})) if rr.Code != http.StatusBadRequest { @@ -407,7 +407,7 @@ func TestHandleInternalUploadURL_ResolvesRegisteredScopedObjectID(t *testing.T) mockUM := &capturingMultipartURLManager{} om = core.NewObjectManager(database, mockUM) req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/"+registered.Id+"?file_name=program-root/"+oid, nil), + httptest.NewRequest(http.MethodGet, "/data/upload/"+registered.Id+"?key=program-root/"+oid, nil), map[string]string{"file_id": registered.Id}, ) rr := doInternalDRSTestRequest(req, om) @@ -616,7 +616,7 @@ func TestHandleInternalUploadURL_UsesExplicitObjectKeyForExistingObject(t *testi } mockUM := &capturingMultipartURLManager{} req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/7b9de5b9-19b2-536f-abcc-fe2a146c4eb5?file_name=program-root/"+checksum, nil), + httptest.NewRequest(http.MethodGet, "/data/upload/7b9de5b9-19b2-536f-abcc-fe2a146c4eb5?key=program-root/"+checksum, nil), map[string]string{"file_id": "7b9de5b9-19b2-536f-abcc-fe2a146c4eb5"}, ) rr := doInternalDRSTestRequest(req, core.NewObjectManager(db, mockUM)) @@ -670,7 +670,7 @@ func TestHandleInternalUploadURL_ExplicitScopeOverridesMalformedExistingObjectUR } mockUM := &capturingMultipartURLManager{} req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/f781273b-52eb-5ac2-a484-775235eef303?organization=syfon&project=e2e&file_name=project-subpath/"+checksum, nil), + httptest.NewRequest(http.MethodGet, "/data/upload/f781273b-52eb-5ac2-a484-775235eef303?organization=syfon&project=e2e&key=project-subpath/"+checksum, nil), map[string]string{"file_id": "f781273b-52eb-5ac2-a484-775235eef303"}, ) rr := doInternalDRSTestRequest(req, core.NewObjectManager(db, mockUM)) @@ -719,7 +719,7 @@ func TestHandleInternalUploadURL_ExplicitScopeIgnoresConflictingObjectMetadata(t } mockUM := &capturingMultipartURLManager{} req := routeutil.WithPathParams( - httptest.NewRequest(http.MethodGet, "/data/upload/4f74e0c2-3c80-5c19-b47c-061b300ae270?organization=syfon&project=e2e&file_name="+checksum, nil), + httptest.NewRequest(http.MethodGet, "/data/upload/4f74e0c2-3c80-5c19-b47c-061b300ae270?organization=syfon&project=e2e&key="+checksum, nil), map[string]string{"file_id": "4f74e0c2-3c80-5c19-b47c-061b300ae270"}, ) rr := doInternalDRSTestRequest(req, core.NewObjectManager(db, mockUM)) diff --git a/internal/api/lfs/lfs.go b/internal/api/lfs/lfs.go index 9a7a69ea..897ba128 100644 --- a/internal/api/lfs/lfs.go +++ b/internal/api/lfs/lfs.go @@ -55,10 +55,3 @@ func RegisterLFSRoutes(router fiber.Router, om *core.ObjectManager, opts ...Opti }) lfsapi.RegisterHandlers(router, strict) } - -func ResetLFSLimitersForTest() { - limitMu.Lock() - defer limitMu.Unlock() - requestWindowMap = map[string]windowCounter{} - bandwidthWindowMap = map[string]windowBytes{} -} diff --git a/internal/api/lfs/reset_test.go b/internal/api/lfs/reset_test.go new file mode 100644 index 00000000..c3e0c5d5 --- /dev/null +++ b/internal/api/lfs/reset_test.go @@ -0,0 +1,8 @@ +package lfs + +func ResetLFSLimitersForTest() { + limitMu.Lock() + defer limitMu.Unlock() + requestWindowMap = map[string]windowCounter{} + bandwidthWindowMap = map[string]windowBytes{} +} diff --git a/internal/api/metrics/file_usage.go b/internal/api/metrics/file_usage.go index ae48b9a1..73e01c25 100644 --- a/internal/api/metrics/file_usage.go +++ b/internal/api/metrics/file_usage.go @@ -3,6 +3,7 @@ package metrics import ( "context" "errors" + "log" "net/http" "sort" "strings" @@ -77,6 +78,62 @@ func (s *MetricsServer) ListMetricsFiles(ctx context.Context, request metricsapi }, nil } +func (s *MetricsServer) BulkMetricsFiles(ctx context.Context, request metricsapi.BulkMetricsFilesRequestObject) (metricsapi.BulkMetricsFilesResponseObject, error) { + started := time.Now() + if request.Body == nil { + return metricsapi.BulkMetricsFiles400Response{}, nil + } + objectIDs := uniqueNonEmptyStrings(request.Body.ObjectIds) + if len(objectIDs) == 0 { + return metricsapi.BulkMetricsFiles400Response{}, nil + } + inactiveSince, err := parseInactiveSince(request.Body.InactiveDays) + if err != nil { + return metricsapi.BulkMetricsFiles400Response{}, nil + } + + access, statusCode, ok := s.checkAuth(ctx) + if !ok { + switch statusCode { + case http.StatusUnauthorized: + return metricsapi.BulkMetricsFiles401Response{}, nil + case http.StatusForbidden: + return metricsapi.BulkMetricsFiles403Response{}, nil + default: + return metricsapi.BulkMetricsFiles400Response{}, nil + } + } + + readableObjectIDs, err := s.readableBulkObjectIDs(ctx, access, objectIDs) + if err != nil { + return metricsapi.BulkMetricsFiles500Response{}, nil + } + data, err := s.database.ListFileUsageByObjectIDs(ctx, readableObjectIDs) + if err != nil { + return metricsapi.BulkMetricsFiles500Response{}, nil + } + items := make([]metricsapi.FileUsage, 0, len(data)) + for _, usage := range data { + if !usageMatchesInactiveFilter(usage, inactiveSince) { + continue + } + items = append(items, toMetricsFileUsage(usage)) + } + + log.Printf( + "INFO: syfon_metrics_files_bulk requested=%d returned=%d scoped=%t aggregate_scopes=%d inactive_days=%t duration_ms=%d", + len(objectIDs), + len(items), + access.isScoped(), + len(access.scopes), + request.Body.InactiveDays != nil, + time.Since(started).Milliseconds(), + ) + return metricsapi.BulkMetricsFiles200JSONResponse{ + Data: &items, + }, nil +} + func (s *MetricsServer) GetMetricsFile(ctx context.Context, request metricsapi.GetMetricsFileRequestObject) (metricsapi.GetMetricsFileResponseObject, error) { objectID := request.ObjectId if objectID == "" { @@ -146,8 +203,18 @@ func (s *MetricsServer) GetMetricsSummary(ctx context.Context, request metricsap if access.isScoped() { if scopedStore, ok := s.database.(db.FileUsageScopedLister); ok { summary, err = scopedStore.GetFileUsageSummaryByScope(ctx, access.organization, access.project, inactiveSince) + if err == nil { + recordSummary, recordErr := scopedStore.GetProjectRecordSummaryByScope(ctx, access.organization, access.project) + if recordErr != nil { + err = recordErr + } else { + summary.RecordCount = recordSummary.RecordCount + summary.RecordLatestUpdatedTime = recordSummary.RecordLatestUpdatedTime + } + } } else { _, summary, err = listScopedFileUsage(ctx, s.database, s.objects, access.organization, access.project, 0, 0, inactiveSince) + summary.RecordCount = summary.TotalFiles } } else if access.hasScopeAggregate() { if scopedStore, ok := s.database.(db.FileUsageScopedLister); ok { @@ -163,13 +230,86 @@ func (s *MetricsServer) GetMetricsSummary(ctx context.Context, request metricsap } return metricsapi.GetMetricsSummary200JSONResponse{ - TotalFiles: &summary.TotalFiles, - TotalUploads: &summary.TotalUploads, - TotalDownloads: &summary.TotalDownloads, - InactiveFileCount: &summary.InactiveFileCount, + TotalFiles: &summary.TotalFiles, + TotalUploads: &summary.TotalUploads, + TotalDownloads: &summary.TotalDownloads, + InactiveFileCount: &summary.InactiveFileCount, + RecordCount: scopedSummaryInt64(access, summary.RecordCount), + RecordLatestUpdatedTime: scopedSummaryTime(access, summary.RecordLatestUpdatedTime), }, nil } +func (s *MetricsServer) readableBulkObjectIDs(ctx context.Context, access metricsAccess, objectIDs []string) ([]string, error) { + if !access.isScoped() && !access.hasScopeAggregate() { + return objectIDs, nil + } + readable := make(map[string]struct{}, len(objectIDs)) + if access.isScoped() { + ids, err := s.objects.ListObjectIDsByScope(ctx, access.organization, access.project, "read") + if err != nil { + return nil, err + } + for _, id := range ids { + readable[strings.TrimSpace(id)] = struct{}{} + } + } else { + for _, scope := range access.scopes { + ids, err := s.objects.ListObjectIDsByScope(ctx, scope.organization, scope.project, "read") + if err != nil { + return nil, err + } + for _, id := range ids { + readable[strings.TrimSpace(id)] = struct{}{} + } + } + } + out := make([]string, 0, len(objectIDs)) + for _, objectID := range objectIDs { + if _, ok := readable[objectID]; ok { + out = append(out, objectID) + } + } + return out, nil +} + +func usageMatchesInactiveFilter(usage models.FileUsage, inactiveSince *time.Time) bool { + if inactiveSince == nil { + return true + } + return usage.LastDownloadTime == nil || usage.LastDownloadTime.Before(*inactiveSince) +} + +func uniqueNonEmptyStrings(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func scopedSummaryInt64(access metricsAccess, value int64) *int64 { + if !access.isScoped() { + return nil + } + return &value +} + +func scopedSummaryTime(access metricsAccess, value *time.Time) *time.Time { + if !access.isScoped() || value == nil { + return nil + } + return value +} + func parseInactiveSince(inactiveDays *int) (*time.Time, error) { if inactiveDays == nil { return nil, nil diff --git a/internal/api/metrics/file_usage_test.go b/internal/api/metrics/file_usage_test.go index ac5f5771..1d412807 100644 --- a/internal/api/metrics/file_usage_test.go +++ b/internal/api/metrics/file_usage_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -125,6 +126,86 @@ func TestMetricsRoutes_GetNotFoundAndValidation(t *testing.T) { } } +func TestMetricsRoutes_BulkFiles(t *testing.T) { + oldDownload := time.Now().UTC().AddDate(0, 0, -40) + recentDownload := time.Now().UTC().AddDate(0, 0, -2) + db := &testutils.MockDatabase{ + Objects: map[string]*drs.DrsObject{ + "obj-a": {Id: "obj-a", Name: common.Ptr("a.txt"), Size: 10}, + "obj-b": {Id: "obj-b", Name: common.Ptr("b.txt"), Size: 20}, + "obj-c": {Id: "obj-c", Name: common.Ptr("c.txt"), Size: 30}, + }, + ObjectAuthz: map[string]map[string][]string{ + "obj-a": {"cbds": {"end_to_end_test"}}, + "obj-b": {"cbds": {"end_to_end_test"}}, + "obj-c": {"other": {"project"}}, + }, + Usage: map[string]models.FileUsage{ + "obj-a": { + ObjectID: "obj-a", + Name: "a.txt", + Size: 10, + DownloadCount: 3, + LastDownloadTime: &oldDownload, + }, + "obj-b": { + ObjectID: "obj-b", + Name: "b.txt", + Size: 20, + DownloadCount: 7, + LastDownloadTime: &recentDownload, + }, + "obj-c": { + ObjectID: "obj-c", + Name: "c.txt", + Size: 30, + DownloadCount: 11, + }, + }, + } + app := fiber.New() + app.Use(func(c fiber.Ctx) error { + if mode := c.Get("X-Test-Auth-Mode"); mode != "" { + var privs map[string]map[string]bool + if privsJSON := c.Get("X-Test-Privileges"); privsJSON != "" { + _ = json.Unmarshal([]byte(privsJSON), &privs) + } + headerRaw := c.Get("X-Test-Auth-Header") + ctx := metricsTestContext(c.Context(), mode, headerRaw != "", headerRaw == "true", privs) + c.SetContext(ctx) + } + return c.Next() + }) + RegisterMetricsRoutes(app, db) + + req := httptest.NewRequest(http.MethodPost, "/index/v1/metrics/files/bulk?organization=cbds&project=end_to_end_test", strings.NewReader(`{"object_ids":["obj-a","obj-b","obj-c","missing","obj-a"],"inactive_days":30}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Test-Auth-Mode", "gen3") + req.Header.Set("X-Test-Auth-Header", "true") + privs, _ := json.Marshal(map[string]map[string]bool{ + "/programs/cbds/projects/end_to_end_test": {"read": true}, + }) + req.Header.Set("X-Test-Privileges", string(privs)) + httpResp, err := app.Test(req) + if err != nil { + t.Fatalf("test request failed: %v", err) + } + body, _ := io.ReadAll(httpResp.Body) + if httpResp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", httpResp.StatusCode, string(body)) + } + var resp metricsapi.MetricsListResponse + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.Data == nil || len(*resp.Data) != 1 { + t.Fatalf("expected only inactive scoped object, got %s", string(body)) + } + if got := *(*resp.Data)[0].ObjectId; got != "obj-a" { + t.Fatalf("expected obj-a, got %q", got) + } +} + func TestMetricsSummaryAuthzAndScope(t *testing.T) { db := &testutils.MockDatabase{ Objects: map[string]*drs.DrsObject{ @@ -187,6 +268,9 @@ func TestMetricsSummaryAuthzAndScope(t *testing.T) { if resp.TotalFiles == nil || resp.TotalUploads == nil || resp.TotalDownloads == nil || *resp.TotalFiles != 1 || *resp.TotalUploads != 2 || *resp.TotalDownloads != 3 { t.Fatalf("unexpected scoped summary: %+v", resp) } + if resp.RecordCount == nil || *resp.RecordCount != 1 { + t.Fatalf("expected exact scoped record count, got %+v", resp.RecordCount) + } }) t.Run("missing auth header returns 401", func(t *testing.T) { @@ -395,4 +479,3 @@ func TestListMultiScopedFileUsage_DeduplicatesAcrossScopes(t *testing.T) { t.Fatalf("expected summary total files > 0, got %+v", summary) } } - diff --git a/internal/api/metrics/transfer_ingest.go b/internal/api/metrics/transfer_ingest.go index 55e13edc..77d2d324 100644 --- a/internal/api/metrics/transfer_ingest.go +++ b/internal/api/metrics/transfer_ingest.go @@ -9,45 +9,12 @@ import ( "github.com/calypr/syfon/apigen/server/metricsapi" sycommon "github.com/calypr/syfon/common" - "github.com/calypr/syfon/internal/api/attribution" apimiddleware "github.com/calypr/syfon/internal/api/middleware" - internalauth "github.com/calypr/syfon/internal/auth" "github.com/calypr/syfon/internal/authz" intcommon "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" ) -type transferEventsRequest struct { - Events []transferEventPayload `json:"events"` -} - -type transferEventPayload struct { - EventID string `json:"event_id"` - EventType string `json:"event_type"` - Direction string `json:"direction"` - EventTime string `json:"event_time"` - RequestID string `json:"request_id"` - ObjectID string `json:"object_id"` - SHA256 string `json:"sha256"` - ObjectSize int64 `json:"object_size"` - Organization string `json:"organization"` - Project string `json:"project"` - AccessID string `json:"access_id"` - Provider string `json:"provider"` - Bucket string `json:"bucket"` - StorageURL string `json:"storage_url"` - RangeStart *int64 `json:"range_start"` - RangeEnd *int64 `json:"range_end"` - BytesRequested int64 `json:"bytes_requested"` - BytesCompleted int64 `json:"bytes_completed"` - ActorEmail string `json:"actor_email"` - ActorSubject string `json:"actor_subject"` - AuthMode string `json:"auth_mode"` - ClientName string `json:"client_name"` - ClientVersion string `json:"client_version"` - TransferSessionID string `json:"transfer_session_id"` -} - type providerTransferPayload struct { ProviderEventID string `json:"provider_event_id"` AccessGrantID string `json:"access_grant_id"` @@ -133,83 +100,6 @@ func providerTransferResource(organization, project string) (string, bool) { return resource, true } -func transferPayloadToModel(ctx context.Context, item transferEventPayload) (models.TransferAttributionEvent, error) { - eventType := strings.TrimSpace(item.EventType) - switch eventType { - case models.TransferEventAccessIssued: - default: - return models.TransferAttributionEvent{}, errors.New("invalid event_type") - } - when := time.Now().UTC() - if strings.TrimSpace(item.EventTime) != "" { - parsed, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(item.EventTime)) - if err != nil { - return models.TransferAttributionEvent{}, errors.New("invalid event_time") - } - when = parsed.UTC() - } - if item.BytesCompleted < 0 || item.BytesRequested < 0 { - return models.TransferAttributionEvent{}, errors.New("bytes cannot be negative") - } - if eventType != models.TransferEventAccessIssued && item.BytesCompleted <= 0 { - return models.TransferAttributionEvent{}, errors.New("bytes_completed is required") - } - ev := models.TransferAttributionEvent{ - EventID: strings.TrimSpace(item.EventID), - EventType: eventType, - Direction: normalizeTransferDirection(item.Direction), - EventTime: when, - RequestID: strings.TrimSpace(item.RequestID), - ObjectID: strings.TrimSpace(item.ObjectID), - SHA256: strings.TrimSpace(item.SHA256), - ObjectSize: item.ObjectSize, - Organization: strings.TrimSpace(item.Organization), - Project: strings.TrimSpace(item.Project), - AccessID: strings.TrimSpace(item.AccessID), - Provider: strings.TrimSpace(item.Provider), - Bucket: strings.TrimSpace(item.Bucket), - StorageURL: strings.TrimSpace(item.StorageURL), - RangeStart: item.RangeStart, - RangeEnd: item.RangeEnd, - BytesRequested: item.BytesRequested, - BytesCompleted: item.BytesCompleted, - ActorEmail: strings.TrimSpace(item.ActorEmail), - ActorSubject: strings.TrimSpace(item.ActorSubject), - AuthMode: strings.TrimSpace(item.AuthMode), - ClientName: strings.TrimSpace(item.ClientName), - ClientVersion: strings.TrimSpace(item.ClientVersion), - TransferSessionID: strings.TrimSpace(item.TransferSessionID), - } - if ev.RequestID == "" { - ev.RequestID = intcommon.GetRequestID(ctx) - } - if ev.ActorEmail == "" { - ev.ActorEmail = attribution.ActorEmail(ctx) - } - if ev.ActorSubject == "" { - ev.ActorSubject = attribution.ActorSubject(ctx) - } - if ev.AuthMode == "" { - ev.AuthMode = strings.TrimSpace(internalauth.FromContext(ctx).Mode) - } - if ev.EventID == "" { - ev.EventID = attribution.EventID(ev) - } - if ev.AccessGrantID == "" { - ev.AccessGrantID = attribution.AccessGrantID(ev) - } - return ev, nil -} - -func normalizeTransferDirection(direction string) string { - switch strings.ToLower(strings.TrimSpace(direction)) { - case models.ProviderTransferDirectionUpload: - return models.ProviderTransferDirectionUpload - default: - return models.ProviderTransferDirectionDownload - } -} - func providerTransferPayloadToModel(item providerTransferPayload) (models.ProviderTransferEvent, error) { direction := strings.ToLower(strings.TrimSpace(item.Direction)) switch direction { diff --git a/internal/api/metrics/transfer_ingest_test.go b/internal/api/metrics/transfer_ingest_test.go deleted file mode 100644 index f2d03a0b..00000000 --- a/internal/api/metrics/transfer_ingest_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package metrics - -import ( - "context" - "net/http" - "testing" - "time" - - "github.com/calypr/syfon/apigen/server/metricsapi" - internalauth "github.com/calypr/syfon/internal/auth" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" -) - -func ptr[T any](v T) *T { return &v } - -func TestNormalizeTransferDirection(t *testing.T) { - if got := normalizeTransferDirection("upload"); got != models.ProviderTransferDirectionUpload { - t.Fatalf("expected upload, got %q", got) - } - if got := normalizeTransferDirection("something-else"); got != models.ProviderTransferDirectionDownload { - t.Fatalf("expected default download, got %q", got) - } -} - -func TestTransferPayloadToModel(t *testing.T) { - ctx := common.WithRequestID(context.Background(), "req-1") - session := internalauth.NewSession("gen3") - session.SetSubject("user-sub") - ctx = internalauth.WithSession(ctx, session) - - payload := transferEventPayload{ - EventType: models.TransferEventAccessIssued, - Direction: "download", - EventTime: time.Now().UTC().Format(time.RFC3339Nano), - ObjectID: "did-1", - BytesRequested: 10, - } - got, err := transferPayloadToModel(ctx, payload) - if err != nil { - t.Fatalf("transferPayloadToModel error: %v", err) - } - if got.EventType != models.TransferEventAccessIssued || got.Direction != models.ProviderTransferDirectionDownload { - t.Fatalf("unexpected mapped event: %+v", got) - } - if got.RequestID != "req-1" { - t.Fatalf("expected request id fallback, got %q", got.RequestID) - } - if got.EventID == "" || got.AccessGrantID == "" { - t.Fatalf("expected generated ids, got event=%q grant=%q", got.EventID, got.AccessGrantID) - } - - payload.BytesRequested = -1 - if _, err := transferPayloadToModel(ctx, payload); err == nil { - t.Fatalf("expected negative-bytes validation error") - } -} - -func TestCheckProviderMetricsIngestAuth(t *testing.T) { - ctxNonGen3 := metricsTestContext(context.Background(), "local", true, true, nil) - body := &metricsapi.RecordProviderTransferEventsJSONRequestBody{ - Events: []metricsapi.ProviderTransferEvent{{ - Organization: ptr("org-a"), - Project: ptr("proj-a"), - }}, - } - if status, ok := checkProviderMetricsIngestAuth(ctxNonGen3, body); !ok || status != 0 { - t.Fatalf("expected non-gen3 auth allowed, got status=%d ok=%v", status, ok) - } - - ctxMissingHeader := metricsTestContext(context.Background(), "gen3", false, false, nil) - if status, ok := checkProviderMetricsIngestAuth(ctxMissingHeader, body); ok || status != http.StatusUnauthorized { - t.Fatalf("expected unauthorized for missing header, got status=%d ok=%v", status, ok) - } - - ctxForbidden := metricsTestContext(context.Background(), "gen3", true, true, map[string]map[string]bool{}) - if status, ok := checkProviderMetricsIngestAuth(ctxForbidden, body); ok || status != http.StatusForbidden { - t.Fatalf("expected forbidden without ingest privilege, got status=%d ok=%v", status, ok) - } - - ctxAllowed := metricsTestContext(context.Background(), "gen3", true, true, map[string]map[string]bool{ - "/programs/org-a/projects/proj-a": {"create": true}, - }) - if status, ok := checkProviderMetricsIngestAuth(ctxAllowed, body); !ok || status != 0 { - t.Fatalf("expected allowed with scoped create privilege, got status=%d ok=%v", status, ok) - } - - ctxOrgAllowed := metricsTestContext(context.Background(), "gen3", true, true, map[string]map[string]bool{ - "/programs/org-a": {"update": true}, - }) - orgBody := &metricsapi.RecordProviderTransferEventsJSONRequestBody{ - Events: []metricsapi.ProviderTransferEvent{{ - Organization: ptr("org-a"), - }}, - } - if status, ok := checkProviderMetricsIngestAuth(ctxOrgAllowed, orgBody); !ok || status != 0 { - t.Fatalf("expected allowed with org-scoped update privilege, got status=%d ok=%v", status, ok) - } -} - -func TestRecordProviderTransferEventsAuthResponse(t *testing.T) { - if _, ok := recordProviderTransferEventsAuthResponse(http.StatusUnauthorized).(metricsapi.RecordProviderTransferEvents401Response); !ok { - t.Fatalf("expected 401 response object") - } - if _, ok := recordProviderTransferEventsAuthResponse(http.StatusForbidden).(metricsapi.RecordProviderTransferEvents403Response); !ok { - t.Fatalf("expected 403 response object") - } - if _, ok := recordProviderTransferEventsAuthResponse(http.StatusBadRequest).(metricsapi.RecordProviderTransferEvents400Response); !ok { - t.Fatalf("expected 400 response object") - } -} diff --git a/internal/api/middleware/middleware_test.go b/internal/api/middleware/middleware_test.go index 93126598..0e274768 100644 --- a/internal/api/middleware/middleware_test.go +++ b/internal/api/middleware/middleware_test.go @@ -276,7 +276,7 @@ func TestGen3ModePopulatesUserPrivilegesFromFence(t *testing.T) { t.Fatalf("sign token: %v", err) } - m := NewAuthzMiddleware(newTestLogger(), "gen3", "", "") + m := NewAuthzMiddleware(slog.Default(), "gen3", "", "") injectDummyAuthenticationPluginManager(m, true) app := fiber.New() app.Use(m.FiberMiddleware()) diff --git a/internal/api/middleware/plugin_clients.go b/internal/api/middleware/plugin_clients.go index 9374f4b2..7eb6b616 100644 --- a/internal/api/middleware/plugin_clients.go +++ b/internal/api/middleware/plugin_clients.go @@ -148,12 +148,5 @@ func (a *AuthzRPC) Authorize(ctx context.Context, in *plugin.AuthorizationInput) return &out, err } -// DummyPluginManager implements the same interface as PluginManager for testing. -type DummyPluginManager struct{} - -func (d *DummyPluginManager) Authorize(ctx context.Context, in *plugin.AuthorizationInput) (*plugin.AuthorizationOutput, error) { - return &plugin.AuthorizationOutput{Allow: true}, nil -} - // Ensure PluginManager implements pluginManagerInterface var _ pluginManagerInterface = (*PluginManager)(nil) diff --git a/internal/api/middleware/plugin_clients_test.go b/internal/api/middleware/plugin_clients_test.go index 5bb5599a..b6927316 100644 --- a/internal/api/middleware/plugin_clients_test.go +++ b/internal/api/middleware/plugin_clients_test.go @@ -2,215 +2,13 @@ package middleware import ( "context" - "errors" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "testing" "github.com/calypr/syfon/plugin" - "github.com/gofiber/fiber/v3" ) -type mockPlugin struct { - output *plugin.AuthorizationOutput - err error -} - -func (m *mockPlugin) Authorize(ctx context.Context, in *plugin.AuthorizationInput) (*plugin.AuthorizationOutput, error) { - return m.output, m.err -} - -func addRecovery(app *fiber.App, t *testing.T) { - app.Use(func(c fiber.Ctx) error { - defer func() { - if r := recover(); r != nil { - t.Fatalf("panic: %v", r) - } - }() - return c.Next() - }) -} - -func newTestLogger() *slog.Logger { - return slog.New(slog.NewTextHandler(io.Discard, nil)) -} - -func TestPluginIntegrationAllow(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{ - Allow: true, - Obligations: map[string]interface{}{ - "resources": []interface{}{"/foo", "/bar"}, - "privileges": map[string]interface{}{ - "/foo": []interface{}{"read", "update"}, - "/bar": []interface{}{"read"}, - }, - }, - }, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } -} - -func TestPluginIntegrationDeny(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{Allow: false, Reason: "denied"}, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 403 { - t.Fatalf("expected 403, got %d", resp.StatusCode) - } -} - -func TestPluginIntegrationError(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: nil, - err: errors.New("plugin error"), - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 401 { - t.Fatalf("expected 401, got %d", resp.StatusCode) - } -} - -func TestPluginIntegration_MissingClaims(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{Allow: true, Obligations: map[string]interface{}{}}, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer invalidtoken") // Not a real JWT - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } -} - -func TestPluginIntegration_PluginUnavailable(t *testing.T) { - mw := &AuthzMiddleware{pluginManager: nil, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 401 { - t.Fatalf("expected 401, got %d", resp.StatusCode) - } -} - -func TestPluginIntegration_UnexpectedObligations(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{ - Allow: true, - Obligations: map[string]interface{}{ - "resources": 123, // Not a []interface{} - "privileges": "not a map", - }, - }, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - req.Header.Set("Authorization", "Bearer testtoken") - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } -} +// DummyPluginManager implements the same interface as PluginManager for testing. +type DummyPluginManager struct{} -func TestPluginIntegration_NoAuthHeader(t *testing.T) { - pm := &PluginManager{client: nil} - pm.client = &PluginClient{raw: &mockPlugin{ - output: &plugin.AuthorizationOutput{Allow: true, Obligations: map[string]interface{}{}}, - err: nil, - }} - mw := &AuthzMiddleware{pluginManager: pm, mode: "gen3", logger: newTestLogger()} - app := fiber.New() - addRecovery(app, t) - app.Use(mw.FiberMiddleware()) - app.Get("/foo", func(c fiber.Ctx) error { - return c.SendStatus(200) - }) - req := httptest.NewRequest(http.MethodGet, "/foo", nil) - // No Authorization header - resp, err := app.Test(req) - if err != nil { - t.Fatalf("test request failed: %v", err) - } - if resp.StatusCode != 200 { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } +func (d *DummyPluginManager) Authorize(ctx context.Context, in *plugin.AuthorizationInput) (*plugin.AuthorizationOutput, error) { + return &plugin.AuthorizationOutput{Allow: true}, nil } diff --git a/internal/api/routeutil/routeutil.go b/internal/api/routeutil/routeutil.go index 02620cba..a85836fc 100644 --- a/internal/api/routeutil/routeutil.go +++ b/internal/api/routeutil/routeutil.go @@ -4,7 +4,6 @@ import ( "context" "net/http" "regexp" - "strings" "github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3/middleware/adaptor" @@ -31,18 +30,6 @@ func WithPathParams(r *http.Request, params map[string]string) *http.Request { return r.WithContext(context.WithValue(r.Context(), pathParamsKey{}, copyParams)) } -// PathParam reads a path parameter that was attached with WithPathParams. -func PathParam(r *http.Request, key string) string { - if r == nil { - return "" - } - params, ok := r.Context().Value(pathParamsKey{}).(map[string]string) - if !ok { - return "" - } - return strings.TrimSpace(params[key]) -} - // Handler adapts a standard net/http handler to fiber while preserving path params. func Handler(h any, paramNames ...string) fiber.Handler { return func(c fiber.Ctx) error { diff --git a/internal/common/browse_path.go b/internal/common/browse_path.go deleted file mode 100644 index 12b5e100..00000000 --- a/internal/common/browse_path.go +++ /dev/null @@ -1,96 +0,0 @@ -package common - -import ( - "fmt" - "strings" -) - -type BrowsePathInfo struct { - Normalized string - ParentPath string - EntryName string - Segments []string -} - -func NormalizeBrowsePath(raw string) (string, []string, error) { - trimmed := strings.TrimSpace(raw) - if trimmed == "" { - return "", nil, nil - } - trimmed = strings.ReplaceAll(trimmed, "\\", "/") - parts := strings.Split(trimmed, "/") - segments := make([]string, 0, len(parts)) - for _, part := range parts { - segment := strings.TrimSpace(part) - if segment == "" { - continue - } - if segment == "." || segment == ".." { - return "", nil, fmt.Errorf("path must not contain '.' or '..' segments") - } - segments = append(segments, segment) - } - return strings.Join(segments, "/"), segments, nil -} - -func BrowsePathInfoFromName(raw string) (BrowsePathInfo, bool, error) { - normalized, segments, err := NormalizeBrowsePath(raw) - if err != nil { - return BrowsePathInfo{}, false, err - } - if normalized == "" || len(segments) == 0 { - return BrowsePathInfo{}, false, nil - } - parentPath := "" - if len(segments) > 1 { - parentPath = strings.Join(segments[:len(segments)-1], "/") - } - return BrowsePathInfo{ - Normalized: normalized, - ParentPath: parentPath, - EntryName: segments[len(segments)-1], - Segments: segments, - }, true, nil -} - -func HasBrowsePathPrefix(candidate, prefix []string) bool { - if len(prefix) > len(candidate) { - return false - } - for i, segment := range prefix { - if candidate[i] != segment { - return false - } - } - return true -} - -func ImmediateBrowseDirectory(requestedPath, candidatePath string) (BrowsePathInfo, bool) { - requestedNormalized, requestedSegments, err := NormalizeBrowsePath(requestedPath) - if err != nil { - return BrowsePathInfo{}, false - } - candidateInfo, ok, err := BrowsePathInfoFromName(candidatePath) - if err != nil || !ok { - return BrowsePathInfo{}, false - } - if !HasBrowsePathPrefix(candidateInfo.Segments, requestedSegments) { - return BrowsePathInfo{}, false - } - remaining := candidateInfo.Segments[len(requestedSegments):] - if len(remaining) < 2 { - return BrowsePathInfo{}, false - } - dirSegments := append(append([]string{}, requestedSegments...), remaining[0]) - dirPath := strings.Join(dirSegments, "/") - parentPath := "" - if requestedNormalized != "" { - parentPath = requestedNormalized - } - return BrowsePathInfo{ - Normalized: dirPath, - ParentPath: parentPath, - EntryName: remaining[0], - Segments: dirSegments, - }, true -} diff --git a/internal/common/browse_path_test.go b/internal/common/browse_path_test.go deleted file mode 100644 index 88512595..00000000 --- a/internal/common/browse_path_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package common - -import "testing" - -func TestBrowsePathInfoFromName(t *testing.T) { - info, ok, err := BrowsePathInfoFromName(` nested\\dir//file.txt `) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !ok { - t.Fatal("expected valid browse path info") - } - if info.Normalized != "nested/dir/file.txt" { - t.Fatalf("unexpected normalized path: %q", info.Normalized) - } - if info.ParentPath != "nested/dir" { - t.Fatalf("unexpected parent path: %q", info.ParentPath) - } - if info.EntryName != "file.txt" { - t.Fatalf("unexpected entry name: %q", info.EntryName) - } -} - -func TestNormalizeBrowsePathRejectsDotSegments(t *testing.T) { - if _, _, err := NormalizeBrowsePath("../nested"); err == nil { - t.Fatal("expected invalid path error") - } -} - -func TestImmediateBrowseDirectory(t *testing.T) { - dir, ok := ImmediateBrowseDirectory("nested", "nested/deep/file.txt") - if !ok { - t.Fatal("expected directory to be derived") - } - if dir.Normalized != "nested/deep" { - t.Fatalf("unexpected directory path: %q", dir.Normalized) - } - if dir.EntryName != "deep" { - t.Fatalf("unexpected directory name: %q", dir.EntryName) - } -} diff --git a/internal/common/checksum.go b/internal/common/checksum.go index 827818d0..9e993b6d 100644 --- a/internal/common/checksum.go +++ b/internal/common/checksum.go @@ -106,23 +106,7 @@ func CanonicalSHA256(checksums []drs.Checksum) (string, bool) { return "", false } -// NormalizeSHA256 cleans and dedups a list of potential SHA256 hashes. -func NormalizeSHA256(values []string) []string { - targets := make([]string, 0, len(values)) - seen := make(map[string]struct{}, len(values)) - for _, raw := range values { - sha := strings.ToLower(strings.TrimSpace(NormalizeChecksum(strings.TrimSpace(raw)))) - if sha == "" { - continue - } - if _, ok := seen[sha]; ok { - continue - } - seen[sha] = struct{}{} - targets = append(targets, sha) - } - return targets -} + // ParseS3URL extracts bucket/key pairs from an s3:// URL. func ParseS3URL(raw string) (bucket string, key string, ok bool) { diff --git a/internal/common/checksum_test.go b/internal/common/checksum_test.go index 9d0aff0a..77e558b2 100644 --- a/internal/common/checksum_test.go +++ b/internal/common/checksum_test.go @@ -57,10 +57,7 @@ func TestSHAHelpers(t *testing.T) { t.Fatalf("CanonicalSHA256 mismatch: ok=%v sha=%q", ok, sha) } - norm := NormalizeSHA256([]string{"sha256:ABC", "abc", "abc", ""}) - if len(norm) != 1 || norm[0] != "abc" { - t.Fatalf("NormalizeSHA256 unexpected output: %+v", norm) - } + obj := models.InternalObject{DrsObject: drs.DrsObject{Checksums: []drs.Checksum{{Type: "md5", Checksum: "m"}}}} if !ObjectHasChecksumTypeAndValue(obj, "md5", "m") { diff --git a/internal/common/cloud.go b/internal/common/cloud.go index 61cc9f75..0b7b53f2 100644 --- a/internal/common/cloud.go +++ b/internal/common/cloud.go @@ -4,11 +4,8 @@ import ( "fmt" "net" "net/url" - "path/filepath" "regexp" "strings" - - "github.com/calypr/syfon/internal/models" ) const ( @@ -77,33 +74,7 @@ func ProviderToScheme(p string) string { } } -func ObjectURLForCredential(cred *models.S3Credential, key string) (string, error) { - if cred == nil { - return "", fmt.Errorf("credential is required") - } - cleanKey := strings.TrimPrefix(strings.TrimSpace(key), "/") - provider := NormalizeProvider(cred.Provider, S3Provider) - switch provider { - case S3Provider: - return fmt.Sprintf("%s%s/%s", S3Prefix, cred.Bucket, cleanKey), nil - case GCSProvider: - return fmt.Sprintf("%s%s/%s", GCSPrefix, cred.Bucket, cleanKey), nil - case AzureProvider: - return fmt.Sprintf("%s%s/%s", AzurePrefix, cred.Bucket, cleanKey), nil - case FileProvider: - root := filepath.Clean(strings.TrimSpace(cred.Endpoint)) - if root == "." || root == "" { - root = strings.TrimPrefix(strings.TrimSpace(cred.Bucket), "/") - } - if root == "" { - return "", fmt.Errorf("file provider requires an endpoint or bucket root") - } - return filepath.ToSlash(filepath.Join(root, cleanKey)), nil - default: - return "", fmt.Errorf("unsupported provider: %s", provider) - } -} func NormalizeStoragePath(rawPath, bucket string) (string, error) { p := strings.TrimSpace(rawPath) @@ -149,9 +120,7 @@ func ParseBucketProvider(raw string) (string, error) { // The rules are intentionally provider-specific: // - s3 and azure share the stricter DNS-style naming rules. // - gcs permits dots and underscores but still requires a DNS-safe shape. -func ValidateBucketName(providerName, bucketName string) error { - return ValidateBucketNameWithEndpoint(providerName, bucketName, "") -} + // ValidateBucketNameWithEndpoint validates a bucket/container name for the // given provider and endpoint. S3-compatible backends with a custom endpoint diff --git a/internal/common/cloud_test.go b/internal/common/cloud_test.go index 4c8ff68c..cba03f36 100644 --- a/internal/common/cloud_test.go +++ b/internal/common/cloud_test.go @@ -61,7 +61,7 @@ func TestValidateBucketName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := ValidateBucketName(tt.provider, tt.bucket) + err := ValidateBucketNameWithEndpoint(tt.provider, tt.bucket, "") if tt.wantErr && err == nil { t.Fatalf("expected error for provider=%q bucket=%q", tt.provider, tt.bucket) } diff --git a/internal/common/errors.go b/internal/common/errors.go index 79b8c816..58276b74 100644 --- a/internal/common/errors.go +++ b/internal/common/errors.go @@ -90,6 +90,3 @@ func IsNotFoundError(err error) bool { return errors.Is(err, ErrNotFound) } -func IsUnauthorizedError(err error) bool { - return errors.Is(err, ErrUnauthorized) -} diff --git a/internal/common/errors_test.go b/internal/common/errors_test.go index c37ac9b1..2f82c5e5 100644 --- a/internal/common/errors_test.go +++ b/internal/common/errors_test.go @@ -11,9 +11,6 @@ func TestAuthorizationError_UnwrapAndClassifiers(t *testing.T) { if !errors.Is(err, ErrUnauthorized) { t.Fatalf("expected AuthorizationError to unwrap to ErrUnauthorized") } - if !IsUnauthorizedError(err) { - t.Fatalf("expected IsUnauthorizedError=true") - } } func TestAuthorizationError_PublicMessage(t *testing.T) { diff --git a/internal/common/ptr.go b/internal/common/ptr.go index c8624842..dd119734 100644 --- a/internal/common/ptr.go +++ b/internal/common/ptr.go @@ -7,13 +7,7 @@ func Ptr[T any](v T) *T { return &v } -// Val returns the value of the pointer if not nil, otherwise the default value. -func Val[T any](p *T, def T) T { - if p == nil { - return def - } - return *p -} + // TimeVal returns the time value if not nil, otherwise zero time. func TimeVal(p *time.Time) time.Time { @@ -44,13 +38,7 @@ func DerefStringSlice(p *[]string) []string { return append([]string(nil), (*p)...) } -// IntVal returns the int value if not nil, otherwise 0. -func IntVal(p *int) int { - if p == nil { - return 0 - } - return *p -} + // Int64Val returns the int64 value if not nil, otherwise 0. func Int64Val(p *int64) int64 { @@ -60,10 +48,4 @@ func Int64Val(p *int64) int64 { return *p } -// FloatVal returns the float64 value if not nil, otherwise 0.0. -func FloatVal(p *float64) float64 { - if p == nil { - return 0.0 - } - return *p -} + diff --git a/internal/common/routes.go b/internal/common/routes.go index 569fc2b9..feaac418 100644 --- a/internal/common/routes.go +++ b/internal/common/routes.go @@ -1,19 +1,28 @@ package common const ( - RouteInternalDownload = "/data/download/{file_id}" - RouteInternalDownloadPart = "/data/download/{file_id}/part" - RouteInternalUpload = "/data/upload" - RouteInternalUploadURL = "/data/upload/{file_id}" - RouteInternalUploadBulk = "/data/upload/bulk" - RouteInternalMultipartInit = "/data/multipart/init" - RouteInternalMultipartUpload = "/data/multipart/upload" - RouteInternalMultipartComplete = "/data/multipart/complete" - RouteInternalInspectObject = "/data/inspect" - RouteInternalBuckets = "/data/buckets" - RouteInternalBucketDetail = "/data/buckets/{bucket}" - RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" - RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" + RouteInternalDownload = "/data/download/{file_id}" + RouteInternalDownloadPart = "/data/download/{file_id}/part" + RouteInternalUpload = "/data/upload" + RouteInternalUploadURL = "/data/upload/{file_id}" + RouteInternalUploadBulk = "/data/upload/bulk" + RouteInternalMultipartInit = "/data/multipart/init" + RouteInternalMultipartUpload = "/data/multipart/upload" + RouteInternalMultipartComplete = "/data/multipart/complete" + RouteInternalInspectObject = "/data/inspect" + RouteInternalInspectObjectBulk = "/data/inspect/bulk" + RouteInternalInspectObjectBulkList = "/data/inspect/bulk-list" + RouteInternalInspectProjectBucket = "/data/inspect/project-bucket" + RouteInternalInspectProjectBucketInventory = "/data/inspect/project-bucket/inventory" + RouteInternalInspectProjectRecords = "/data/inspect/project-records" + RouteInternalInspectProjectScopes = "/data/inspect/project-scopes" + RouteInternalDeleteProjectBucketObjects = "/data/inspect/project-bucket/delete" + RouteInternalBuckets = "/data/buckets" + RouteInternalBucketDetail = "/data/buckets/{bucket}" + RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" + RouteInternalProjectCleanup = "/data/projects/{organization}/{project_id}" + RouteInternalRepairScopeAudit = "/data/repair/project-scope/audit" + RouteInternalRepairScopeApply = "/data/repair/project-scope/apply" RouteInternalIndex = "/index" RouteInternalIndexDetail = "/index/{id}" @@ -21,6 +30,7 @@ const ( RouteInternalBulkHashes = "/index/bulk/hashes" RouteInternalBulkDeleteHashes = "/index/bulk/delete" RouteInternalBulkSHA256 = "/index/bulk/sha256/validity" + RouteInternalBulkSHA256Missing = "/index/bulk/sha256/missing" RouteInternalBulkCreate = "/index/bulk" RouteInternalBulkDocs = "/index/bulk/documents" ) diff --git a/internal/common/util.go b/internal/common/util.go index c2e72b77..7f4786b7 100644 --- a/internal/common/util.go +++ b/internal/common/util.go @@ -2,41 +2,41 @@ package common import ( "fmt" + "path/filepath" + "sort" "strings" ) -// UniqueStrings returns a deduped slice of strings, preserving order. -func UniqueStrings(values []string) []string { - seen := make(map[string]struct{}, len(values)) - out := make([]string, 0, len(values)) - for _, v := range values { - if v == "" { - continue - } - if _, exists := seen[v]; exists { - continue - } - seen[v] = struct{}{} - out = append(out, v) +// CleanToBasename extracts the basename from a path (handling both windows and unix separators). +func CleanToBasename(name string) string { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return "" } - return out + trimmed = strings.ReplaceAll(trimmed, "\\", "/") + base := filepath.Base(trimmed) + if base == "." || base == "/" || base == "" { + base = trimmed + } + return base } -// UniqueStringsCaseInsensitive returns a deduped slice of strings based on lowercase comparison, preserving the first-seen original string. -func UniqueStringsCaseInsensitive(values []string) []string { - seen := make(map[string]struct{}, len(values)) - out := make([]string, 0, len(values)) - for _, v := range values { - normalized := strings.ToLower(strings.TrimSpace(v)) - if normalized == "" { +func NormalizeNameAliases(primary string, aliases []string) []string { + primary = CleanToBasename(primary) + seen := make(map[string]struct{}, len(aliases)+1) + out := make([]string, 0, len(aliases)) + for _, alias := range aliases { + name := CleanToBasename(alias) + if name == "" || name == primary { continue } - if _, exists := seen[normalized]; exists { + if _, ok := seen[name]; ok { continue } - seen[normalized] = struct{}{} - out = append(out, v) + seen[name] = struct{}{} + out = append(out, name) } + sort.Strings(out) return out } @@ -48,15 +48,6 @@ func SchemeFromURL(raw string) string { return "" } -// NormalizeUploadKey ensures a key is valid for upload and defaults to ID if empty. -func NormalizeUploadKey(inputKey, id string) string { - k := strings.TrimSpace(inputKey) - if k == "" { - return id - } - return k -} - // BucketToURL converts a bucket and key to an s3:// URL. func BucketToURL(bucket, key string) string { return fmt.Sprintf("s3://%s/%s", strings.TrimPrefix(bucket, "s3://"), strings.TrimPrefix(key, "/")) diff --git a/internal/common/util_test.go b/internal/common/util_test.go index cfbc6c88..92cf10ef 100644 --- a/internal/common/util_test.go +++ b/internal/common/util_test.go @@ -1,52 +1,10 @@ package common import ( - "reflect" "testing" ) -func TestUniqueStrings(t *testing.T) { - tests := []struct { - name string - input []string - expected []string - }{ - {"Empty", []string{}, []string{}}, - {"No Duplicates", []string{"a", "b", "c"}, []string{"a", "b", "c"}}, - {"With Duplicates", []string{"a", "b", "a", "c", "b"}, []string{"a", "b", "c"}}, - {"With Empty Strings", []string{"a", "", "b", ""}, []string{"a", "b"}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := UniqueStrings(tt.input); !reflect.DeepEqual(got, tt.expected) { - t.Errorf("UniqueStrings() = %v, want %v", got, tt.expected) - } - }) - } -} -func TestUniqueStringsCaseInsensitive(t *testing.T) { - tests := []struct { - name string - input []string - expected []string - }{ - {"Empty", []string{}, []string{}}, - {"No Duplicates", []string{"a", "b", "c"}, []string{"a", "b", "c"}}, - {"With Duplicates Case", []string{"A", "a", "B", "b"}, []string{"A", "B"}}, - {"With Spacing", []string{" a ", "a", " B"}, []string{" a ", " B"}}, - {"With Empty Strings", []string{" ", ""}, []string{}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := UniqueStringsCaseInsensitive(tt.input); !reflect.DeepEqual(got, tt.expected) { - t.Errorf("UniqueStringsCaseInsensitive() = %v, want %v", got, tt.expected) - } - }) - } -} func TestSchemeFromURL(t *testing.T) { tests := []struct { @@ -70,45 +28,49 @@ func TestSchemeFromURL(t *testing.T) { } } -func TestNormalizeUploadKey(t *testing.T) { + + +func TestBucketToURL(t *testing.T) { tests := []struct { name string + bucket string key string - id string expected string }{ - {"Normal Key", "my-key", "my-id", "my-key"}, - {"Empty Key", "", "my-id", "my-id"}, - {"Space Key", " ", "my-id", "my-id"}, - {"Untrimmed Key", " my-key ", "my-id", "my-key"}, + {"Normal", "my-bucket", "my-key", "s3://my-bucket/my-key"}, + {"Bucket With Scheme", "s3://my-bucket", "my-key", "s3://my-bucket/my-key"}, + {"Key With Leading Slash", "my-bucket", "/my-key", "s3://my-bucket/my-key"}, + {"Both With Extras", "s3://my-bucket", "/my-key", "s3://my-bucket/my-key"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := NormalizeUploadKey(tt.key, tt.id); got != tt.expected { - t.Errorf("NormalizeUploadKey() = %v, want %v", got, tt.expected) + if got := BucketToURL(tt.bucket, tt.key); got != tt.expected { + t.Errorf("BucketToURL() = %v, want %v", got, tt.expected) } }) } } -func TestBucketToURL(t *testing.T) { +func TestCleanToBasename(t *testing.T) { tests := []struct { name string - bucket string - key string + input string expected string }{ - {"Normal", "my-bucket", "my-key", "s3://my-bucket/my-key"}, - {"Bucket With Scheme", "s3://my-bucket", "my-key", "s3://my-bucket/my-key"}, - {"Key With Leading Slash", "my-bucket", "/my-key", "s3://my-bucket/my-key"}, - {"Both With Extras", "s3://my-bucket", "/my-key", "s3://my-bucket/my-key"}, + {"Empty", "", ""}, + {"Spaces", " ", ""}, + {"Unix path", "/foo/bar/baz.txt", "baz.txt"}, + {"Windows path", `C:\foo\bar\baz.txt`, "baz.txt"}, + {"Relative path", "foo/bar.txt", "bar.txt"}, + {"No path", "baz.txt", "baz.txt"}, + {"Slash end", "foo/bar/", "bar"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := BucketToURL(tt.bucket, tt.key); got != tt.expected { - t.Errorf("BucketToURL() = %v, want %v", got, tt.expected) + if got := CleanToBasename(tt.input); got != tt.expected { + t.Errorf("CleanToBasename(%q) = %v, want %v", tt.input, got, tt.expected) } }) } diff --git a/internal/config/config.go b/internal/config/config.go index a0bef340..918c0df5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -59,13 +59,14 @@ const ( RouteInternalBucketScopes = "/data/buckets/{bucket}/scopes" // Internal DRS Index - RouteInternalIndex = "/index" - RouteInternalIndexDetail = "/index/{id}" - RouteInternalBulkHashes = "/index/bulk/hashes" - RouteInternalBulkDeleteHashes = "/index/bulk/delete" - RouteInternalBulkSHA256 = "/index/bulk/sha256/validity" - RouteInternalBulkCreate = "/index/bulk" - RouteInternalBulkDocs = "/index/bulk/documents" + RouteInternalIndex = "/index" + RouteInternalIndexDetail = "/index/{id}" + RouteInternalBulkHashes = "/index/bulk/hashes" + RouteInternalBulkDeleteHashes = "/index/bulk/delete" + RouteInternalBulkSHA256 = "/index/bulk/sha256/validity" + RouteInternalBulkSHA256Missing = "/index/bulk/sha256/missing" + RouteInternalBulkCreate = "/index/bulk" + RouteInternalBulkDocs = "/index/bulk/documents" // Core API RouteCoreSHA256 = "/index/v1/sha256/validity" diff --git a/internal/converters/converters.go b/internal/converters/converters.go deleted file mode 100644 index b9604c7d..00000000 --- a/internal/converters/converters.go +++ /dev/null @@ -1,150 +0,0 @@ -package converters - -import ( - "fmt" - "strings" - "time" - - "github.com/calypr/syfon/apigen/server/drs" - "github.com/calypr/syfon/apigen/server/lfsapi" - syfoncommon "github.com/calypr/syfon/common" - "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" -) - -func CandidateToInternalObject(c drs.DrsObjectCandidate, now time.Time) (models.InternalObject, error) { - oid, ok := common.CanonicalSHA256(c.Checksums) - if !ok { - return models.InternalObject{}, fmt.Errorf("candidate must include sha256 checksum") - } - if c.AccessMethods == nil || len(*c.AccessMethods) == 0 { - return models.InternalObject{}, common.ErrAccessMethodsRequired - } - authzMap := syfoncommon.ControlledAccessToAuthzMap(common.DerefStringSlice(c.ControlledAccess)) - id, err := common.MintObjectIDFromChecksum(oid, syfoncommon.AuthzMapToList(authzMap)) - if err != nil { - return models.InternalObject{}, err - } - obj := drs.DrsObject{ - Id: id, - Size: c.Size, - CreatedTime: now, - UpdatedTime: &now, - Version: common.Ptr("1"), - MimeType: c.MimeType, - Description: c.Description, - Aliases: common.Ptr(append([]string(nil), common.DerefStringSlice(c.Aliases)...)), - Checksums: []drs.Checksum{{Type: "sha256", Checksum: oid}}, - } - if c.ControlledAccess != nil { - controlled := syfoncommon.NormalizeAccessResources(*c.ControlledAccess) - obj.ControlledAccess = &controlled - } - if c.Name != nil { - obj.Name = c.Name - } - obj.SelfUri = "drs://" + obj.Id - if obj.Name == nil || strings.TrimSpace(*obj.Name) == "" { - obj.Name = &oid - } - - seenAccess := make(map[string]struct{}) - if c.AccessMethods != nil { - for _, am := range *c.AccessMethods { - url := "" - if am.AccessUrl != nil { - url = strings.TrimSpace(am.AccessUrl.Url) - } - if url == "" { - continue - } - key := string(am.Type) + "|" + url - if _, ok := seenAccess[key]; ok { - continue - } - seenAccess[key] = struct{}{} - accessID := am.AccessId - if accessID == nil || strings.TrimSpace(*accessID) == "" { - accessID = common.Ptr(string(am.Type)) - } - if obj.AccessMethods == nil { - obj.AccessMethods = &[]drs.AccessMethod{} - } - - newMethod := drs.AccessMethod{ - Type: drs.AccessMethodType(am.Type), - AccessId: accessID, - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: url}, - } - *obj.AccessMethods = append(*obj.AccessMethods, newMethod) - } - } - if obj.AccessMethods == nil || len(*obj.AccessMethods) == 0 { - return models.InternalObject{}, common.ErrAccessMethodsRequired - } - return models.InternalObject{ - DrsObject: obj, - Authorizations: authzMap, - }, nil -} - -func LFSCandidateToDRS(in lfsapi.DrsObjectCandidate) drs.DrsObjectCandidate { - var size int64 - if in.Size != nil { - size = *in.Size - } - aliases := append([]string(nil), common.DerefStringSlice(in.Aliases)...) - explicitID := strings.TrimSpace(common.DerefString(in.Id)) - out := drs.DrsObjectCandidate{ - Name: in.Name, - Size: size, - MimeType: in.MimeType, - Description: in.Description, - } - if in.Checksums != nil { - out.Checksums = make([]drs.Checksum, 0, len(*in.Checksums)) - for _, c := range *in.Checksums { - out.Checksums = append(out.Checksums, drs.Checksum{ - Type: c.Type, - Checksum: c.Checksum, - }) - } - if explicitID == "" { - for _, c := range out.Checksums { - if strings.EqualFold(strings.TrimSpace(c.Type), "sha256") { - explicitID = syfoncommon.NormalizeOid(c.Checksum) - break - } - } - } - } - if explicitID != "" { - aliases = append([]string{"id:" + explicitID}, aliases...) - } - out.Aliases = common.Ptr(aliases) - if in.AccessMethods != nil { - ams := make([]drs.AccessMethod, 0, len(*in.AccessMethods)) - for _, am := range *in.AccessMethods { - drsMethod := drs.AccessMethod{ - Type: drs.AccessMethodType(common.DerefString(am.Type)), - Region: am.Region, - AccessId: am.AccessId, - } - if am.AccessUrl != nil { - drsMethod.AccessUrl = &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{} - if am.AccessUrl.Url != nil { - drsMethod.AccessUrl.Url = *am.AccessUrl.Url - } - } - ams = append(ams, drsMethod) - } - out.AccessMethods = &ams - } - return out -} diff --git a/internal/converters/converters_test.go b/internal/converters/converters_test.go deleted file mode 100644 index df1e8a05..00000000 --- a/internal/converters/converters_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package converters - -import ( - "strings" - "testing" - "time" - - "github.com/calypr/syfon/apigen/server/drs" - "github.com/calypr/syfon/apigen/server/lfsapi" -) - -func TestCandidateToInternalObjectAndLFSCandidateToDRS(t *testing.T) { - t.Run("candidate to internal object", func(t *testing.T) { - authz := []string{"/programs/syfon/projects/e2e"} - url := "https://storage.example/object.bin" - name := "object.bin" - size := int64(42) - candidate := drs.DrsObjectCandidate{ - Name: &name, - Size: size, - Checksums: []drs.Checksum{{Type: "sha256", Checksum: strings.Repeat("a", 64)}}, - AccessMethods: &[]drs.AccessMethod{{ - Type: "https", - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: url}, - }}, - ControlledAccess: &authz, - } - - obj, err := CandidateToInternalObject(candidate, time.Unix(123, 0)) - if err != nil { - t.Fatalf("CandidateToInternalObject returned error: %v", err) - } - if got := obj.Authorizations["syfon"]; len(got) != 1 || got[0] != "e2e" { - t.Fatalf("unexpected internal authz map: %+v", obj.Authorizations) - } - if obj.DrsObject.AccessMethods == nil || len(*obj.DrsObject.AccessMethods) != 1 { - t.Fatalf("expected one access method, got %+v", obj.DrsObject.AccessMethods) - } - if (*obj.DrsObject.AccessMethods)[0].Authorizations != nil { - t.Fatalf("did not expect per-method authz, got %+v", (*obj.DrsObject.AccessMethods)[0].Authorizations) - } - }) - - t.Run("candidate without access methods fails", func(t *testing.T) { - candidate := drs.DrsObjectCandidate{ - Size: 42, - Checksums: []drs.Checksum{{Type: "sha256", Checksum: strings.Repeat("b", 64)}}, - } - if _, err := CandidateToInternalObject(candidate, time.Unix(123, 0)); err == nil || !strings.Contains(err.Error(), "access method") { - t.Fatalf("expected access-method validation error, got %v", err) - } - }) - - t.Run("lfs candidate to drs", func(t *testing.T) { - url := "https://storage.example/object.bin" - size := int64(42) - candidate := lfsapi.DrsObjectCandidate{ - Name: strPtr("object.bin"), - Size: &size, - Checksums: &[]lfsapi.Checksum{{ - Type: "sha256", Checksum: "abc123", - }}, - AccessMethods: &[]lfsapi.AccessMethod{{ - Type: strPtr("https"), - AccessUrl: &lfsapi.AccessMethodAccessUrl{ - Url: &url, - }, - }}, - } - - got := LFSCandidateToDRS(candidate) - if got.AccessMethods == nil || len(*got.AccessMethods) != 1 { - t.Fatalf("expected one access method, got %+v", got.AccessMethods) - } - if (*got.AccessMethods)[0].Authorizations != nil { - t.Fatalf("did not expect authz map on access method, got %+v", (*got.AccessMethods)[0].Authorizations) - } - }) -} - -func strPtr(s string) *string { return &s } diff --git a/internal/converters/util.go b/internal/converters/util.go deleted file mode 100644 index 6064ead1..00000000 --- a/internal/converters/util.go +++ /dev/null @@ -1,13 +0,0 @@ -package converters - -import ( - "sort" - - "github.com/calypr/syfon/internal/models" -) - -func SortInternalObjects(objects []models.InternalObject) { - sort.Slice(objects, func(i, j int) bool { - return objects[i].Id < objects[j].Id - }) -} diff --git a/internal/core/bucket_catalog.go b/internal/core/bucket_catalog.go index f511f3d6..3fc895f3 100644 --- a/internal/core/bucket_catalog.go +++ b/internal/core/bucket_catalog.go @@ -58,6 +58,10 @@ func (m *ObjectManager) ListBucketScopes(ctx context.Context) ([]models.BucketSc } func (m *ObjectManager) ListVisibleBuckets(ctx context.Context) (map[string]VisibleBucket, error) { + return m.listVisibleBucketsCached(ctx) +} + +func (m *ObjectManager) listVisibleBucketsUncached(ctx context.Context) (map[string]VisibleBucket, error) { creds, err := m.db.ListS3Credentials(ctx) if err != nil { return nil, err @@ -290,20 +294,7 @@ func (m *ObjectManager) listBucketsVisibleObjects(ctx context.Context) ([]models return m.filterObjectsByMethod(ctx, objects, objectMethodRead), nil } -func bucketReferencedByPublicObject(objects []models.InternalObject, cred models.S3Credential, creds []models.S3Credential) bool { - credentialID := credentialIDForCredential(cred) - for _, obj := range objects { - if len(ObjectAccessResources(&obj)) != 0 || obj.AccessMethods == nil { - continue - } - for _, method := range *obj.AccessMethods { - if id, ok := credentialIDForAccessMethod(method, creds); ok && id == credentialID { - return true - } - } - } - return false -} + func credentialIDForCredential(cred models.S3Credential) string { if credentialID := strings.TrimSpace(cred.CredentialID); credentialID != "" { diff --git a/internal/core/converters.go b/internal/core/converters.go index 9c1c7302..c97b6824 100644 --- a/internal/core/converters.go +++ b/internal/core/converters.go @@ -2,7 +2,7 @@ package core import ( "fmt" - "path" + "path/filepath" "strings" "time" @@ -14,6 +14,27 @@ import ( "github.com/calypr/syfon/internal/models" ) +func EnforceCanonicalProjectScope(obj models.InternalObject, organization, project string) (models.InternalObject, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if project != "" && organization == "" { + return models.InternalObject{}, fmt.Errorf("organization is required when project is set") + } + if organization == "" || project == "" { + return obj, nil + } + + resource, err := syfoncommon.ResourcePath(organization, project) + if err != nil { + return models.InternalObject{}, err + } + controlled := append(ObjectAccessResources(&obj), resource) + controlled = syfoncommon.NormalizeAccessResources(controlled) + obj.ControlledAccess = &controlled + obj.Authorizations = syfoncommon.ControlledAccessToAuthzMap(controlled) + return obj, nil +} + // LFSCandidateToDRS converts an LFS-specific candidate to a DRS-generic one. func LFSCandidateToDRS(in lfsapi.DrsObjectCandidate) drs.DrsObjectCandidate { aliases := append([]string(nil), common.DerefStringSlice(in.Aliases)...) @@ -138,7 +159,7 @@ func CandidateToInternalObject(c drs.DrsObjectCandidate, now time.Time) (models. obj.ControlledAccess = &controlled } if c.Name != nil { - obj.Name = c.Name + obj.Name = normalizedObjectNamePtr(c.Name) } if obj.Name == nil || strings.TrimSpace(*obj.Name) == "" { obj.Name = &oid @@ -182,7 +203,7 @@ func MergeInternalObjectUpdate(existing models.InternalObject, update models.Int } if update.DrsObject.Name != nil { - merged.DrsObject.Name = update.DrsObject.Name + merged.DrsObject.Name = normalizedObjectNamePtr(update.DrsObject.Name) } if update.DrsObject.Description != nil { merged.DrsObject.Description = update.DrsObject.Description @@ -230,17 +251,9 @@ func InternalRecordToInternalObject(r internalapi.InternalRecord, now time.Time) updatedTime := parseInternalRecordTime(r.UpdatedTime, obj.CreatedTime) obj.UpdatedTime = &updatedTime if r.Name != nil && strings.TrimSpace(*r.Name) != "" { - obj.Name = common.Ptr(strings.TrimSpace(*r.Name)) - } else if r.FileName != nil { - fileName := strings.TrimSpace(*r.FileName) - if fileName != "" { - base := path.Base(strings.Trim(fileName, "/")) - if base == "." || base == "/" { - base = fileName - } - obj.Name = common.Ptr(base) - } + obj.Name = normalizedObjectNamePtr(r.Name) } + objectName := common.StringVal(obj.Name) if v := r.Version; v != nil { obj.Version = v } @@ -263,13 +276,29 @@ func InternalRecordToInternalObject(r internalapi.InternalRecord, now time.Time) methods := append([]drs.AccessMethod(nil), (*r.AccessMethods)...) obj.AccessMethods = &methods } - return models.InternalObject{ + internalObj := models.InternalObject{ DrsObject: obj, + NameAliases: common.NormalizeNameAliases(objectName, common.DerefStringSlice(r.NameAliases)), Authorizations: authzMap, - Properties: map[string]interface{}{ - "file_name": common.StringVal(r.FileName), - }, - }, nil + Properties: map[string]interface{}{}, + } + return EnforceCanonicalProjectScope(internalObj, common.StringVal(r.Organization), common.StringVal(r.Project)) +} + +func normalizedObjectNamePtr(name *string) *string { + if name == nil { + return nil + } + trimmed := strings.TrimSpace(*name) + if trimmed == "" { + return nil + } + trimmed = strings.ReplaceAll(trimmed, "\\", "/") + base := filepath.Base(trimmed) + if base == "." || base == "/" || base == "" { + base = trimmed + } + return common.Ptr(base) } func parseInternalRecordTime(raw *string, fallback time.Time) time.Time { @@ -286,17 +315,13 @@ func parseInternalRecordTime(raw *string, fallback time.Time) time.Time { // InternalObjectToInternalRecord converts our internal domain model back to an API record. func InternalObjectToInternalRecord(obj models.InternalObject) internalapi.InternalRecord { - fileName := common.StringVal(obj.Name) - if raw, ok := obj.Properties["file_name"].(string); ok && strings.TrimSpace(raw) != "" { - fileName = strings.TrimSpace(raw) - } res := internalapi.InternalRecord{ Did: obj.Id, Size: &obj.Size, CreatedTime: common.Ptr(obj.CreatedTime.Format(time.RFC3339)), Description: obj.Description, - FileName: common.Ptr(fileName), Name: obj.Name, + NameAliases: common.Ptr(common.NormalizeNameAliases(common.StringVal(obj.Name), obj.NameAliases)), Version: obj.Version, AccessMethods: obj.AccessMethods, } @@ -326,8 +351,8 @@ func InternalObjectToInternalRecordResponse(obj models.InternalObject) internala Size: rec.Size, CreatedTime: rec.CreatedTime, Description: rec.Description, - FileName: rec.FileName, Name: rec.Name, + NameAliases: rec.NameAliases, Version: rec.Version, UpdatedTime: rec.UpdatedTime, Hashes: rec.Hashes, diff --git a/internal/core/converters_test.go b/internal/core/converters_test.go index 5e107eb2..f87c5fcf 100644 --- a/internal/core/converters_test.go +++ b/internal/core/converters_test.go @@ -7,6 +7,7 @@ import ( "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/apigen/server/lfsapi" + "github.com/calypr/syfon/internal/models" ) func TestConverters(t *testing.T) { @@ -95,6 +96,22 @@ func TestConverters(t *testing.T) { t.Fatalf("expected oid-derived id alias, got %+v", got.Aliases) } }) + + t.Run("enforce canonical project scope appends exact controlled access", func(t *testing.T) { + obj, err := EnforceCanonicalProjectScope(models.InternalObject{ + DrsObject: drs.DrsObject{ + Id: "obj-1", + }, + Authorizations: map[string][]string{"other": {"proj"}}, + }, "org", "proj") + if err != nil { + t.Fatalf("EnforceCanonicalProjectScope returned error: %v", err) + } + got := ObjectAccessResources(&obj) + if len(got) != 2 || got[0] != "/organization/other/project/proj" || got[1] != "/organization/org/project/proj" { + t.Fatalf("unexpected controlled access: %+v", got) + } + }) } func strPtr(s string) *string { return &s } diff --git a/internal/core/object_canonicalization.go b/internal/core/object_canonicalization.go new file mode 100644 index 00000000..b8eafa01 --- /dev/null +++ b/internal/core/object_canonicalization.go @@ -0,0 +1,340 @@ +package core + +import ( + "sort" + "strings" + "time" + + "github.com/calypr/syfon/apigen/server/drs" + syfoncommon "github.com/calypr/syfon/common" + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/models" +) + +func canonicalizeProjectScopedObjects(objects []models.InternalObject, organization, project string) []models.InternalObject { + if len(objects) <= 1 { + return cloneObjects(objects) + } + + forcedResource := "" + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization != "" && project != "" { + if resource, err := syfoncommon.ResourcePath(organization, project); err == nil { + forcedResource = resource + } + } + + grouped := make(map[string][]models.InternalObject) + passthrough := make([]models.InternalObject, 0) + for _, obj := range objects { + key, ok := canonicalProjectChecksumKey(&obj, forcedResource) + if !ok { + passthrough = append(passthrough, cloneObject(obj)) + continue + } + grouped[key] = append(grouped[key], cloneObject(obj)) + } + + keys := make([]string, 0, len(grouped)) + for key := range grouped { + keys = append(keys, key) + } + sort.Strings(keys) + + out := make([]models.InternalObject, 0, len(keys)+len(passthrough)) + for _, key := range keys { + out = append(out, collapseCanonicalGroup(grouped[key])) + } + out = append(out, passthrough...) + sort.Slice(out, func(i, j int) bool { + if out[i].Id == out[j].Id { + return canonicalObjectSortTime(out[i]).After(canonicalObjectSortTime(out[j])) + } + return out[i].Id < out[j].Id + }) + return out +} + +func canonicalProjectChecksumKey(obj *models.InternalObject, forcedResource string) (string, bool) { + if obj == nil { + return "", false + } + sha, ok := common.CanonicalSHA256(obj.Checksums) + if !ok || strings.TrimSpace(sha) == "" { + return "", false + } + resource := strings.TrimSpace(forcedResource) + if resource == "" { + resources := projectScopeResources(obj) + if len(resources) != 1 { + return "", false + } + resource = resources[0] + } + return resource + "|" + sha, true +} + +func projectScopeResources(obj *models.InternalObject) []string { + resources := ObjectAccessResources(obj) + out := make([]string, 0, len(resources)) + for _, resource := range resources { + org, project, ok := syfoncommon.ResourceScope(resource) + if !ok || strings.TrimSpace(org) == "" || strings.TrimSpace(project) == "" { + continue + } + out = append(out, resource) + } + return syfoncommon.NormalizeAccessResources(out) +} + +func collapseCanonicalGroup(group []models.InternalObject) models.InternalObject { + if len(group) == 0 { + return models.InternalObject{} + } + + canonical := cloneObject(group[0]) + latest := canonical + for i := 1; i < len(group); i++ { + obj := cloneObject(group[i]) + if canonicalObjectNewer(obj, canonical) { + canonical = obj + } + if canonicalObjectNewer(obj, latest) { + latest = obj + } + } + + merged := cloneObject(canonical) + merged.Name = latest.Name + merged.Size = pickLatestNonZeroSize(group, canonical.Size) + merged.Description = pickLatestStringPtr(group, func(obj models.InternalObject) *string { return obj.Description }, canonical.Description) + merged.MimeType = pickLatestStringPtr(group, func(obj models.InternalObject) *string { return obj.MimeType }, canonical.MimeType) + merged.Version = pickLatestStringPtr(group, func(obj models.InternalObject) *string { return obj.Version }, canonical.Version) + updated := canonicalObjectSortTime(latest) + merged.UpdatedTime = &updated + merged.Checksums = mergeChecksums(group) + merged.AccessMethods = mergeAccessMethods(group) + controlled := mergeControlledAccess(group) + if len(controlled) > 0 { + merged.ControlledAccess = &controlled + merged.Authorizations = syfoncommon.ControlledAccessToAuthzMap(controlled) + } + merged.NameAliases = mergeNameAliases(merged.Name, group) + merged.Aliases = mergeStringPointerValues(func(obj models.InternalObject) []string { return common.DerefStringSlice(obj.Aliases) }, group) + merged.SelfUri = "drs://" + merged.Id + return merged +} + +func canonicalObjectNewer(a, b models.InternalObject) bool { + at := canonicalObjectSortTime(a) + bt := canonicalObjectSortTime(b) + if !at.Equal(bt) { + return at.After(bt) + } + return a.Id > b.Id +} + +func canonicalObjectSortTime(obj models.InternalObject) time.Time { + if obj.UpdatedTime != nil && !obj.UpdatedTime.IsZero() { + return obj.UpdatedTime.UTC() + } + return obj.CreatedTime.UTC() +} + +func cloneObjects(objects []models.InternalObject) []models.InternalObject { + out := make([]models.InternalObject, 0, len(objects)) + for _, obj := range objects { + out = append(out, cloneObject(obj)) + } + return out +} + +func cloneObject(obj models.InternalObject) models.InternalObject { + cloned := obj + cloned.Checksums = append([]drs.Checksum(nil), obj.Checksums...) + cloned.NameAliases = append([]string(nil), obj.NameAliases...) + if obj.AccessMethods != nil { + methods := append([]drs.AccessMethod(nil), (*obj.AccessMethods)...) + cloned.AccessMethods = &methods + } + if obj.ControlledAccess != nil { + controlled := append([]string(nil), (*obj.ControlledAccess)...) + cloned.ControlledAccess = &controlled + } + if obj.Aliases != nil { + aliases := append([]string(nil), (*obj.Aliases)...) + cloned.Aliases = &aliases + } + if obj.Authorizations != nil { + authz := make(map[string][]string, len(obj.Authorizations)) + for org, projects := range obj.Authorizations { + authz[org] = append([]string(nil), projects...) + } + cloned.Authorizations = authz + } + if obj.Properties != nil { + props := make(map[string]interface{}, len(obj.Properties)) + for key, value := range obj.Properties { + props[key] = value + } + cloned.Properties = props + } + return cloned +} + +func mergeChecksums(group []models.InternalObject) []drs.Checksum { + seen := make(map[string]struct{}) + merged := make([]drs.Checksum, 0) + for _, obj := range group { + for _, checksum := range obj.Checksums { + key := checksum.Type + "|" + checksum.Checksum + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + merged = append(merged, checksum) + } + } + sort.Slice(merged, func(i, j int) bool { + if merged[i].Type == merged[j].Type { + return merged[i].Checksum < merged[j].Checksum + } + return merged[i].Type < merged[j].Type + }) + return merged +} + +func mergeAccessMethods(group []models.InternalObject) *[]drs.AccessMethod { + seen := make(map[string]struct{}) + methods := make([]drs.AccessMethod, 0) + for _, obj := range group { + if obj.AccessMethods == nil { + continue + } + for _, method := range *obj.AccessMethods { + url := "" + if method.AccessUrl != nil { + url = method.AccessUrl.Url + } + key := string(method.Type) + "|" + url + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + methods = append(methods, method) + } + } + if len(methods) == 0 { + return nil + } + sort.Slice(methods, func(i, j int) bool { + iURL := "" + jURL := "" + if methods[i].AccessUrl != nil { + iURL = methods[i].AccessUrl.Url + } + if methods[j].AccessUrl != nil { + jURL = methods[j].AccessUrl.Url + } + if methods[i].Type == methods[j].Type { + return iURL < jURL + } + return methods[i].Type < methods[j].Type + }) + return &methods +} + +func mergeControlledAccess(group []models.InternalObject) []string { + resources := make([]string, 0) + for _, obj := range group { + resources = append(resources, ObjectAccessResources(&obj)...) + } + return syfoncommon.NormalizeAccessResources(resources) +} + +func mergeNameAliases(primary *string, group []models.InternalObject) []string { + candidates := make([]string, 0) + for _, obj := range group { + if obj.Name != nil { + candidates = append(candidates, *obj.Name) + } + candidates = append(candidates, obj.NameAliases...) + } + return common.NormalizeNameAliases(common.StringVal(primary), candidates) +} + +func pickLatestNonZeroSize(group []models.InternalObject, fallback int64) int64 { + best := fallback + var bestTime time.Time + for _, obj := range group { + if obj.Size <= 0 { + continue + } + when := canonicalObjectSortTime(obj) + if best <= 0 || when.After(bestTime) { + best = obj.Size + bestTime = when + } + } + return best +} + +func pickLatestStringPtr(group []models.InternalObject, getter func(models.InternalObject) *string, fallback *string) *string { + best := fallback + var bestTime time.Time + for _, obj := range group { + value := getter(obj) + if value == nil || strings.TrimSpace(*value) == "" { + continue + } + when := canonicalObjectSortTime(obj) + if best == nil || when.After(bestTime) { + trimmed := strings.TrimSpace(*value) + best = &trimmed + bestTime = when + } + } + return best +} + +func mergeStringPointerValues(getter func(models.InternalObject) []string, group []models.InternalObject) *[]string { + seen := make(map[string]struct{}) + values := make([]string, 0) + for _, obj := range group { + for _, value := range getter(obj) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + values = append(values, trimmed) + } + } + if len(values) == 0 { + return nil + } + sort.Strings(values) + return &values +} + +func uniqueStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + sort.Strings(out) + return out +} diff --git a/internal/core/object_mutations.go b/internal/core/object_mutations.go index fe8bc417..d0b28418 100644 --- a/internal/core/object_mutations.go +++ b/internal/core/object_mutations.go @@ -92,6 +92,10 @@ func (m *ObjectManager) DeleteObjectWithOptions(ctx context.Context, id string, } func (m *ObjectManager) BulkDeleteObjects(ctx context.Context, ids []string) error { + return m.BulkDeleteObjectsWithOptions(ctx, ids, DeleteOptions{}) +} + +func (m *ObjectManager) BulkDeleteObjectsWithOptions(ctx context.Context, ids []string, opts DeleteOptions) error { toDelete, err := m.deletableObjectIDs(ctx, ids) if err != nil { return err @@ -99,6 +103,15 @@ func (m *ObjectManager) BulkDeleteObjects(ctx context.Context, ids []string) err if len(toDelete) == 0 { return nil } + if opts.DeleteStorageData { + objects, err := m.db.GetBulkObjects(ctx, toDelete) + if err != nil { + return err + } + if err := m.deleteObjectsStorage(ctx, objects); err != nil { + return err + } + } return m.db.BulkDeleteObjects(ctx, toDelete) } @@ -183,7 +196,192 @@ func (m *ObjectManager) RegisterObjects(ctx context.Context, objs []models.Inter if err := m.bulkObjectMethodError(ctx, objs, objectMethodCreate); err != nil { return err } - return m.db.RegisterObjects(ctx, objs) + canonical, aliases, err := m.canonicalizeRegistrationObjects(ctx, objs) + if err != nil { + return err + } + if err := m.db.RegisterObjects(ctx, canonical); err != nil { + return err + } + for aliasID, canonicalID := range aliases { + if err := m.db.CreateObjectAlias(ctx, aliasID, canonicalID); err != nil { + return err + } + } + return nil +} + +func (m *ObjectManager) CollapseProjectChecksumDuplicates(ctx context.Context, organization, project string) (int, error) { + ids, err := m.db.ListObjectIDsByScope(ctx, organization, project) + if err != nil { + return 0, err + } + objects, err := m.db.GetBulkObjects(ctx, ids) + if err != nil { + return 0, err + } + if err := m.bulkObjectMethodError(ctx, objects, objectMethodUpdate); err != nil { + return 0, err + } + + grouped := make(map[string][]models.InternalObject) + for _, obj := range objects { + key, ok := canonicalProjectChecksumKey(&obj, "") + if !ok { + continue + } + grouped[key] = append(grouped[key], cloneObject(obj)) + } + + merged := make([]models.InternalObject, 0, len(grouped)) + aliasMap := make(map[string]string) + toDelete := make([]string, 0) + keys := make([]string, 0, len(grouped)) + for key, group := range grouped { + if len(group) < 2 { + continue + } + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + group := grouped[key] + canonical := collapseCanonicalGroup(group) + merged = append(merged, canonical) + for _, obj := range group { + if obj.Id == canonical.Id { + continue + } + aliasMap[obj.Id] = canonical.Id + toDelete = append(toDelete, obj.Id) + } + } + + if len(merged) == 0 { + return 0, nil + } + if err := m.db.RegisterObjects(ctx, merged); err != nil { + return 0, err + } + for aliasID, canonicalID := range aliasMap { + if err := m.db.CreateObjectAlias(ctx, aliasID, canonicalID); err != nil { + return 0, err + } + } + if err := m.db.BulkDeleteObjects(ctx, uniqueStrings(toDelete)); err != nil { + return 0, err + } + return len(aliasMap), nil +} + +func (m *ObjectManager) canonicalizeRegistrationObjects(ctx context.Context, objs []models.InternalObject) ([]models.InternalObject, map[string]string, error) { + if len(objs) == 0 { + return nil, nil, nil + } + + checksums := make([]string, 0, len(objs)) + seenChecksums := make(map[string]struct{}, len(objs)) + for _, obj := range objs { + if sha, ok := common.CanonicalSHA256(obj.Checksums); ok { + if _, seen := seenChecksums[sha]; seen { + continue + } + seenChecksums[sha] = struct{}{} + checksums = append(checksums, sha) + } + } + + existingByChecksum, err := m.db.GetObjectsByChecksums(ctx, checksums) + if err != nil { + return nil, nil, err + } + + type registrationGroup struct { + existingCount int + objects []models.InternalObject + } + groups := make(map[string]*registrationGroup) + passthrough := make([]models.InternalObject, 0) + + for _, obj := range objs { + sha, ok := common.CanonicalSHA256(obj.Checksums) + if !ok { + passthrough = append(passthrough, cloneObject(obj)) + continue + } + resources := projectScopeResources(&obj) + if len(resources) != 1 { + passthrough = append(passthrough, cloneObject(obj)) + continue + } + key := resources[0] + "|" + sha + group, ok := groups[key] + if !ok { + group = ®istrationGroup{} + for _, existing := range existingByChecksum[sha] { + existingKey, ok := canonicalProjectChecksumKey(&existing, "") + if ok && existingKey == key { + group.objects = append(group.objects, cloneObject(existing)) + } + } + group.existingCount = len(group.objects) + groups[key] = group + } + group.objects = append(group.objects, cloneObject(obj)) + } + + keys := make([]string, 0, len(groups)) + for key := range groups { + keys = append(keys, key) + } + sort.Strings(keys) + + merged := make([]models.InternalObject, 0, len(keys)+len(passthrough)) + aliasMap := make(map[string]string) + + for _, key := range keys { + state := groups[key] + group := state.objects + if len(group) == 0 { + continue + } + + hasExisting := state.existingCount > 0 + canonical := cloneObject(group[0]) + latest := canonical + if hasExisting { + canonical = cloneObject(group[0]) + } + for i, obj := range group { + if hasExisting && i < state.existingCount && canonicalObjectNewer(obj, canonical) { + canonical = cloneObject(obj) + } + if canonicalObjectNewer(obj, latest) { + latest = cloneObject(obj) + } + } + if !hasExisting { + canonical = cloneObject(latest) + } + + collapsed := collapseCanonicalGroup(group) + collapsed.Id = canonical.Id + collapsed.SelfUri = "drs://" + canonical.Id + collapsed.CreatedTime = canonical.CreatedTime + collapsed.Name = latest.Name + collapsed.NameAliases = common.NormalizeNameAliases(common.StringVal(collapsed.Name), append(collapsed.NameAliases, common.StringVal(canonical.Name))) + merged = append(merged, collapsed) + + for _, obj := range group { + if obj.Id == collapsed.Id || strings.TrimSpace(obj.Id) == "" { + continue + } + aliasMap[obj.Id] = collapsed.Id + } + } + + merged = append(merged, passthrough...) + return merged, aliasMap, nil } func (m *ObjectManager) ReplaceObjects(ctx context.Context, objs []models.InternalObject) error { diff --git a/internal/core/object_mutations_test.go b/internal/core/object_mutations_test.go index 0d12ecee..78676e0e 100644 --- a/internal/core/object_mutations_test.go +++ b/internal/core/object_mutations_test.go @@ -2,10 +2,13 @@ package core import ( "context" + "slices" "testing" + "time" "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/db" + "github.com/calypr/syfon/internal/models" ) func ptr[T any](v T) *T { return &v } @@ -97,3 +100,88 @@ func TestBulkDeleteObjects_DeletesAuthorizedObjects(t *testing.T) { t.Fatalf("expected object to be deleted") } } + +func TestRegisterObjects_CanonicalizesProjectChecksumDuplicates(t *testing.T) { + database := db.NewInMemoryDB() + om := NewObjectManager(database, nil) + now := time.Now().UTC() + later := now.Add(time.Minute) + accessURL1 := "s3://bucket/original" + accessURL2 := "s3://bucket/renamed" + + first := models.InternalObject{ + Authorizations: map[string][]string{"org": {"proj"}}, + DrsObject: drs.DrsObject{ + Id: "did-1", + Name: ptr("original.tsv"), + Size: 42, + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}}, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: accessURL1}, + }}, + }, + } + second := models.InternalObject{ + Authorizations: map[string][]string{"org": {"proj"}}, + DrsObject: drs.DrsObject{ + Id: "did-2", + Name: ptr("renamed.tsv"), + Size: 42, + CreatedTime: later, + UpdatedTime: &later, + Checksums: first.Checksums, + AccessMethods: &[]drs.AccessMethod{{ + Type: "s3", + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: accessURL2}, + }}, + }, + } + + if err := om.RegisterObjects(context.Background(), []models.InternalObject{first}); err != nil { + t.Fatalf("RegisterObjects(first) error: %v", err) + } + if err := om.RegisterObjects(context.Background(), []models.InternalObject{second}); err != nil { + t.Fatalf("RegisterObjects(second) error: %v", err) + } + + records, err := om.GetObjectsByChecksum(context.Background(), first.Checksums[0].Checksum, "") + if err != nil { + t.Fatalf("GetObjectsByChecksum error: %v", err) + } + if len(records) != 1 { + t.Fatalf("expected 1 canonical record, got %d", len(records)) + } + if records[0].Id != "did-1" { + t.Fatalf("expected canonical did-1, got %q", records[0].Id) + } + if got := records[0].Name; got == nil || *got != "renamed.tsv" { + t.Fatalf("expected latest name renamed.tsv, got %+v", got) + } + if !slices.Equal(records[0].NameAliases, []string{"original.tsv"}) { + t.Fatalf("unexpected name aliases: %#v", records[0].NameAliases) + } + + aliasObj, err := om.GetObject(context.Background(), "did-2", "") + if err != nil { + t.Fatalf("GetObject(alias) error: %v", err) + } + if aliasObj.Id != "did-2" { + t.Fatalf("expected alias lookup to preserve requested did, got %q", aliasObj.Id) + } + scopeIDs, err := om.ListObjectIDsByScope(context.Background(), "org", "proj", "") + if err != nil { + t.Fatalf("ListObjectIDsByScope error: %v", err) + } + if !slices.Equal(scopeIDs, []string{"did-1"}) { + t.Fatalf("unexpected scoped ids: %#v", scopeIDs) + } +} diff --git a/internal/core/object_reads.go b/internal/core/object_reads.go index ee69086e..b74a4f0d 100644 --- a/internal/core/object_reads.go +++ b/internal/core/object_reads.go @@ -2,8 +2,10 @@ package core import ( "context" + "log" "sort" "strings" + "time" syfoncommon "github.com/calypr/syfon/common" "github.com/calypr/syfon/internal/authz" @@ -40,45 +42,23 @@ func (m *ObjectManager) GetObject(ctx context.Context, ident string, requiredMet } func (m *ObjectManager) lookupObjectByChecksum(ctx context.Context, ident string, requiredMethod string) (*models.InternalObject, bool, error) { - if ids, optimized, err := m.authorizedChecksumIDs(ctx, ident, requiredMethod); err != nil { - return nil, false, err - } else if optimized { - if len(ids) == 0 { - if strings.TrimSpace(requiredMethod) != "" { - if allMatches, ok, err := m.authorizedChecksumIDs(ctx, ident, ""); err != nil { - return nil, false, err - } else if ok && len(allMatches) > 0 { - return nil, true, common.ErrUnauthorized - } - } - return nil, false, nil - } - objects, err := m.db.GetBulkObjects(ctx, ids[:1]) - if err != nil { - return nil, false, err - } - if len(objects) > 0 { - return &objects[0], true, nil - } - return nil, false, nil - } - - byChecksum, err := m.db.GetObjectsByChecksum(ctx, ident) + byChecksum, err := m.GetObjectsByChecksum(ctx, ident, requiredMethod) if err != nil { return nil, false, err } if len(byChecksum) == 0 { - return nil, false, nil - } - for i := range byChecksum { - if m.hasObjectMethod(ctx, &byChecksum[i], requiredMethod) { - return &byChecksum[i], true, nil + if strings.TrimSpace(requiredMethod) != "" { + allMatches, err := m.GetObjectsByChecksum(ctx, ident, "") + if err != nil { + return nil, false, err + } + if len(allMatches) > 0 { + return nil, true, common.ErrUnauthorized + } } + return nil, false, nil } - if requiredMethod != "" { - return nil, true, common.ErrUnauthorized - } - return nil, false, nil + return &byChecksum[0], true, nil } func (m *ObjectManager) lookupObjectByID(ctx context.Context, ident string) (*models.InternalObject, bool, error) { @@ -132,26 +112,17 @@ func (m *ObjectManager) GetObjectsByChecksums(ctx context.Context, hashes []stri } filtered := make(map[string][]models.InternalObject, len(objectsByChecksum)) for checksum, objects := range objectsByChecksum { - filtered[checksum] = m.filterObjectsByMethod(ctx, objects, requiredMethod) + filtered[checksum] = canonicalizeProjectScopedObjects(m.filterObjectsByMethod(ctx, objects, requiredMethod), "", "") } return filtered, nil } func (m *ObjectManager) GetObjectsByChecksum(ctx context.Context, checksum string, requiredMethod string) ([]models.InternalObject, error) { - if ids, optimized, err := m.authorizedChecksumIDs(ctx, checksum, requiredMethod); err != nil { - return nil, err - } else if optimized { - if len(ids) == 0 { - return []models.InternalObject{}, nil - } - return m.db.GetBulkObjects(ctx, ids) - } - objects, err := m.db.GetObjectsByChecksum(ctx, checksum) if err != nil { return nil, err } - return m.filterObjectsByMethod(ctx, objects, requiredMethod), nil + return canonicalizeProjectScopedObjects(m.filterObjectsByMethod(ctx, objects, requiredMethod), "", ""), nil } func (m *ObjectManager) GetBulkObjects(ctx context.Context, ids []string, requiredMethod string) ([]models.InternalObject, error) { @@ -162,15 +133,99 @@ func (m *ObjectManager) GetBulkObjects(ctx context.Context, ids []string, requir return m.filterObjectsByMethod(ctx, objects, requiredMethod), nil } -func (m *ObjectManager) ListObjectIDsPageByChecksum(ctx context.Context, checksum, checksumType, organization, project, requiredMethod, startAfter string, limit, offset int) ([]string, error) { +func (m *ObjectManager) PrepareScopedObjects(ctx context.Context, objects []models.InternalObject, organization, project, requiredMethod string) ([]models.InternalObject, error) { + started := time.Now() + expanded, err := m.expandProjectChecksumSiblingObjects(ctx, objects, organization, project) + if err != nil { + return nil, err + } + filtered := m.filterObjectsByMethod(ctx, expanded, requiredMethod) + canonicalStart := time.Now() + canonical := canonicalizeProjectScopedObjects(filtered, organization, project) + log.Printf("INFO: syfon_prepare_scoped_objects organization=%s project=%s input_count=%d expanded_count=%d filtered_count=%d output_count=%d canonicalize_scoped_objects_ms=%d duration_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), len(objects), len(expanded), len(filtered), len(canonical), time.Since(canonicalStart).Milliseconds(), time.Since(started).Milliseconds()) + return canonical, nil +} + +func (m *ObjectManager) ListPreparedObjectsPageByScope(ctx context.Context, organization, project, requiredMethod, startAfter string, limit, offset int) ([]models.InternalObject, error) { if limit <= 0 { - return []string{}, nil + return []models.InternalObject{}, nil + } + if offset < 0 { + offset = 0 + } + + startAfter = strings.TrimSpace(startAfter) + skip := offset + if startAfter != "" { + skip = 0 + } + target := limit + skip + batchSize := target + if batchSize < 100 { + batchSize = 100 } - if pager, ok := m.db.(db.ObjectChecksumPageLister); ok && strings.EqualFold(strings.TrimSpace(requiredMethod), objectMethodRead) { - resources, includeUnscoped, restrictToResources, ok := m.readableChecksumFilter(ctx, organization, project) - if ok { - return pager.ListObjectIDsPageByChecksum(ctx, checksum, checksumType, organization, project, startAfter, limit, offset, resources, includeUnscoped, restrictToResources) + + rawStart := startAfter + collected := make([]models.InternalObject, 0, target) + seen := make(map[string]struct{}, target) + started := time.Now() + rawPages := 0 + rawIDs := 0 + + for len(collected) < target { + ids, err := m.ListObjectIDsPageByScope(ctx, organization, project, requiredMethod, rawStart, batchSize, 0) + if err != nil { + return nil, err + } + if len(ids) == 0 { + break } + rawPages++ + rawIDs += len(ids) + rawStart = ids[len(ids)-1] + + objects, err := m.GetBulkObjects(ctx, ids, requiredMethod) + if err != nil { + return nil, err + } + prepared, err := m.PrepareScopedObjects(ctx, objects, organization, project, requiredMethod) + if err != nil { + return nil, err + } + for _, obj := range prepared { + if startAfter != "" && obj.Id <= startAfter { + continue + } + if _, ok := seen[obj.Id]; ok { + continue + } + seen[obj.Id] = struct{}{} + collected = append(collected, obj) + if len(collected) >= target { + break + } + } + if len(ids) < batchSize { + break + } + } + + if skip >= len(collected) { + log.Printf("INFO: syfon_list_prepared_objects_page_by_scope organization=%s project=%s start_after=%t limit=%d offset=%d raw_pages=%d raw_ids=%d records=0 duration_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), startAfter != "", limit, offset, rawPages, rawIDs, time.Since(started).Milliseconds()) + return []models.InternalObject{}, nil + } + end := skip + limit + if end > len(collected) { + end = len(collected) + } + out := collected[skip:end] + log.Printf("INFO: syfon_list_prepared_objects_page_by_scope organization=%s project=%s start_after=%t limit=%d offset=%d raw_pages=%d raw_ids=%d records=%d duration_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), startAfter != "", limit, offset, rawPages, rawIDs, len(out), time.Since(started).Milliseconds()) + return out, nil +} + +func (m *ObjectManager) ListObjectIDsPageByChecksum(ctx context.Context, checksum, checksumType, organization, project, requiredMethod, startAfter string, limit, offset int) ([]string, error) { + if limit <= 0 { + return []string{}, nil } objects, err := m.GetObjectsByChecksum(ctx, checksum, requiredMethod) @@ -205,16 +260,12 @@ func (m *ObjectManager) ListObjectIDsPageByScope(ctx context.Context, organizati if limit <= 0 { return []string{}, nil } - if pager, ok := m.db.(db.ObjectIDPageLister); ok { - if strings.TrimSpace(organization) == "" && strings.EqualFold(strings.TrimSpace(requiredMethod), objectMethodRead) { - if ids, ok, err := m.listReadableObjectIDsPage(ctx, startAfter, limit, offset); ok || err != nil { - return ids, err - } - return pager.ListObjectIDsPageByScope(ctx, organization, project, startAfter, limit, offset) - } - if strings.EqualFold(strings.TrimSpace(requiredMethod), objectMethodRead) && m.canPageScopeRead(ctx, organization, project) { - return pager.ListObjectIDsPageByScope(ctx, organization, project, startAfter, limit, offset) - } + + if lister, ok := m.db.(db.ObjectIDPageLister); ok && canUseUnrestrictedScopePage(ctx, requiredMethod) { + pageStart := time.Now() + ids, err := lister.ListObjectIDsPageByScope(ctx, organization, project, startAfter, limit, offset) + log.Printf("INFO: syfon_list_object_ids_page_by_scope organization=%s project=%s start_after=%t limit=%d offset=%d ids=%d db_page_ms=%d optimized=%t", strings.TrimSpace(organization), strings.TrimSpace(project), strings.TrimSpace(startAfter) != "", limit, offset, len(ids), time.Since(pageStart).Milliseconds(), true) + return ids, err } ids, err := m.ListObjectIDsByScope(ctx, organization, project, requiredMethod) @@ -238,28 +289,9 @@ func (m *ObjectManager) ListObjectIDsPageByScope(ctx context.Context, organizati return ids[offset:end], nil } -func (m *ObjectManager) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, requiredMethod, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { - if limit <= 0 { - return []string{}, []models.BrowseDirectory{}, nil - } - if strings.TrimSpace(requiredMethod) == objectMethodRead { - if authz.IsGen3Mode(ctx) && authz.IsAuthzEnforced(ctx) && !authz.HasAuthHeader(ctx) { - return []string{}, []models.BrowseDirectory{}, nil - } - if authz.IsAuthzEnforced(ctx) && !m.canPageScopeRead(ctx, organization, project) { - return []string{}, []models.BrowseDirectory{}, nil - } - } - - if pager, ok := m.db.(db.ObjectPathPageLister); ok { - return pager.ListObjectIDsPageByPath(ctx, organization, project, path, startAfter, limit, offset) - } - - ids, err := m.ListObjectIDsPageByScope(ctx, organization, project, requiredMethod, startAfter, limit, offset) - if err != nil { - return nil, nil, err - } - return ids, []models.BrowseDirectory{}, nil +func canUseUnrestrictedScopePage(ctx context.Context, requiredMethod string) bool { + resources, includeUnscoped, restrictToResources := objectMethodResourceFilter(ctx, requiredMethod) + return !restrictToResources && len(resources) == 0 && includeUnscoped } func (m *ObjectManager) ListObjectIDsPageByURL(ctx context.Context, objectURL, organization, project, requiredMethod, startAfter string, limit, offset int) ([]string, error) { @@ -312,15 +344,20 @@ func (m *ObjectManager) ListObjectIDsByScope(ctx context.Context, organization, return ids, err } } + listStart := time.Now() ids, err := m.db.ListObjectIDsByScope(ctx, organization, project) if err != nil { return nil, err } + log.Printf("INFO: syfon_list_object_ids_by_scope organization=%s project=%s ids=%d list_scope_ids_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), len(ids), time.Since(listStart).Milliseconds()) objects, err := m.db.GetBulkObjects(ctx, ids) if err != nil { return nil, err } - filtered := m.filterObjectsByMethod(ctx, objects, requiredMethod) + filtered, err := m.PrepareScopedObjects(ctx, objects, organization, project, requiredMethod) + if err != nil { + return nil, err + } out := make([]string, 0, len(filtered)) for _, obj := range filtered { out = append(out, obj.Id) @@ -328,6 +365,139 @@ func (m *ObjectManager) ListObjectIDsByScope(ctx context.Context, organization, return out, nil } +func (m *ObjectManager) ListObjectsByScope(ctx context.Context, organization, project, requiredMethod string) ([]models.InternalObject, error) { + if strings.TrimSpace(organization) == "" && strings.EqualFold(strings.TrimSpace(requiredMethod), objectMethodRead) { + if ids, ok, err := m.listReadableObjectIDs(ctx); ok { + if err != nil { + return nil, err + } + objects, err := m.db.GetBulkObjects(ctx, ids) + if err != nil { + return nil, err + } + return m.filterObjectsByMethod(ctx, objects, requiredMethod), nil + } + } + ids, err := m.db.ListObjectIDsByScope(ctx, organization, project) + if err != nil { + return nil, err + } + objects, err := m.db.GetBulkObjects(ctx, ids) + if err != nil { + return nil, err + } + return m.PrepareScopedObjects(ctx, objects, organization, project, requiredMethod) +} + +// ListMissingScopedSHA256 returns the requested SHA-256 checksums that are not +// registered for the given project. It deliberately uses the indexed checksum +// lookup and does not hydrate complete DRS records or access methods. +func (m *ObjectManager) ListMissingScopedSHA256(ctx context.Context, organization, project string, checksums []string) ([]string, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" || project == "" || len(checksums) == 0 { + return nil, common.ErrUnauthorized + } + if err := m.requireScopeMethod(ctx, organization, project, objectMethodRead); err != nil { + return nil, err + } + + existingByChecksum, err := m.db.ListScopedObjectIDsByChecksums(ctx, organization, project, checksums) + if err != nil { + return nil, err + } + + missing := make([]string, 0, len(checksums)) + for _, checksum := range checksums { + if objectIDs := existingByChecksum[checksum]; len(objectIDs) == 0 { + missing = append(missing, checksum) + } + } + return missing, nil +} + +func (m *ObjectManager) expandProjectChecksumSiblingObjects(ctx context.Context, objects []models.InternalObject, organization, project string) ([]models.InternalObject, error) { + if len(objects) == 0 { + return []models.InternalObject{}, nil + } + + wantedKeys := make(map[string]struct{}, len(objects)) + checksums := make([]string, 0, len(objects)) + seenChecksums := make(map[string]struct{}, len(objects)) + for _, obj := range objects { + key, ok := canonicalProjectChecksumKey(&obj, "") + if !ok { + continue + } + wantedKeys[key] = struct{}{} + sha, _ := common.CanonicalSHA256(obj.Checksums) + if _, seen := seenChecksums[sha]; seen { + continue + } + seenChecksums[sha] = struct{}{} + checksums = append(checksums, sha) + } + if len(wantedKeys) == 0 || len(checksums) == 0 || strings.TrimSpace(organization) == "" || strings.TrimSpace(project) == "" { + return objects, nil + } + + listStart := time.Now() + idsByChecksum, err := m.db.ListScopedObjectIDsByChecksums(ctx, organization, project, checksums) + if err != nil { + return nil, err + } + listDuration := time.Since(listStart) + + expanded := make([]models.InternalObject, 0, len(objects)) + seenIDs := make(map[string]struct{}, len(objects)) + missingIDs := make([]string, 0) + missingSeen := make(map[string]struct{}) + for _, obj := range objects { + if _, seen := seenIDs[obj.Id]; seen { + continue + } + seenIDs[obj.Id] = struct{}{} + expanded = append(expanded, obj) + } + for _, sha := range checksums { + for _, id := range idsByChecksum[sha] { + if _, seen := seenIDs[id]; seen { + continue + } + if _, queued := missingSeen[id]; queued { + continue + } + missingSeen[id] = struct{}{} + missingIDs = append(missingIDs, id) + } + } + + hydrateStart := time.Now() + if len(missingIDs) > 0 { + siblings, err := m.db.GetBulkObjects(ctx, missingIDs) + if err != nil { + return nil, err + } + for _, obj := range siblings { + key, ok := canonicalProjectChecksumKey(&obj, "") + if !ok { + continue + } + if _, wanted := wantedKeys[key]; !wanted { + continue + } + if _, seen := seenIDs[obj.Id]; seen { + continue + } + seenIDs[obj.Id] = struct{}{} + expanded = append(expanded, obj) + } + } + hydrateDuration := time.Since(hydrateStart) + log.Printf("INFO: syfon_expand_scoped_checksum_siblings organization=%s project=%s checksums=%d input_count=%d missing_ids=%d output_count=%d list_scoped_sibling_ids_ms=%d hydrate_missing_siblings_ms=%d", strings.TrimSpace(organization), strings.TrimSpace(project), len(checksums), len(objects), len(missingIDs), len(expanded), listDuration.Milliseconds(), hydrateDuration.Milliseconds()) + return expanded, nil +} + func objectHasAccessURL(obj *models.InternalObject, objectURL string) bool { if obj == nil || obj.AccessMethods == nil { return false diff --git a/internal/core/object_reads_test.go b/internal/core/object_reads_test.go index d66aad41..00454158 100644 --- a/internal/core/object_reads_test.go +++ b/internal/core/object_reads_test.go @@ -2,13 +2,36 @@ package core import ( "context" + "slices" "testing" + "time" "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/db" + "github.com/calypr/syfon/internal/db/sqlite" "github.com/calypr/syfon/internal/models" ) +type pageSpyDB struct { + db.DatabaseInterface + pageCalls int + listCalls int +} + +func (s *pageSpyDB) ListObjectIDsPageByScope(ctx context.Context, organization, project, startAfter string, limit, offset int) ([]string, error) { + s.pageCalls++ + return s.DatabaseInterface.(db.ObjectIDPageLister).ListObjectIDsPageByScope(ctx, organization, project, startAfter, limit, offset) +} + +func (s *pageSpyDB) ListObjectIDsPageByResources(ctx context.Context, resources []string, includeUnscoped bool, startAfter string, limit, offset int) ([]string, error) { + return s.DatabaseInterface.(db.ObjectIDPageLister).ListObjectIDsPageByResources(ctx, resources, includeUnscoped, startAfter, limit, offset) +} + +func (s *pageSpyDB) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) { + s.listCalls++ + return s.DatabaseInterface.ListObjectIDsByScope(ctx, organization, project) +} + func registerScopedCandidate(t *testing.T, om *ObjectManager, id, checksum, org, project string) { t.Helper() controlled := []string{"/organization/" + org + "/project/" + project} @@ -39,8 +62,8 @@ func TestListObjectIDsPageByChecksum_StartAfterAndLimit(t *testing.T) { checksum := "1111111111111111111111111111111111111111111111111111111111111111" registerScopedCandidate(t, om, "chk-a", checksum, "org1", "proj1") - registerScopedCandidate(t, om, "chk-b", checksum, "org1", "proj1") - registerScopedCandidate(t, om, "chk-c", checksum, "org1", "proj1") + registerScopedCandidate(t, om, "chk-b", checksum, "org1", "proj2") + registerScopedCandidate(t, om, "chk-c", checksum, "org2", "proj1") ids, err := om.ListObjectIDsPageByChecksum(context.Background(), checksum, "sha256", "", "", "read", "chk-a", 2, 0) if err != nil { @@ -70,6 +93,53 @@ func TestListObjectIDsPageByScope_StartAfterAndScopeFilter(t *testing.T) { } } +func TestListObjectIDsPageByScope_UsesDatabasePaginationForUnrestrictedScope(t *testing.T) { + database := &pageSpyDB{DatabaseInterface: db.NewInMemoryDB()} + om := NewObjectManager(database, nil) + + registerScopedCandidate(t, om, "scope-a", "2222222222222222222222222222222222222222222222222222222222222222", "org1", "proj1") + registerScopedCandidate(t, om, "scope-b", "3333333333333333333333333333333333333333333333333333333333333333", "org1", "proj1") + registerScopedCandidate(t, om, "scope-c", "4444444444444444444444444444444444444444444444444444444444444444", "org1", "proj1") + + ids, err := om.ListObjectIDsPageByScope(context.Background(), "org1", "proj1", "read", "scope-a", 1, 0) + if err != nil { + t.Fatalf("ListObjectIDsPageByScope error: %v", err) + } + if len(ids) != 1 || ids[0] != "scope-b" { + t.Fatalf("unexpected scoped page ids: %+v", ids) + } + if database.pageCalls != 1 { + t.Fatalf("expected one database page call, got %d", database.pageCalls) + } + if database.listCalls != 0 { + t.Fatalf("expected no full scope list calls, got %d", database.listCalls) + } +} + +func TestListObjectIDsPageByScope_FallsBackWhenAuthzRestrictsResources(t *testing.T) { + database := &pageSpyDB{DatabaseInterface: db.NewInMemoryDB()} + om := NewObjectManager(database, nil) + + registerScopedCandidate(t, om, "secure-obj", "5555555555555555555555555555555555555555555555555555555555555555", "secure", "p1") + restrictedCtx := buildLocalAuthzContext(map[string]map[string]bool{ + "/organization/other/project/p2": {"read": true}, + }) + + ids, err := om.ListObjectIDsPageByScope(restrictedCtx, "secure", "p1", "read", "", 10, 0) + if err != nil { + t.Fatalf("ListObjectIDsPageByScope error: %v", err) + } + if len(ids) != 0 { + t.Fatalf("expected authz fallback to filter ids, got %+v", ids) + } + if database.pageCalls != 0 { + t.Fatalf("expected no unrestricted page calls, got %d", database.pageCalls) + } + if database.listCalls == 0 { + t.Fatalf("expected fallback to full scope list") + } +} + func TestListObjectIDsByScope_AuthzFiltering(t *testing.T) { database := db.NewInMemoryDB() om := NewObjectManager(database, nil) @@ -126,6 +196,105 @@ func TestObjectMatchesScope(t *testing.T) { } } +type trackingDB struct { + db.DatabaseInterface + bulkCalls [][]string +} + +func (t *trackingDB) GetBulkObjects(ctx context.Context, ids []string) ([]models.InternalObject, error) { + copyIDs := append([]string(nil), ids...) + t.bulkCalls = append(t.bulkCalls, copyIDs) + return t.DatabaseInterface.GetBulkObjects(ctx, ids) +} + +func TestPrepareScopedObjects_HydratesOnlyMissingSiblingIDs(t *testing.T) { + base, err := sqlite.NewSqliteDB(":memory:") + if err != nil { + t.Fatalf("NewSqliteDB failed: %v", err) + } + tracked := &trackingDB{DatabaseInterface: base} + om := NewObjectManager(tracked, nil) + checksum := "6666666666666666666666666666666666666666666666666666666666666666" + controlled := []string{"/organization/org/project/proj"} + + for _, obj := range []models.InternalObject{ + { + Authorizations: map[string][]string{"org": {"proj"}}, + DrsObject: drs.DrsObject{ + Id: "dup-a", + CreatedTime: drsISOTime("2026-01-01T00:00:00Z"), + UpdatedTime: ptrTime("2026-01-01T00:00:00Z"), + Checksums: []drs.Checksum{{Type: "sha256", Checksum: checksum}}, + ControlledAccess: &controlled, + AccessMethods: &[]drs.AccessMethod{{ + Type: drs.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/dup-a"}, + }}, + }, + }, + { + Authorizations: map[string][]string{"org": {"proj"}}, + DrsObject: drs.DrsObject{ + Id: "dup-b", + CreatedTime: drsISOTime("2026-01-02T00:00:00Z"), + UpdatedTime: ptrTime("2026-01-02T00:00:00Z"), + Checksums: []drs.Checksum{{Type: "sha256", Checksum: checksum}}, + ControlledAccess: &controlled, + AccessMethods: &[]drs.AccessMethod{{ + Type: drs.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: "s3://bucket/dup-b"}, + }}, + }, + }, + } { + if err := tracked.CreateObject(context.Background(), &obj); err != nil { + t.Fatalf("CreateObject failed: %v", err) + } + } + + initial, err := tracked.GetBulkObjects(context.Background(), []string{"dup-a"}) + if err != nil { + t.Fatalf("GetBulkObjects failed: %v", err) + } + tracked.bulkCalls = nil + + prepared, err := om.PrepareScopedObjects(context.Background(), initial, "org", "proj", "") + if err != nil { + t.Fatalf("PrepareScopedObjects failed: %v", err) + } + if len(prepared) != 1 { + t.Fatalf("expected 1 canonical record, got %d", len(prepared)) + } + if prepared[0].AccessMethods == nil || len(*prepared[0].AccessMethods) != 2 { + t.Fatalf("expected merged access methods, got %+v", prepared[0].AccessMethods) + } + if len(tracked.bulkCalls) != 1 { + t.Fatalf("expected 1 sibling hydration call, got %d", len(tracked.bulkCalls)) + } + if !slices.Equal(tracked.bulkCalls[0], []string{"dup-b"}) { + t.Fatalf("expected only missing sibling id to be hydrated, got %+v", tracked.bulkCalls[0]) + } +} + +func drsISOTime(raw string) time.Time { + tm, err := time.Parse(time.RFC3339, raw) + if err != nil { + panic(err) + } + return tm +} + +func ptrTime(raw string) *time.Time { + tm := drsISOTime(raw) + return &tm +} + func TestReadableChecksumFilter(t *testing.T) { database := db.NewInMemoryDB() om := NewObjectManager(database, nil) diff --git a/internal/core/object_storage_delete.go b/internal/core/object_storage_delete.go index 45d106e2..8c58ac8d 100644 --- a/internal/core/object_storage_delete.go +++ b/internal/core/object_storage_delete.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "path/filepath" + "sort" "strings" "cloud.google.com/go/storage" @@ -17,6 +18,7 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" @@ -31,16 +33,78 @@ type storageTarget struct { path string } +type s3ObjectDeleter interface { + DeleteObject(context.Context, *awss3.DeleteObjectInput, ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) + DeleteObjects(context.Context, *awss3.DeleteObjectsInput, ...func(*awss3.Options)) (*awss3.DeleteObjectsOutput, error) +} + +var newS3ObjectDeleter = func(ctx context.Context, cred *models.S3Credential) (s3ObjectDeleter, error) { + cfg, err := awsConfigForStorageCredential(ctx, cred) + if err != nil { + return nil, err + } + return awss3.NewFromConfig(cfg, func(o *awss3.Options) { + if strings.TrimSpace(cred.Endpoint) != "" { + o.UsePathStyle = true + } + }), nil +} + func (m *ObjectManager) deleteObjectStorage(ctx context.Context, obj *models.InternalObject) error { targets, err := m.storageTargetsForObject(ctx, obj) if err != nil { return err } + return m.deleteStorageTargets(ctx, targets) +} + +func (m *ObjectManager) deleteObjectsStorage(ctx context.Context, objects []models.InternalObject) error { + targets := make([]storageTarget, 0, len(objects)) + seen := make(map[string]struct{}) + for i := range objects { + objectTargets, err := m.storageTargetsForObject(ctx, &objects[i]) + if err != nil { + return err + } + for _, target := range objectTargets { + key := storageTargetKey(target) + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + targets = append(targets, target) + } + } + return m.deleteStorageTargets(ctx, targets) +} + +func (m *ObjectManager) deleteStorageTargets(ctx context.Context, targets []storageTarget) error { + s3ByBucket := make(map[string][]string) for _, target := range targets { + if target.provider == common.S3Provider { + bucket := strings.TrimSpace(target.bucket) + key := strings.Trim(strings.TrimSpace(target.key), "/") + if bucket == "" || key == "" { + continue + } + s3ByBucket[bucket] = append(s3ByBucket[bucket], key) + continue + } if err := m.deleteStorageTarget(ctx, target); err != nil { return err } } + buckets := make([]string, 0, len(s3ByBucket)) + for bucket := range s3ByBucket { + buckets = append(buckets, bucket) + } + sort.Strings(buckets) + for _, bucket := range buckets { + keys := dedupeSortedStrings(s3ByBucket[bucket]) + if err := m.deleteS3Objects(ctx, bucket, keys); err != nil { + return err + } + } return nil } @@ -85,7 +149,7 @@ func (m *ObjectManager) storageTargetsForObject(ctx context.Context, obj *models continue } - key := target.provider + "\x00" + target.bucket + "\x00" + target.key + "\x00" + target.path + key := storageTargetKey(target) if _, exists := seen[key]; exists { continue } @@ -106,7 +170,7 @@ func (m *ObjectManager) storageTargetFromURL(ctx context.Context, raw string) (s if bucket == "" || key == "" { return storageTarget{}, false, nil } - cred, err := m.db.GetS3Credential(ctx, bucket) + cred, err := m.credentialForBucket(ctx, bucket) if err != nil { return storageTarget{}, false, fmt.Errorf("lookup credential for bucket %s: %w", bucket, err) } @@ -163,21 +227,70 @@ func (m *ObjectManager) deleteStorageTarget(ctx context.Context, target storageT } func (m *ObjectManager) deleteS3Object(ctx context.Context, bucket, key string) error { - cred, err := m.db.GetS3Credential(ctx, bucket) + return m.deleteS3Objects(ctx, bucket, []string{key}) +} + +func (m *ObjectManager) deleteS3Objects(ctx context.Context, bucket string, keys []string) error { + keys = dedupeSortedStrings(keys) + if len(keys) == 0 { + return nil + } + cred, err := m.credentialForBucket(ctx, bucket) if err != nil { return fmt.Errorf("lookup s3 credential for bucket %s: %w", bucket, err) } if cred == nil { return fmt.Errorf("credentials not found for bucket %s", bucket) } + client, err := newS3ObjectDeleter(ctx, cred) + if err != nil { + return err + } + + if len(keys) == 1 { + _, err = client.DeleteObject(ctx, &awss3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(keys[0]), + }) + return err + } + + const maxS3DeleteObjects = 1000 + quiet := true + for start := 0; start < len(keys); start += maxS3DeleteObjects { + end := start + maxS3DeleteObjects + if end > len(keys) { + end = len(keys) + } + objects := make([]s3types.ObjectIdentifier, 0, end-start) + for _, key := range keys[start:end] { + objects = append(objects, s3types.ObjectIdentifier{Key: aws.String(key)}) + } + out, err := client.DeleteObjects(ctx, &awss3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &s3types.Delete{ + Objects: objects, + Quiet: aws.Bool(quiet), + }, + }) + if err != nil { + return err + } + if out != nil && len(out.Errors) > 0 { + return fmt.Errorf("s3 bulk delete failed for bucket %s: %s", bucket, formatS3DeleteErrors(out.Errors)) + } + } + return nil +} +func awsConfigForStorageCredential(ctx context.Context, cred *models.S3Credential) (aws.Config, error) { loadOpts := []func(*awsconfig.LoadOptions) error{ awsconfig.WithRegion(cred.Region), awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cred.AccessKey, cred.SecretKey, "")), } cfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...) if err != nil { - return fmt.Errorf("load aws config: %w", err) + return aws.Config{}, fmt.Errorf("load aws config: %w", err) } if endpoint := strings.TrimSpace(cred.Endpoint); endpoint != "" { if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { @@ -189,17 +302,57 @@ func (m *ObjectManager) deleteS3Object(ctx context.Context, bucket, key string) } cfg.BaseEndpoint = aws.String(endpoint) } + return cfg, nil +} + +func storageTargetKey(target storageTarget) string { + return target.provider + "\x00" + target.bucket + "\x00" + target.key + "\x00" + target.path +} + +func dedupeSortedStrings(values []string) []string { + if len(values) == 0 { + return nil + } + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + sort.Strings(out) + return out +} - client := awss3.NewFromConfig(cfg, func(o *awss3.Options) { - if strings.TrimSpace(cred.Endpoint) != "" { - o.UsePathStyle = true +func formatS3DeleteErrors(errors []s3types.Error) string { + const maxErrors = 5 + parts := make([]string, 0, len(errors)) + for i, item := range errors { + if i >= maxErrors { + parts = append(parts, fmt.Sprintf("and %d more", len(errors)-maxErrors)) + break } - }) - _, err = client.DeleteObject(ctx, &awss3.DeleteObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) - return err + key := strings.TrimSpace(aws.ToString(item.Key)) + code := strings.TrimSpace(aws.ToString(item.Code)) + message := strings.TrimSpace(aws.ToString(item.Message)) + switch { + case key != "" && code != "" && message != "": + parts = append(parts, fmt.Sprintf("%s: %s: %s", key, code, message)) + case key != "" && code != "": + parts = append(parts, fmt.Sprintf("%s: %s", key, code)) + case code != "": + parts = append(parts, code) + default: + parts = append(parts, "unknown delete error") + } + } + return strings.Join(parts, "; ") } func (m *ObjectManager) deleteGCSObject(ctx context.Context, bucket, key string) error { diff --git a/internal/core/object_storage_delete_test.go b/internal/core/object_storage_delete_test.go index cf33d854..1e4dea80 100644 --- a/internal/core/object_storage_delete_test.go +++ b/internal/core/object_storage_delete_test.go @@ -2,11 +2,15 @@ package core import ( "context" + "fmt" "os" "path/filepath" + "reflect" "strings" "testing" + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/calypr/syfon/apigen/server/drs" "github.com/calypr/syfon/internal/common" "github.com/calypr/syfon/internal/models" @@ -81,6 +85,87 @@ func TestDeleteStorageTargetFileProvider(t *testing.T) { } } +func TestBulkDeleteObjectsWithOptionsUsesS3BatchDelete(t *testing.T) { + deleter := &fakeS3ObjectDeleter{} + restore := replaceS3ObjectDeleterForTest(deleter) + defer restore() + + accessMethods := func(rawURL string) *[]drs.AccessMethod { + return &[]drs.AccessMethod{{ + Type: drs.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: rawURL}, + }} + } + db := &testutils.MockDatabase{ + Objects: map[string]*drs.DrsObject{ + "obj-1": {Id: "obj-1", AccessMethods: accessMethods("s3://bucket/path/a.txt")}, + "obj-2": {Id: "obj-2", AccessMethods: accessMethods("s3://bucket/path/b.txt")}, + "obj-3": {Id: "obj-3", AccessMethods: accessMethods("s3://bucket/path/a.txt")}, + }, + Credentials: map[string]models.S3Credential{ + "bucket": { + Bucket: "bucket", + Provider: common.S3Provider, + Region: "us-east-1", + AccessKey: "test-key", + SecretKey: "test-secret", + }, + }, + } + om := NewObjectManager(db, &capturingURLManager{}) + + if err := om.BulkDeleteObjectsWithOptions(context.Background(), []string{"obj-1", "obj-2", "obj-3"}, DeleteOptions{DeleteStorageData: true}); err != nil { + t.Fatalf("BulkDeleteObjectsWithOptions failed: %v", err) + } + if deleter.deleteObjectCalls != 0 { + t.Fatalf("expected no single DeleteObject calls, got %d", deleter.deleteObjectCalls) + } + if len(deleter.deleteObjectsKeys) != 1 { + t.Fatalf("expected one DeleteObjects call, got %d", len(deleter.deleteObjectsKeys)) + } + if got, want := deleter.deleteObjectsKeys[0], []string{"path/a.txt", "path/b.txt"}; !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected batch keys: got %+v want %+v", got, want) + } + for _, id := range []string{"obj-1", "obj-2", "obj-3"} { + if _, exists := db.Objects[id]; exists { + t.Fatalf("expected %s to be removed from db", id) + } + } +} + +func TestDeleteS3ObjectsChunksLargeBatches(t *testing.T) { + deleter := &fakeS3ObjectDeleter{} + restore := replaceS3ObjectDeleterForTest(deleter) + defer restore() + + keys := make([]string, 0, 1002) + for i := 0; i < 1002; i++ { + keys = append(keys, fmt.Sprintf("key-%04d", i)) + } + keys = append(keys, keys[0]) + om := NewObjectManager(&testutils.MockDatabase{ + Credentials: map[string]models.S3Credential{ + "bucket": {Bucket: "bucket", Provider: common.S3Provider, Region: "us-east-1"}, + }, + }, &capturingURLManager{}) + + if err := om.deleteS3Objects(context.Background(), "bucket", keys); err != nil { + t.Fatalf("deleteS3Objects failed: %v", err) + } + if len(deleter.deleteObjectsKeys) != 2 { + t.Fatalf("expected two DeleteObjects chunks, got %d", len(deleter.deleteObjectsKeys)) + } + if got := len(deleter.deleteObjectsKeys[0]); got != 1000 { + t.Fatalf("expected first chunk to contain 1000 keys, got %d", got) + } + if got := len(deleter.deleteObjectsKeys[1]); got != 2 { + t.Fatalf("expected second chunk to contain 2 keys, got %d", got) + } +} + func TestStorageTargetsForScopedObjectUseCanonicalChecksumPath(t *testing.T) { checksum := strings.Repeat("a", 64) om := NewObjectManager(&testutils.MockDatabase{ @@ -132,6 +217,35 @@ func TestStorageTargetsForScopedObjectUseCanonicalChecksumPath(t *testing.T) { } } +type fakeS3ObjectDeleter struct { + deleteObjectCalls int + deleteObjectsKeys [][]string +} + +func (f *fakeS3ObjectDeleter) DeleteObject(context.Context, *awss3.DeleteObjectInput, ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) { + f.deleteObjectCalls++ + return &awss3.DeleteObjectOutput{}, nil +} + +func (f *fakeS3ObjectDeleter) DeleteObjects(_ context.Context, input *awss3.DeleteObjectsInput, _ ...func(*awss3.Options)) (*awss3.DeleteObjectsOutput, error) { + keys := make([]string, 0, len(input.Delete.Objects)) + for _, obj := range input.Delete.Objects { + keys = append(keys, aws.ToString(obj.Key)) + } + f.deleteObjectsKeys = append(f.deleteObjectsKeys, keys) + return &awss3.DeleteObjectsOutput{}, nil +} + +func replaceS3ObjectDeleterForTest(deleter s3ObjectDeleter) func() { + previous := newS3ObjectDeleter + newS3ObjectDeleter = func(context.Context, *models.S3Credential) (s3ObjectDeleter, error) { + return deleter, nil + } + return func() { + newS3ObjectDeleter = previous + } +} + func TestScopedStorageHelperUtilities(t *testing.T) { bucket, key, ok := parseS3Location("s3://bucket-name/path/to/object") if !ok || bucket != "bucket-name" || key != "path/to/object" { diff --git a/internal/core/object_storage_inspect.go b/internal/core/object_storage_inspect.go index 3cdcd766..cf4f8ee8 100644 --- a/internal/core/object_storage_inspect.go +++ b/internal/core/object_storage_inspect.go @@ -4,8 +4,11 @@ import ( "context" "errors" "fmt" + "log" "path" + "sort" "strings" + "sync" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -20,23 +23,67 @@ import ( ) type InspectStorageRequest struct { - Organization string - Project string - Key string - Scheme string - ObjectURL string + ID string + Organization string + Project string + Key string + Scheme string + ObjectURL string + ExpectedSizeBytes *int64 + ExpectedSHA256 string } type StorageObjectMetadata struct { - ObjectURL string - Provider string - Bucket string - Key string - Path string - SizeBytes int64 - MetaSHA256 string - ETag string - LastModTime time.Time + ObjectURL string + Provider string + Bucket string + Key string + Path string + SizeBytes int64 + MetaSHA256 string + ETag string + LastModTime time.Time +} + +type StorageProbeStatus string + +const ( + StorageProbeStatusPresent StorageProbeStatus = "present" + StorageProbeStatusNotFound StorageProbeStatus = "not_found" + StorageProbeStatusForbidden StorageProbeStatus = "forbidden" + StorageProbeStatusInvalid StorageProbeStatus = "invalid" + StorageProbeStatusUnsupported StorageProbeStatus = "unsupported" + StorageProbeStatusError StorageProbeStatus = "error" +) + +type StorageValidationStatus string + +const ( + StorageValidationNotRequested StorageValidationStatus = "not_requested" + StorageValidationMatched StorageValidationStatus = "matched" + StorageValidationMismatched StorageValidationStatus = "mismatched" + StorageValidationUnverifiable StorageValidationStatus = "unverifiable" +) + +type StorageProbeResult struct { + ID string + ObjectURL string + Provider string + Bucket string + Key string + Path string + Exists bool + Status StorageProbeStatus + Error string + ErrorKind string + SizeBytes *int64 + MetaSHA256 string + ETag string + LastModTime time.Time + ValidationStatus StorageValidationStatus + SizeMatch *bool + SHA256Match *bool + ValidationMismatches []string } type StorageInspectErrorKind string @@ -47,14 +94,44 @@ const ( StorageInspectCredentialMissing StorageInspectErrorKind = "credential_missing" StorageInspectPermissionDenied StorageInspectErrorKind = "permission_denied" StorageInspectObjectNotFound StorageInspectErrorKind = "object_not_found" + StorageInspectBucketUnavailable StorageInspectErrorKind = "bucket_unavailable" + StorageInspectListingIncomplete StorageInspectErrorKind = "listing_incomplete" StorageInspectUnsupported StorageInspectErrorKind = "unsupported" ) +const maxStorageInspectWorkers = 8 + +const ( + defaultS3HeadMaxAttempts = 3 + envS3HeadMaxAttempts = "SYFON_S3_HEAD_MAX_ATTEMPTS" +) + type StorageInspectError struct { Kind StorageInspectErrorKind Message string } +type s3HeadObjectClient interface { + HeadObject(context.Context, *awss3.HeadObjectInput, ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) +} + +var storageInspectCacheKey contextKey = "storageInspectCache" + +type storageInspectCredentialCacheEntry struct { + cred *models.S3Credential + err error +} + +type storageInspectRequestCache struct { + mu sync.Mutex + + credentials map[string]storageInspectCredentialCacheEntry + s3Clients map[string]*awss3.Client + visible map[string]VisibleBucket + visibleErr error + visibleLoaded bool +} + func (e *StorageInspectError) Error() string { if e == nil { return "storage inspect failed" @@ -65,6 +142,21 @@ func (e *StorageInspectError) Error() string { return fmt.Sprintf("storage inspect failed: %s", e.Kind) } +func WithStorageInspectCache(ctx context.Context) context.Context { + if storageInspectCacheFromContext(ctx) != nil { + return ctx + } + return context.WithValue(ctx, storageInspectCacheKey, &storageInspectRequestCache{ + credentials: map[string]storageInspectCredentialCacheEntry{}, + s3Clients: map[string]*awss3.Client{}, + }) +} + +func storageInspectCacheFromContext(ctx context.Context) *storageInspectRequestCache { + cache, _ := ctx.Value(storageInspectCacheKey).(*storageInspectRequestCache) + return cache +} + func (m *ObjectManager) SetS3ObjectInspector(fn func(context.Context, models.S3Credential, string, string) (*StorageObjectMetadata, error)) { if fn == nil { m.inspectS3Object = defaultS3ObjectInspector @@ -74,12 +166,196 @@ func (m *ObjectManager) SetS3ObjectInspector(fn func(context.Context, models.S3C } func (m *ObjectManager) InspectStorageObject(ctx context.Context, req InspectStorageRequest) (*StorageObjectMetadata, error) { + ctx = withS3ProbeLimiter(ctx, m.s3ProbeLimiter) if strings.TrimSpace(req.ObjectURL) != "" { return m.inspectRawStorageObject(ctx, req) } return m.inspectScopedStorageObject(ctx, req) } +func (m *ObjectManager) InspectStorageObjects(ctx context.Context, items []InspectStorageRequest) []StorageProbeResult { + ctx = withS3ProbeLimiter(ctx, m.s3ProbeLimiter) + ctx = WithStorageInspectCache(ctx) + if len(items) == 0 { + return []StorageProbeResult{} + } + results := make([]StorageProbeResult, len(items)) + workers := len(items) + if workers > maxStorageInspectWorkers { + workers = maxStorageInspectWorkers + } + type probeWork struct { + index int + req InspectStorageRequest + } + workCh := make(chan probeWork) + cache := newStorageProbeResultCache() + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for work := range workCh { + results[work.index] = cache.probe(ctx, m, work.req) + } + }() + } + for i, item := range items { + workCh <- probeWork{index: i, req: item} + } + close(workCh) + wg.Wait() + return results +} + +type storageProbeResultCache struct { + mu sync.Mutex + results map[string]StorageProbeResult +} + +func newStorageProbeResultCache() *storageProbeResultCache { + return &storageProbeResultCache{results: map[string]StorageProbeResult{}} +} + +func (c *storageProbeResultCache) probe(ctx context.Context, m *ObjectManager, req InspectStorageRequest) StorageProbeResult { + key := storageProbeCacheKey(req) + c.mu.Lock() + if result, ok := c.results[key]; ok { + c.mu.Unlock() + return cloneStorageProbeResultForRequest(result, req) + } + c.mu.Unlock() + + result := m.inspectStorageProbe(ctx, req) + + c.mu.Lock() + c.results[key] = result + c.mu.Unlock() + return result +} + +func storageProbeCacheKey(req InspectStorageRequest) string { + return strings.TrimSpace(req.ObjectURL) + "|" + + strings.TrimSpace(req.Organization) + "|" + + strings.TrimSpace(req.Project) + "|" + + strings.TrimSpace(req.Key) + "|" + + strings.TrimSpace(req.Scheme) + "|" + + normalizeExpectedSize(req.ExpectedSizeBytes) + "|" + + strings.ToLower(strings.TrimSpace(strings.TrimPrefix(req.ExpectedSHA256, "sha256:"))) +} + +func normalizeExpectedSize(v *int64) string { + if v == nil { + return "" + } + return fmt.Sprintf("%d", *v) +} + +func cloneStorageProbeResultForRequest(result StorageProbeResult, req InspectStorageRequest) StorageProbeResult { + clone := result + clone.ID = strings.TrimSpace(req.ID) + if clone.ID == "" { + clone.ID = strings.TrimSpace(result.ID) + } + if clone.ObjectURL == "" { + clone.ObjectURL = strings.TrimSpace(req.ObjectURL) + } + clone.ValidationMismatches = append([]string(nil), result.ValidationMismatches...) + return clone +} + +func (m *ObjectManager) inspectStorageProbe(ctx context.Context, req InspectStorageRequest) StorageProbeResult { + result := StorageProbeResult{ + ID: strings.TrimSpace(req.ID), + ObjectURL: strings.TrimSpace(req.ObjectURL), + Status: StorageProbeStatusError, + ValidationStatus: StorageValidationNotRequested, + } + meta, err := m.InspectStorageObject(ctx, req) + if err != nil { + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = strings.TrimSpace(err.Error()) + result.Exists = false + result.ValidationStatus = storageValidationStatusForError(req) + return result + } + result.ObjectURL = strings.TrimSpace(meta.ObjectURL) + result.Provider = strings.TrimSpace(meta.Provider) + result.Bucket = strings.TrimSpace(meta.Bucket) + result.Key = strings.TrimSpace(meta.Key) + result.Path = strings.TrimSpace(meta.Path) + result.Exists = true + result.Status = StorageProbeStatusPresent + result.SizeBytes = ptrInt64(meta.SizeBytes) + result.MetaSHA256 = strings.TrimSpace(meta.MetaSHA256) + result.ETag = strings.TrimSpace(meta.ETag) + result.LastModTime = meta.LastModTime + result.ValidationStatus, result.SizeMatch, result.SHA256Match, result.ValidationMismatches = validateStorageProbe(req, meta) + return result +} + +func classifyStorageProbeError(err error) (StorageProbeStatus, string) { + var inspectErr *StorageInspectError + if errors.As(err, &inspectErr) { + switch inspectErr.Kind { + case StorageInspectObjectNotFound: + return StorageProbeStatusNotFound, string(inspectErr.Kind) + case StorageInspectPermissionDenied, StorageInspectBucketUnavailable: + return StorageProbeStatusForbidden, string(inspectErr.Kind) + case StorageInspectInvalidInput, StorageInspectScopeNotFound, StorageInspectCredentialMissing: + return StorageProbeStatusInvalid, string(inspectErr.Kind) + case StorageInspectUnsupported: + return StorageProbeStatusUnsupported, string(inspectErr.Kind) + default: + return StorageProbeStatusError, string(inspectErr.Kind) + } + } + return StorageProbeStatusError, "error" +} + +func storageValidationStatusForError(req InspectStorageRequest) StorageValidationStatus { + if req.ExpectedSizeBytes == nil && strings.TrimSpace(req.ExpectedSHA256) == "" { + return StorageValidationNotRequested + } + return StorageValidationUnverifiable +} + +func validateStorageProbe(req InspectStorageRequest, meta *StorageObjectMetadata) (StorageValidationStatus, *bool, *bool, []string) { + if req.ExpectedSizeBytes == nil && strings.TrimSpace(req.ExpectedSHA256) == "" { + return StorageValidationNotRequested, nil, nil, nil + } + mismatches := make([]string, 0, 2) + var sizeMatch *bool + if req.ExpectedSizeBytes != nil { + matched := meta.SizeBytes == *req.ExpectedSizeBytes + sizeMatch = &matched + if !matched { + mismatches = append(mismatches, "size_mismatch") + } + } + var shaMatch *bool + expectedSHA := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(req.ExpectedSHA256, "sha256:"))) + if expectedSHA != "" { + if strings.TrimSpace(meta.MetaSHA256) == "" { + return StorageValidationUnverifiable, sizeMatch, nil, append(mismatches, "missing_remote_sha256") + } + matched := strings.EqualFold(strings.TrimSpace(meta.MetaSHA256), expectedSHA) + shaMatch = &matched + if !matched { + mismatches = append(mismatches, "sha256_mismatch") + } + } + if len(mismatches) > 0 { + return StorageValidationMismatched, sizeMatch, shaMatch, mismatches + } + return StorageValidationMatched, sizeMatch, shaMatch, nil +} + +func ptrInt64(v int64) *int64 { + copy := v + return © +} + func (m *ObjectManager) inspectScopedStorageObject(ctx context.Context, req InspectStorageRequest) (*StorageObjectMetadata, error) { organization := strings.TrimSpace(req.Organization) project := strings.TrimSpace(req.Project) @@ -173,20 +449,38 @@ func (m *ObjectManager) credentialForBucket(ctx context.Context, bucket string) if bucket == "" { return nil, &StorageInspectError{Kind: StorageInspectInvalidInput, Message: "bucket is required"} } + if cache := storageInspectCacheFromContext(ctx); cache != nil { + if cred, err, ok := cache.getCredential(bucket); ok { + return cred, err + } + } if cred, err := m.db.GetS3Credential(ctx, bucket); err == nil && cred != nil { + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setCredential(bucket, cred, nil) + } return cred, nil } creds, err := m.db.ListS3Credentials(ctx) if err != nil { + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setCredential(bucket, nil, err) + } return nil, err } for _, cred := range creds { if strings.EqualFold(strings.TrimSpace(cred.Bucket), bucket) || strings.EqualFold(strings.TrimSpace(cred.CredentialID), bucket) { copy := cred + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setCredential(bucket, ©, nil) + } return ©, nil } } - return nil, &StorageInspectError{Kind: StorageInspectCredentialMissing, Message: fmt.Sprintf("no stored bucket credential found for bucket %q", bucket)} + err = &StorageInspectError{Kind: StorageInspectCredentialMissing, Message: fmt.Sprintf("no stored bucket credential found for bucket %q", bucket)} + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setCredential(bucket, nil, err) + } + return nil, err } func bucketVisibleToCaller(visible map[string]VisibleBucket, bucket string, credentialID string) bool { @@ -198,7 +492,101 @@ func bucketVisibleToCaller(visible map[string]VisibleBucket, bucket string, cred return false } -func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, bucket string, key string) (*StorageObjectMetadata, error) { +func (m *ObjectManager) listVisibleBucketsCached(ctx context.Context) (map[string]VisibleBucket, error) { + if cache := storageInspectCacheFromContext(ctx); cache != nil { + if visible, err, ok := cache.getVisible(); ok { + return visible, err + } + } + visible, err := m.listVisibleBucketsUncached(ctx) + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setVisible(visible, err) + } + return visible, err +} + +func (c *storageInspectRequestCache) getCredential(bucket string) (*models.S3Credential, error, bool) { + c.mu.Lock() + defer c.mu.Unlock() + entry, ok := c.credentials[strings.ToLower(strings.TrimSpace(bucket))] + if !ok { + return nil, nil, false + } + if entry.cred == nil { + return nil, entry.err, true + } + copy := *entry.cred + return ©, entry.err, true +} + +func (c *storageInspectRequestCache) setCredential(bucket string, cred *models.S3Credential, err error) { + c.mu.Lock() + defer c.mu.Unlock() + key := strings.ToLower(strings.TrimSpace(bucket)) + if cred == nil { + c.credentials[key] = storageInspectCredentialCacheEntry{err: err} + return + } + copy := *cred + c.credentials[key] = storageInspectCredentialCacheEntry{cred: ©, err: err} +} + +func (c *storageInspectRequestCache) getVisible() (map[string]VisibleBucket, error, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if !c.visibleLoaded { + return nil, nil, false + } + return cloneVisibleBuckets(c.visible), c.visibleErr, true +} + +func (c *storageInspectRequestCache) setVisible(visible map[string]VisibleBucket, err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.visible = cloneVisibleBuckets(visible) + c.visibleErr = err + c.visibleLoaded = true +} + +func (c *storageInspectRequestCache) getS3Client(key string) (*awss3.Client, bool) { + c.mu.Lock() + defer c.mu.Unlock() + client, ok := c.s3Clients[key] + return client, ok +} + +func (c *storageInspectRequestCache) setS3Client(key string, client *awss3.Client) { + if client == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.s3Clients[key] = client +} + +func cloneVisibleBuckets(in map[string]VisibleBucket) map[string]VisibleBucket { + if in == nil { + return nil + } + out := make(map[string]VisibleBucket, len(in)) + for key, bucket := range in { + programs := append([]string(nil), bucket.Programs...) + sort.Strings(programs) + out[key] = VisibleBucket{ + Credential: bucket.Credential, + Programs: programs, + } + } + return out +} + +func s3ClientFromContext(ctx context.Context, cred models.S3Credential) (*awss3.Client, error) { + cacheKey := s3ClientCacheKey(cred) + if cache := storageInspectCacheFromContext(ctx); cache != nil { + if client, ok := cache.getS3Client(cacheKey); ok { + return client, nil + } + } loadOpts := []func(*awsconfig.LoadOptions) error{ awsconfig.WithRegion(cred.Region), awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cred.AccessKey, cred.SecretKey, "")), @@ -222,18 +610,37 @@ func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, buc o.UsePathStyle = true } }) - out, err := client.HeadObject(ctx, &awss3.HeadObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) + if cache := storageInspectCacheFromContext(ctx); cache != nil { + cache.setS3Client(cacheKey, client) + } + return client, nil +} + +func s3ClientCacheKey(cred models.S3Credential) string { + return strings.ToLower(strings.TrimSpace(cred.Provider)) + "|" + + strings.TrimSpace(cred.Endpoint) + "|" + + strings.TrimSpace(cred.Region) + "|" + + strings.TrimSpace(cred.AccessKey) + "|" + + strings.TrimSpace(cred.Bucket) + "|" + + strings.TrimSpace(cred.CredentialID) +} + +func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, bucket string, key string) (*StorageObjectMetadata, error) { + client, err := s3ClientFromContext(ctx, cred) + if err != nil { + return nil, err + } + out, err := headS3ObjectWithRetry(ctx, client, bucket, key) if err != nil { var apiErr smithy.APIError if errors.As(err, &apiErr) { switch strings.ToLower(strings.TrimSpace(apiErr.ErrorCode())) { case "forbidden", "accessdenied", "permissiondenied": - return nil, &StorageInspectError{Kind: StorageInspectPermissionDenied, Message: fmt.Sprintf("provider denied access to s3://%s/%s", bucket, key)} - case "notfound", "nosuchkey", "nosuchbucket": + return nil, &StorageInspectError{Kind: StorageInspectBucketUnavailable, Message: fmt.Sprintf("provider rejected object probe for s3://%s/%s; mapped bucket target may be missing or inaccessible", bucket, key)} + case "notfound", "nosuchkey": return nil, &StorageInspectError{Kind: StorageInspectObjectNotFound, Message: fmt.Sprintf("provider could not find s3://%s/%s", bucket, key)} + case "nosuchbucket": + return nil, &StorageInspectError{Kind: StorageInspectBucketUnavailable, Message: fmt.Sprintf("provider could not find bucket %q", bucket)} } } return nil, fmt.Errorf("inspect s3 object %s/%s: %w", bucket, key, err) @@ -257,3 +664,25 @@ func defaultS3ObjectInspector(ctx context.Context, cred models.S3Credential, buc LastModTime: lastMod, }, nil } + +func headS3ObjectWithRetry(ctx context.Context, client s3HeadObjectClient, bucket, key string) (*awss3.HeadObjectOutput, error) { + policy := s3ListPageRetryPolicyFromEnv() + maxAttempts := intEnvOrDefault(envS3HeadMaxAttempts, defaultS3HeadMaxAttempts, 1) + for attempt := 1; attempt <= maxAttempts; attempt++ { + release, acquireErr := acquireS3Probe(ctx, "head", bucket, key) + if acquireErr != nil { + return nil, acquireErr + } + out, err := client.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}) + release() + if err == nil || !isRetryableS3ListPageError(err) || attempt == maxAttempts { + return out, err + } + backoff := policy.backoff(attempt) + log.Printf("INFO: syfon_s3_head_retry request_id=%s bucket=%s key=%q attempt=%d max_attempts=%d backoff_ms=%d error=%q", common.GetRequestID(ctx), bucket, key, attempt+1, maxAttempts, backoff.Milliseconds(), err.Error()) + if err := sleepS3ListPageRetry(ctx, backoff); err != nil { + return nil, err + } + } + return nil, nil +} diff --git a/internal/core/object_storage_inspect_test.go b/internal/core/object_storage_inspect_test.go new file mode 100644 index 00000000..fe6e0661 --- /dev/null +++ b/internal/core/object_storage_inspect_test.go @@ -0,0 +1,87 @@ +package core + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" +) + +func TestClassifyProbeErrorCredentialMissingIsNotBucketMiss(t *testing.T) { + status, kind := classifyStorageProbeError(&StorageInspectError{ + Kind: StorageInspectCredentialMissing, + Message: `no stored bucket credential found for bucket "missing"`, + }) + if status != StorageProbeStatusInvalid || kind != "credential_missing" { + t.Fatalf("expected credential_missing to classify as invalid status, got status=%q kind=%q", status, kind) + } +} + +func TestClassifyProbeErrorObjectNotFoundRemainsBucketMiss(t *testing.T) { + status, kind := classifyStorageProbeError(&StorageInspectError{ + Kind: StorageInspectObjectNotFound, + Message: "provider could not find s3://bucket/key", + }) + if status != StorageProbeStatusNotFound || kind != "object_not_found" { + t.Fatalf("expected object_not_found to classify as not_found, got status=%q kind=%q", status, kind) + } +} + +type fakeS3HeadClient struct { + calls atomic.Int32 +} + +func (client *fakeS3HeadClient) HeadObject(_ context.Context, _ *awss3.HeadObjectInput, _ ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) { + if client.calls.Add(1) == 1 { + return nil, &smithy.GenericAPIError{Code: "SlowDown", Message: "retry"} + } + return &awss3.HeadObjectOutput{ContentLength: aws.Int64(42)}, nil +} + +func TestHeadS3ObjectWithRetryRecoversTransientFailure(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3HeadMaxAttempts, "2") + client := &fakeS3HeadClient{} + + out, err := headS3ObjectWithRetry(context.Background(), client, "bucket", "key") + if err != nil { + t.Fatalf("head object: %v", err) + } + if out == nil || aws.ToInt64(out.ContentLength) != 42 || client.calls.Load() != 2 { + t.Fatalf("expected retry then successful HEAD, output=%+v calls=%d", out, client.calls.Load()) + } +} + +func TestS3ProbeLimiterSharesCapacityAcrossContexts(t *testing.T) { + limiter := newS3ProbeLimiter(1) + ctx := withS3ProbeLimiter(context.Background(), limiter) + release, err := acquireS3Probe(ctx, "head", "bucket", "first") + if err != nil { + t.Fatalf("acquire first permit: %v", err) + } + + acquired := make(chan struct{}) + go func() { + secondRelease, err := acquireS3Probe(ctx, "list", "bucket", "second") + if err == nil { + secondRelease() + close(acquired) + } + }() + select { + case <-acquired: + t.Fatal("second operation acquired before the first released its permit") + case <-time.After(20 * time.Millisecond): + } + release() + select { + case <-acquired: + case <-time.After(time.Second): + t.Fatal("second operation did not acquire after release") + } +} diff --git a/internal/core/object_storage_inventory.go b/internal/core/object_storage_inventory.go new file mode 100644 index 00000000..b37b2da9 --- /dev/null +++ b/internal/core/object_storage_inventory.go @@ -0,0 +1,1291 @@ +package core + +import ( + "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "math/rand" + "net" + "os" + "path" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + syfoncommon "github.com/calypr/syfon/common" + "github.com/calypr/syfon/internal/authz" + "github.com/calypr/syfon/internal/common" + "github.com/calypr/syfon/internal/models" +) + +type StorageBucketObject struct { + ObjectURL string + Provider string + Bucket string + Key string + Path string + SizeBytes int64 + MetaSHA256 string + ETag string + LastModTime time.Time +} + +type StoragePrefixListOptions struct { + IncludeHead bool + ExactPrefix bool + MaxKeys int32 +} + +type ProjectStorageInspectMode string + +const ( + ProjectStorageInspectItems ProjectStorageInspectMode = "items" + ProjectStorageInspectExists ProjectStorageInspectMode = "exists" + ProjectStorageInspectSummary ProjectStorageInspectMode = "summary" +) + +const ( + storageListCoalesceThreshold = 25 + storageListFallbackObjectLimit = 5000 + storageListSlowExactThreshold = time.Second + storageListSlowPrefixThreshold = 3 * time.Second + storagePrefixListLoggingDisabled = "disabled" +) + +const ( + defaultS3ListPageSize = int32(1000) + defaultS3ListPageMaxAttempts = 12 + defaultS3ExactProbeMaxAttempts = 3 + defaultS3ListPageInitialBackoff = 500 * time.Millisecond + defaultS3ListPageMaxBackoff = 10 * time.Second + defaultS3InventoryTerminalReplays = 3 + envS3ListPageMaxAttempts = "SYFON_S3_LIST_PAGE_MAX_ATTEMPTS" + envS3ExactProbeMaxAttempts = "SYFON_S3_EXACT_PROBE_MAX_ATTEMPTS" + envS3ListPageInitialBackoffMillis = "SYFON_S3_LIST_PAGE_INITIAL_BACKOFF_MS" + envS3ListPageMaxBackoffMillis = "SYFON_S3_LIST_PAGE_MAX_BACKOFF_MS" + envS3InventoryTerminalReplays = "SYFON_S3_INVENTORY_TERMINAL_REPLAY_ATTEMPTS" +) + +type ProjectStorageSummary struct { + Provider string + Bucket string + Prefix string + ObjectURL string + Exists bool + ObjectCount int + TotalBytes int64 + ComputedAt time.Time + Mode ProjectStorageInspectMode +} + +type ProjectStorageInspectResult struct { + Summary ProjectStorageSummary + Items []StorageBucketObject +} + +type ProjectStorageInspectOptions struct { + Mode ProjectStorageInspectMode + IncludeHead bool + PathPrefix string +} + +type storageListTargetWork struct { + bucket string + key string + cred models.S3Credential + baseResult StorageListValidationResult + requestIndexes []int +} + +type storageListRunStats struct { + mu sync.Mutex + inputItemCount int + distinctTargetCount int + duplicateCount int + bucketCount int + coalescedPrefixCount int + exactListCallCount int + prefixListPageCount int + fallbackCount int +} + +func (s *storageListRunStats) incrementExactListCalls() { + s.mu.Lock() + s.exactListCallCount++ + s.mu.Unlock() +} + +type StorageListValidationRequest struct { + ID string + ObjectURL string + ExpectedSizeBytes *int64 + ExpectedName string +} + +type StorageListValidationResult struct { + ID string + ObjectURL string + Provider string + Bucket string + Key string + Path string + Exists bool + Status StorageProbeStatus + Error string + ErrorKind string + SizeBytes *int64 + ETag string + LastModTime time.Time + ValidationStatus StorageValidationStatus + SizeMatch *bool + NameMatch *bool + ValidationMismatches []string +} + +type resolvedStorageScopeTarget struct { + provider string + bucket string + prefix string + cred models.S3Credential +} + +type ProjectStorageDeleteResult struct { + ObjectURL string + Status string + Error string +} + +type s3ListObjectsV2Client interface { + ListObjectsV2(context.Context, *awss3.ListObjectsV2Input, ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) +} + +type s3PrefixListStats struct { + Pages int + Retries int + LastKey string + FailedPage int + LastTokenID string + TerminalReplayAttempts int + TerminalDisagreements int +} + +type s3ListPageRetryPolicy struct { + MaxAttempts int + InitialBackoff time.Duration + MaxBackoff time.Duration +} + +func (m *ObjectManager) SetS3PrefixLister(fn func(context.Context, models.S3Credential, string, string, bool) ([]StorageBucketObject, error)) { + if fn == nil { + m.listS3Prefix = defaultS3PrefixLister + return + } + m.listS3Prefix = func(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options StoragePrefixListOptions) ([]StorageBucketObject, error) { + return fn(ctx, cred, bucket, prefix, options.IncludeHead) + } +} + +func (m *ObjectManager) SetS3PrefixListerWithOptions(fn func(context.Context, models.S3Credential, string, string, StoragePrefixListOptions) ([]StorageBucketObject, error)) { + if fn == nil { + m.listS3Prefix = defaultS3PrefixLister + return + } + m.listS3Prefix = fn +} + +func (m *ObjectManager) ListProjectStorageObjects(ctx context.Context, organization, project string, includeHead bool) ([]StorageBucketObject, error) { + result, err := m.InspectProjectStorage(ctx, organization, project, ProjectStorageInspectOptions{Mode: ProjectStorageInspectItems, IncludeHead: includeHead}) + if err != nil { + return nil, err + } + return result.Items, nil +} + +func (m *ObjectManager) InspectProjectStorage(ctx context.Context, organization, project string, inspectOptions ProjectStorageInspectOptions) (*ProjectStorageInspectResult, error) { + started := time.Now() + ctx = withS3ProbeLimiter(ctx, m.s3ProbeLimiter) + ctx = WithStorageInspectCache(ctx) + target, err := m.resolveProjectStorageScopeTarget(ctx, organization, project) + if err != nil { + log.Printf("INFO: syfon_project_storage_inspect_done organization=%s project=%s mode=%s path_prefix=%q duration_ms=%d error=%q", organization, project, inspectOptions.Mode, inspectOptions.PathPrefix, time.Since(started).Milliseconds(), err.Error()) + return nil, err + } + target = target.withPathPrefix(inspectOptions.PathPrefix) + normalizedMode := normalizeProjectStorageInspectMode(inspectOptions.Mode) + options := StoragePrefixListOptions{IncludeHead: inspectOptions.IncludeHead} + if normalizedMode == ProjectStorageInspectExists { + options.MaxKeys = 1 + } + items, err := m.listS3Prefix(ctx, target.cred, target.bucket, target.prefix, options) + if err != nil { + log.Printf("INFO: syfon_project_storage_inspect_done organization=%s project=%s mode=%s path_prefix=%q bucket=%s prefix=%q max_keys=%d include_head=%t duration_ms=%d error=%q", organization, project, normalizedMode, inspectOptions.PathPrefix, target.bucket, target.prefix, options.MaxKeys, options.IncludeHead, time.Since(started).Milliseconds(), err.Error()) + return nil, err + } + out := normalizeStorageBucketObjects(items, target) + summary := summarizeProjectStorageObjects(out, target, normalizedMode) + if normalizedMode != ProjectStorageInspectItems { + out = []StorageBucketObject{} + } + log.Printf("INFO: syfon_project_storage_inspect_done organization=%s project=%s mode=%s path_prefix=%q bucket=%s prefix=%q max_keys=%d include_head=%t exists=%t object_count=%d returned_items=%d total_bytes=%d duration_ms=%d", organization, project, normalizedMode, inspectOptions.PathPrefix, target.bucket, target.prefix, options.MaxKeys, options.IncludeHead, summary.Exists, summary.ObjectCount, len(out), summary.TotalBytes, time.Since(started).Milliseconds()) + return &ProjectStorageInspectResult{Summary: summary, Items: out}, nil +} + +func (m *ObjectManager) ResolveProjectStoragePathPrefix(ctx context.Context, organization, project, pathPrefix string) (string, error) { + target, err := m.resolveProjectStorageScopeTarget(ctx, organization, project) + if err != nil { + return "", err + } + target = target.withPathPrefix(pathPrefix) + if target == nil { + return "", nil + } + return strings.Trim(strings.TrimSpace(target.prefix), "/"), nil +} + +func (target *resolvedStorageScopeTarget) withPathPrefix(pathPrefix string) *resolvedStorageScopeTarget { + trimmed := strings.Trim(strings.TrimSpace(pathPrefix), "/") + if target == nil || trimmed == "" { + return target + } + copyTarget := *target + if copyTarget.prefix == "" { + copyTarget.prefix = trimmed + } else { + copyTarget.prefix = strings.Trim(strings.TrimSpace(copyTarget.prefix), "/") + "/" + trimmed + } + return ©Target +} + +func withStoragePrefixListLogging(ctx context.Context, mode string) context.Context { + return context.WithValue(ctx, contextKey("storagePrefixListLogging"), strings.TrimSpace(mode)) +} + +func storagePrefixListLoggingEnabled(ctx context.Context) bool { + mode, _ := ctx.Value(contextKey("storagePrefixListLogging")).(string) + return strings.TrimSpace(mode) != storagePrefixListLoggingDisabled +} + +func normalizeProjectStorageInspectMode(mode ProjectStorageInspectMode) ProjectStorageInspectMode { + switch mode { + case ProjectStorageInspectExists, ProjectStorageInspectSummary: + return mode + default: + return ProjectStorageInspectItems + } +} + +func normalizeStorageBucketObjects(items []StorageBucketObject, target *resolvedStorageScopeTarget) []StorageBucketObject { + out := make([]StorageBucketObject, 0, len(items)) + for _, item := range items { + entry := item + entry.Provider = common.S3Provider + entry.Bucket = target.bucket + entry.Key = strings.Trim(strings.TrimSpace(entry.Key), "/") + entry.ObjectURL = common.BucketToURL(target.bucket, entry.Key) + if strings.TrimSpace(entry.Path) == "" { + entry.Path = path.Base(entry.Key) + } + out = append(out, entry) + } + sort.Slice(out, func(i, j int) bool { + return out[i].Key < out[j].Key + }) + return out +} + +func summarizeProjectStorageObjects(items []StorageBucketObject, target *resolvedStorageScopeTarget, mode ProjectStorageInspectMode) ProjectStorageSummary { + summary := ProjectStorageSummary{ + Provider: target.provider, + Bucket: target.bucket, + Prefix: strings.Trim(strings.TrimSpace(target.prefix), "/"), + ObjectURL: common.BucketToURL(target.bucket, strings.Trim(strings.TrimSpace(target.prefix), "/")), + Exists: len(items) > 0, + ObjectCount: len(items), + ComputedAt: time.Now().UTC(), + Mode: mode, + } + for _, item := range items { + summary.TotalBytes += item.SizeBytes + } + return summary +} + +func (m *ObjectManager) resolveProjectStorageScopeTarget(ctx context.Context, organization, project string) (*resolvedStorageScopeTarget, error) { + return m.resolveProjectStorageScopeTargetForMethod(ctx, organization, project, objectMethodRead) +} + +func (m *ObjectManager) resolveProjectStorageScopeTargetForMethod(ctx context.Context, organization, project, method string) (*resolvedStorageScopeTarget, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" { + return nil, &StorageInspectError{Kind: StorageInspectInvalidInput, Message: "organization is required"} + } + resource, err := syfoncommon.ResourcePath(organization, project) + if err != nil { + return nil, &StorageInspectError{Kind: StorageInspectInvalidInput, Message: err.Error()} + } + if authz.IsAuthzEnforced(ctx) && !authz.HasMethodAccess(ctx, method, []string{resource}) { + return nil, &common.AuthorizationError{Method: method, Resources: []string{resource}} + } + + scopes := make([]models.BucketScope, 0, 2) + if scope, found, err := m.lookupBucketScope(ctx, organization, ""); err != nil { + return nil, err + } else if found { + scopes = append(scopes, scope) + } + if project != "" { + if scope, found, err := m.lookupBucketScope(ctx, organization, project); err != nil { + return nil, err + } else if found { + scopes = append(scopes, scope) + } + } + if len(scopes) == 0 { + if project != "" { + return nil, &StorageInspectError{Kind: StorageInspectScopeNotFound, Message: fmt.Sprintf("no bucket scope configured for organization %q project %q", organization, project)} + } + return nil, &StorageInspectError{Kind: StorageInspectScopeNotFound, Message: fmt.Sprintf("no bucket scope configured for organization %q", organization)} + } + + bucket := "" + for _, scope := range scopes { + if strings.TrimSpace(scope.Bucket) != "" { + bucket = strings.TrimSpace(scope.Bucket) + } + } + if bucket == "" { + return nil, &StorageInspectError{Kind: StorageInspectInvalidInput, Message: fmt.Sprintf("unable to resolve scoped storage bucket for organization %q project %q", organization, project)} + } + cred, err := m.credentialForBucket(ctx, bucket) + if err != nil { + return nil, err + } + if common.NormalizeProvider(cred.Provider, common.S3Provider) != common.S3Provider { + return nil, &StorageInspectError{Kind: StorageInspectUnsupported, Message: fmt.Sprintf("provider %q is not supported for scoped bucket listing", cred.Provider)} + } + return &resolvedStorageScopeTarget{ + provider: common.S3Provider, + bucket: bucket, + prefix: strings.Join(normalizedScopePrefixes(scopes), "/"), + cred: *cred, + }, nil +} + +func (m *ObjectManager) DeleteProjectStorageObjects(ctx context.Context, organization, project string, objectURLs []string) []ProjectStorageDeleteResult { + ctx = WithStorageInspectCache(ctx) + normalized := uniqueStorageObjectURLs(objectURLs) + if len(normalized) == 0 { + return []ProjectStorageDeleteResult{} + } + results := make([]ProjectStorageDeleteResult, 0, len(normalized)) + target, err := m.resolveProjectStorageScopeTargetForMethod(ctx, organization, project, objectMethodDelete) + if err != nil { + for _, objectURL := range normalized { + results = append(results, ProjectStorageDeleteResult{ + ObjectURL: objectURL, + Status: "error", + Error: err.Error(), + }) + } + return results + } + for _, objectURL := range normalized { + result := ProjectStorageDeleteResult{ + ObjectURL: objectURL, + Status: "deleted", + } + storageTarget, ok, targetErr := m.storageTargetFromURL(ctx, objectURL) + switch { + case targetErr != nil: + result.Status = "error" + result.Error = targetErr.Error() + case !ok: + result.Status = "invalid" + result.Error = "object_url must resolve to a deletable storage target" + case !projectStorageTargetAllowed(storageTarget, target): + result.Status = "forbidden" + result.Error = fmt.Sprintf("object_url %q is outside configured project bucket scope", objectURL) + default: + err := m.deleteStorageTarget(ctx, storageTarget) + if err == nil { + break + } + result.Status = "error" + result.Error = err.Error() + } + results = append(results, result) + } + return results +} + +func projectStorageTargetAllowed(candidate storageTarget, target *resolvedStorageScopeTarget) bool { + if target == nil { + return false + } + if common.NormalizeProvider(candidate.provider, common.S3Provider) != common.S3Provider { + return false + } + if strings.TrimSpace(candidate.bucket) != strings.TrimSpace(target.bucket) { + return false + } + return projectStorageKeyWithinPrefix(candidate.key, target.prefix) +} + +func projectStorageKeyWithinPrefix(key, prefix string) bool { + cleanKey := strings.Trim(strings.TrimSpace(key), "/") + cleanPrefix := strings.Trim(strings.TrimSpace(prefix), "/") + if cleanPrefix == "" { + return cleanKey != "" + } + return cleanKey == cleanPrefix || strings.HasPrefix(cleanKey, cleanPrefix+"/") +} + +func uniqueStorageObjectURLs(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func (m *ObjectManager) ListValidateStorageObjects(ctx context.Context, items []StorageListValidationRequest) []StorageListValidationResult { + started := time.Now() + ctx = withS3ProbeLimiter(ctx, m.s3ProbeLimiter) + ctx = WithStorageInspectCache(ctx) + ctx = withStoragePrefixListLogging(ctx, storagePrefixListLoggingDisabled) + if len(items) == 0 { + log.Printf("INFO: syfon_bulk_list_validate_done items=0 duration_ms=0") + return []StorageListValidationResult{} + } + visible, visibleErr := m.ListVisibleBuckets(ctx) + results := make([]StorageListValidationResult, len(items)) + workByTarget := make(map[string]*storageListTargetWork) + buckets := map[string]struct{}{} + stats := storageListRunStats{inputItemCount: len(items)} + for index, item := range items { + result, targetWork, ok := m.resolveListValidationTarget(ctx, item, index, visible, visibleErr) + if !ok { + results[index] = result + continue + } + targetKey := storageListTargetKey(targetWork.bucket, targetWork.key) + if existing, found := workByTarget[targetKey]; found { + existing.requestIndexes = append(existing.requestIndexes, index) + stats.duplicateCount++ + continue + } + workByTarget[targetKey] = targetWork + buckets[targetWork.bucket] = struct{}{} + } + stats.distinctTargetCount = len(workByTarget) + stats.bucketCount = len(buckets) + + prefixGroups := groupStorageListTargetsByDirectory(workByTarget) + coalescedPrefixGroups := make([][]*storageListTargetWork, 0, len(prefixGroups)) + for _, group := range prefixGroups { + if len(group) < storageListCoalesceThreshold { + continue + } + stats.coalescedPrefixCount++ + coalescedPrefixGroups = append(coalescedPrefixGroups, group) + } + log.Printf( + "INFO: syfon_bulk_list_validate_start items=%d distinct_targets=%d duplicates=%d buckets=%d coalesced_prefixes=%d", + stats.inputItemCount, + stats.distinctTargetCount, + stats.duplicateCount, + stats.bucketCount, + stats.coalescedPrefixCount, + ) + + outcomes := make(map[string]StorageListValidationResult, len(workByTarget)) + matchedItems := make(map[string]StorageBucketObject, len(workByTarget)) + unresolved := cloneStorageListWorkMap(workByTarget) + + for _, group := range coalescedPrefixGroups { + m.runCoalescedStorageListGroup(ctx, group, outcomes, matchedItems, unresolved, &stats) + } + m.runExactStorageListTargets(ctx, unresolved, outcomes, matchedItems, &stats) + + for targetKey, work := range workByTarget { + outcome, ok := outcomes[targetKey] + if !ok { + outcome = work.baseResult + outcome.Status = StorageProbeStatusNotFound + outcome.ErrorKind = string(StorageInspectObjectNotFound) + outcome.Error = fmt.Sprintf("object %q was not found", work.baseResult.ObjectURL) + } + for _, index := range work.requestIndexes { + req := items[index] + if item, found := matchedItems[targetKey]; found { + result := storageListValidationPresentResult(req, work.baseResult, item) + result.ID = strings.TrimSpace(req.ID) + result.ObjectURL = strings.TrimSpace(req.ObjectURL) + results[index] = result + continue + } + result := outcome + result.ID = strings.TrimSpace(req.ID) + result.ObjectURL = strings.TrimSpace(req.ObjectURL) + result.ValidationStatus = storageListValidationStatusForError(req) + results[index] = result + } + } + + statusCounts := map[StorageProbeStatus]int{} + validationCounts := map[StorageValidationStatus]int{} + for _, result := range results { + statusCounts[result.Status]++ + validationCounts[result.ValidationStatus]++ + } + log.Printf( + "INFO: syfon_bulk_list_validate_done items=%d distinct_targets=%d duplicates=%d buckets=%d coalesced_prefixes=%d exact_list_calls=%d prefix_list_pages=%d fallbacks=%d visible_error=%t present=%d not_found=%d invalid=%d error=%d matched=%d mismatched=%d unverifiable=%d duration_ms=%d", + stats.inputItemCount, + stats.distinctTargetCount, + stats.duplicateCount, + stats.bucketCount, + stats.coalescedPrefixCount, + stats.exactListCallCount, + stats.prefixListPageCount, + stats.fallbackCount, + visibleErr != nil, + statusCounts[StorageProbeStatusPresent], + statusCounts[StorageProbeStatusNotFound], + statusCounts[StorageProbeStatusInvalid], + statusCounts[StorageProbeStatusError], + validationCounts[StorageValidationMatched], + validationCounts[StorageValidationMismatched], + validationCounts[StorageValidationUnverifiable], + time.Since(started).Milliseconds(), + ) + return results +} + +func (m *ObjectManager) resolveListValidationTarget(ctx context.Context, req StorageListValidationRequest, index int, visible map[string]VisibleBucket, visibleErr error) (StorageListValidationResult, *storageListTargetWork, bool) { + result := StorageListValidationResult{ + ID: strings.TrimSpace(req.ID), + ObjectURL: strings.TrimSpace(req.ObjectURL), + Status: StorageProbeStatusError, + ValidationStatus: StorageValidationNotRequested, + } + target, ok, err := m.storageTargetFromURL(ctx, req.ObjectURL) + if err != nil { + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = strings.TrimSpace(err.Error()) + result.ValidationStatus = storageListValidationStatusForError(req) + return result, nil, false + } + if !ok || target.provider != common.S3Provider { + result.Status = StorageProbeStatusInvalid + result.ErrorKind = string(StorageInspectInvalidInput) + result.Error = "object_url must be a valid s3://bucket/key URL" + result.ValidationStatus = storageListValidationStatusForError(req) + return result, nil, false + } + result.Provider = common.S3Provider + result.Bucket = target.bucket + result.Key = target.key + result.Path = path.Base(target.key) + + cred, err := m.credentialForBucket(ctx, target.bucket) + if err != nil { + log.Printf("INFO: syfon_bulk_list_validate_credential id=%s bucket=%s key=%q error=%q", result.ID, target.bucket, target.key, err.Error()) + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = strings.TrimSpace(err.Error()) + result.ValidationStatus = storageListValidationStatusForError(req) + return result, nil, false + } + if visibleErr != nil { + log.Printf("INFO: syfon_bulk_list_validate_visible id=%s bucket=%s key=%q error=%q", result.ID, target.bucket, target.key, visibleErr.Error()) + result.Status, result.ErrorKind = classifyStorageProbeError(visibleErr) + result.Error = strings.TrimSpace(visibleErr.Error()) + result.ValidationStatus = storageListValidationStatusForError(req) + return result, nil, false + } + if !bucketVisibleToCaller(visible, target.bucket, credentialIDForCredential(*cred)) { + err := &StorageInspectError{Kind: StorageInspectPermissionDenied, Message: fmt.Sprintf("bucket %q is not visible to the caller", target.bucket)} + log.Printf("INFO: syfon_bulk_list_validate_visible id=%s bucket=%s key=%q visible_count=%d error=%q", result.ID, target.bucket, target.key, len(visible), err.Error()) + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = err.Error() + result.ValidationStatus = storageListValidationStatusForError(req) + return result, nil, false + } + return result, &storageListTargetWork{ + bucket: target.bucket, + key: target.key, + cred: *cred, + baseResult: result, + requestIndexes: []int{index}, + }, true +} + +func groupStorageListTargetsByDirectory(workByTarget map[string]*storageListTargetWork) map[string][]*storageListTargetWork { + groups := make(map[string][]*storageListTargetWork) + for _, work := range workByTarget { + dirPrefix := storageListDirectoryPrefix(work.key) + groupKey := strings.TrimSpace(work.bucket) + "\x00" + dirPrefix + groups[groupKey] = append(groups[groupKey], work) + } + return groups +} + +func cloneStorageListWorkMap(input map[string]*storageListTargetWork) map[string]*storageListTargetWork { + out := make(map[string]*storageListTargetWork, len(input)) + for key, work := range input { + out[key] = work + } + return out +} + +func storageListTargetKey(bucket string, key string) string { + return strings.TrimSpace(bucket) + "\x00" + strings.Trim(strings.TrimSpace(key), "/") +} + +func storageListDirectoryPrefix(key string) string { + trimmed := strings.Trim(strings.TrimSpace(key), "/") + if trimmed == "" { + return "" + } + dir := strings.Trim(strings.TrimSpace(path.Dir(trimmed)), "/") + if dir == "." || dir == "" { + return "" + } + return dir + "/" +} + +func (m *ObjectManager) runCoalescedStorageListGroup(ctx context.Context, group []*storageListTargetWork, outcomes map[string]StorageListValidationResult, matchedItems map[string]StorageBucketObject, unresolved map[string]*storageListTargetWork, stats *storageListRunStats) { + if len(group) == 0 { + return + } + requestedKeys := make(map[string]*storageListTargetWork, len(group)) + for _, work := range group { + requestedKeys[storageListTargetKey(work.bucket, work.key)] = work + } + dirPrefix := storageListDirectoryPrefix(group[0].key) + started := time.Now() + listed, err := m.listS3Prefix(ctx, group[0].cred, group[0].bucket, dirPrefix, StoragePrefixListOptions{ExactPrefix: true}) + if err != nil { + log.Printf("INFO: syfon_bulk_list_validate_prefix bucket=%s prefix=%q targets=%d duration_ms=%d error=%q", group[0].bucket, dirPrefix, len(group), time.Since(started).Milliseconds(), err.Error()) + stats.fallbackCount += len(group) + return + } + pageEstimate := 1 + if len(listed) > 1000 { + pageEstimate = (len(listed) + 999) / 1000 + } + stats.prefixListPageCount += pageEstimate + found := 0 + for _, item := range listed { + key := storageListTargetKey(group[0].bucket, item.Key) + work, ok := requestedKeys[key] + if !ok { + continue + } + normalized := normalizeStorageBucketObjects([]StorageBucketObject{item}, &resolvedStorageScopeTarget{provider: common.S3Provider, bucket: work.bucket, prefix: work.key}) + if len(normalized) == 0 { + continue + } + matchedItems[key] = normalized[0] + result := work.baseResult + result.Exists = true + result.Status = StorageProbeStatusPresent + result.Error = "" + result.ErrorKind = "" + outcomes[key] = result + delete(unresolved, key) + found++ + } + if len(listed) > storageListFallbackObjectLimit && found < len(group) { + stats.fallbackCount += len(group) - found + } + if duration := time.Since(started); duration > storageListSlowPrefixThreshold { + log.Printf("INFO: syfon_bulk_list_validate_prefix bucket=%s prefix=%q targets=%d listed_objects=%d matched_targets=%d duration_ms=%d", group[0].bucket, dirPrefix, len(group), len(listed), found, duration.Milliseconds()) + } +} + +func (m *ObjectManager) runExactStorageListTargets(ctx context.Context, unresolved map[string]*storageListTargetWork, outcomes map[string]StorageListValidationResult, matchedItems map[string]StorageBucketObject, stats *storageListRunStats) { + workers := len(unresolved) + if workers > maxStorageInspectWorkers { + workers = maxStorageInspectWorkers + } + if workers == 0 { + return + } + keys := make([]string, 0, len(unresolved)) + for key := range unresolved { + keys = append(keys, key) + } + sort.Strings(keys) + workCh := make(chan string) + var wg sync.WaitGroup + var mu sync.Mutex + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for targetKey := range workCh { + work := unresolved[targetKey] + outcome, item, matched := m.runExactStorageListTarget(ctx, work, stats) + mu.Lock() + outcomes[targetKey] = outcome + if matched { + matchedItems[targetKey] = item + } + mu.Unlock() + } + }() + } + for _, key := range keys { + workCh <- key + } + close(workCh) + wg.Wait() +} + +func (m *ObjectManager) runExactStorageListTarget(ctx context.Context, work *storageListTargetWork, stats *storageListRunStats) (StorageListValidationResult, StorageBucketObject, bool) { + started := time.Now() + matches, err := m.listS3Prefix(ctx, work.cred, work.bucket, work.key, StoragePrefixListOptions{ExactPrefix: true, MaxKeys: 1}) + if err != nil { + log.Printf("INFO: syfon_bulk_list_validate_exact request_id=%s bucket=%s key=%q duration_ms=%d error=%q", common.GetRequestID(ctx), work.bucket, work.key, time.Since(started).Milliseconds(), err.Error()) + result := work.baseResult + result.Status, result.ErrorKind = classifyStorageProbeError(err) + result.Error = strings.TrimSpace(err.Error()) + stats.incrementExactListCalls() + return result, StorageBucketObject{}, false + } + stats.incrementExactListCalls() + for _, item := range matches { + key := strings.Trim(strings.TrimSpace(item.Key), "/") + if key != work.key { + continue + } + normalized := normalizeStorageBucketObjects([]StorageBucketObject{item}, &resolvedStorageScopeTarget{provider: common.S3Provider, bucket: work.bucket, prefix: work.key}) + if len(normalized) == 0 { + break + } + if duration := time.Since(started); duration > storageListSlowExactThreshold { + log.Printf("INFO: syfon_bulk_list_validate_exact request_id=%s bucket=%s key=%q duration_ms=%d status=present", common.GetRequestID(ctx), work.bucket, work.key, duration.Milliseconds()) + } + result := work.baseResult + result.Exists = true + result.Status = StorageProbeStatusPresent + result.Error = "" + result.ErrorKind = "" + return result, normalized[0], true + } + if duration := time.Since(started); duration > storageListSlowExactThreshold { + log.Printf("INFO: syfon_bulk_list_validate_exact request_id=%s bucket=%s key=%q duration_ms=%d status=not_found", common.GetRequestID(ctx), work.bucket, work.key, duration.Milliseconds()) + } + result := work.baseResult + result.Status = StorageProbeStatusNotFound + result.ErrorKind = string(StorageInspectObjectNotFound) + result.Error = fmt.Sprintf("object %q was not found", work.baseResult.ObjectURL) + return result, StorageBucketObject{}, false +} + +func storageListValidationPresentResult(req StorageListValidationRequest, result StorageListValidationResult, item StorageBucketObject) StorageListValidationResult { + result.ObjectURL = item.ObjectURL + result.Key = item.Key + result.Path = item.Path + result.Exists = true + result.Status = StorageProbeStatusPresent + result.Error = "" + result.ErrorKind = "" + result.SizeBytes = ptrInt64(item.SizeBytes) + result.ETag = strings.TrimSpace(item.ETag) + result.LastModTime = item.LastModTime + result.ValidationStatus, result.SizeMatch, result.NameMatch, result.ValidationMismatches = validateStorageListResult(req, item) + return result +} + +func storageListValidationStatusForError(req StorageListValidationRequest) StorageValidationStatus { + if req.ExpectedSizeBytes == nil && strings.TrimSpace(req.ExpectedName) == "" { + return StorageValidationNotRequested + } + return StorageValidationUnverifiable +} + +func validateStorageListResult(req StorageListValidationRequest, item StorageBucketObject) (StorageValidationStatus, *bool, *bool, []string) { + if req.ExpectedSizeBytes == nil && strings.TrimSpace(req.ExpectedName) == "" { + return StorageValidationNotRequested, nil, nil, nil + } + mismatches := make([]string, 0, 2) + var sizeMatch *bool + if req.ExpectedSizeBytes != nil { + matched := item.SizeBytes == *req.ExpectedSizeBytes + sizeMatch = &matched + if !matched { + mismatches = append(mismatches, "size_mismatch") + } + } + var nameMatch *bool + if expectedName := strings.TrimSpace(req.ExpectedName); expectedName != "" { + matched := path.Base(item.Key) == expectedName + nameMatch = &matched + if !matched { + mismatches = append(mismatches, "name_mismatch") + } + } + if len(mismatches) > 0 { + return StorageValidationMismatched, sizeMatch, nameMatch, mismatches + } + return StorageValidationMatched, sizeMatch, nameMatch, nil +} + +func defaultS3PrefixLister(ctx context.Context, cred models.S3Credential, bucket string, prefix string, options StoragePrefixListOptions) ([]StorageBucketObject, error) { + started := time.Now() + client, err := s3ClientFromContext(ctx, cred) + if err != nil { + return nil, err + } + input := &awss3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + MaxKeys: aws.Int32(defaultS3ListPageSize), + } + if trimmedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); trimmedPrefix != "" { + if options.ExactPrefix { + input.Prefix = aws.String(trimmedPrefix) + } else { + input.Prefix = aws.String(trimmedPrefix + "/") + } + } + if options.MaxKeys > 0 { + input.MaxKeys = aws.Int32(options.MaxKeys) + } + requestPrefix := aws.ToString(input.Prefix) + logEnabled := storagePrefixListLoggingEnabled(ctx) + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_start request_id=%s bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t", common.GetRequestID(ctx), bucket, prefix, requestPrefix, options.ExactPrefix, aws.ToInt32(input.MaxKeys), options.IncludeHead) + } + out, stats, firstKeys, err := listS3PrefixPagesWithExactProbeRetry(ctx, client, input, bucket, prefix, requestPrefix, options, logEnabled) + if err != nil { + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_done request_id=%s bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d retries=%d terminal_replay_attempts=%d terminal_disagreements=%d failed_page=%d token=%s last_key=%q duration_ms=%d error=%q", common.GetRequestID(ctx), bucket, prefix, requestPrefix, options.ExactPrefix, aws.ToInt32(input.MaxKeys), options.IncludeHead, stats.Pages, len(out), stats.Retries, stats.TerminalReplayAttempts, stats.TerminalDisagreements, stats.FailedPage, stats.LastTokenID, stats.LastKey, time.Since(started).Milliseconds(), err.Error()) + } + var inspectErr *StorageInspectError + if errors.As(err, &inspectErr) { + return nil, inspectErr + } + return nil, classifyS3ListError(bucket, prefix, err) + } + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_done request_id=%s bucket=%s requested_prefix=%q input_prefix=%q exact_prefix=%t max_keys=%d include_head=%t pages=%d objects=%d retries=%d terminal_replay_attempts=%d terminal_disagreements=%d first_keys=%q duration_ms=%d", common.GetRequestID(ctx), bucket, prefix, requestPrefix, options.ExactPrefix, aws.ToInt32(input.MaxKeys), options.IncludeHead, stats.Pages, len(out), stats.Retries, stats.TerminalReplayAttempts, stats.TerminalDisagreements, strings.Join(firstKeys, ","), time.Since(started).Milliseconds()) + } + if !options.IncludeHead || len(out) == 0 { + return out, nil + } + + workers := len(out) + if workers > maxStorageInspectWorkers { + workers = maxStorageInspectWorkers + } + if workers == 0 { + return out, nil + } + indexCh := make(chan int) + var wg sync.WaitGroup + var firstErr error + var errMu sync.Mutex + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for index := range indexCh { + meta, err := defaultS3ObjectInspector(ctx, cred, bucket, out[index].Key) + if err != nil { + errMu.Lock() + if firstErr == nil { + firstErr = err + } + errMu.Unlock() + continue + } + out[index].MetaSHA256 = strings.TrimSpace(meta.MetaSHA256) + if out[index].ETag == "" { + out[index].ETag = strings.TrimSpace(meta.ETag) + } + if out[index].LastModTime.IsZero() { + out[index].LastModTime = meta.LastModTime + } + } + }() + } + for index := range out { + indexCh <- index + } + close(indexCh) + wg.Wait() + if firstErr != nil { + return nil, firstErr + } + return out, nil +} + +// Full-prefix inventories use one token-only scan per request. Exact probes +// retain their cheaper bounded retry behavior because a transient omission is +// more common than a real deletion for a single known key. +func listS3PrefixPagesWithExactProbeRetry(ctx context.Context, client s3ListObjectsV2Client, input *awss3.ListObjectsV2Input, bucket, prefix, requestPrefix string, options StoragePrefixListOptions, logEnabled bool) ([]StorageBucketObject, s3PrefixListStats, []string, error) { + if options.MaxKeys != 1 { + return listS3PrefixPages(ctx, client, input, bucket, prefix, requestPrefix, options, logEnabled) + } + if !isExactProbeList(options) { + return listS3PrefixPages(ctx, client, input, bucket, prefix, requestPrefix, options, logEnabled) + } + policy := s3ListPageRetryPolicyFromEnv() + maxAttempts := intEnvOrDefault(envS3ExactProbeMaxAttempts, defaultS3ExactProbeMaxAttempts, 1) + for attempt := 1; attempt <= maxAttempts; attempt++ { + out, stats, firstKeys, err := listS3PrefixPages(ctx, client, input, bucket, prefix, requestPrefix, options, logEnabled) + if err != nil || hasExactListedKey(out, prefix) || attempt == maxAttempts { + return out, stats, firstKeys, err + } + backoff := policy.backoff(attempt) + if logEnabled { + log.Printf("INFO: syfon_s3_exact_probe_retry request_id=%s bucket=%s key=%q attempt=%d max_attempts=%d backoff_ms=%d objects=%d", common.GetRequestID(ctx), bucket, prefix, attempt+1, maxAttempts, backoff.Milliseconds(), len(out)) + } + if err := sleepS3ListPageRetry(ctx, backoff); err != nil { + return nil, stats, firstKeys, err + } + } + return nil, s3PrefixListStats{}, nil, nil +} + +func isExactProbeList(options StoragePrefixListOptions) bool { + return options.ExactPrefix && options.MaxKeys == 1 +} + +func hasExactListedKey(items []StorageBucketObject, key string) bool { + want := strings.Trim(strings.TrimSpace(key), "/") + for _, item := range items { + if strings.Trim(strings.TrimSpace(item.Key), "/") == want { + return true + } + } + return false +} + +func listS3PrefixPages(ctx context.Context, client s3ListObjectsV2Client, input *awss3.ListObjectsV2Input, bucket string, prefix string, requestPrefix string, options StoragePrefixListOptions, logEnabled bool) ([]StorageBucketObject, s3PrefixListStats, []string, error) { + policy := s3ListPageRetryPolicyFromEnv() + out := make([]StorageBucketObject, 0) + stats := s3PrefixListStats{} + firstKeys := make([]string, 0, 5) + seenKeys := make(map[string]struct{}) + continuationToken := "" + baseInput := cloneListObjectsV2Input(input, "") + baseInput.StartAfter = nil + for { + pageNumber := stats.Pages + 1 + page, tokenID, retries, err := listS3PrefixPageWithRetry(ctx, client, baseInput, continuationToken, bucket, prefix, requestPrefix, policy, pageNumber, len(out), stats.LastKey, logEnabled) + stats.LastTokenID = tokenID + stats.Retries += retries + if err != nil { + stats.FailedPage = pageNumber + return out, stats, firstKeys, err + } + stats.Pages++ + added := appendS3ListPageObjects(&out, page, bucket, &firstKeys, seenKeys) + if len(out) > 0 { + stats.LastKey = out[len(out)-1].Key + } + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_page_done request_id=%s bucket=%s requested_prefix=%q input_prefix=%q page=%d token=%s objects_added=%d objects_total=%d last_key=%q truncated=%t", common.GetRequestID(ctx), bucket, prefix, requestPrefix, pageNumber, tokenID, added, len(out), stats.LastKey, aws.ToBool(page.IsTruncated)) + } + if !aws.ToBool(page.IsTruncated) { + if options.MaxKeys != 1 { + replays, replayRetries, replayErr := replayS3TerminalPage(ctx, client, baseInput, continuationToken, page, bucket, prefix, requestPrefix, policy, pageNumber, len(out), stats.LastKey, logEnabled) + stats.TerminalReplayAttempts += replays + stats.Retries += replayRetries + if replayErr != nil { + stats.TerminalDisagreements++ + stats.FailedPage = pageNumber + return out, stats, firstKeys, replayErr + } + } + return out, stats, firstKeys, nil + } + continuationToken = strings.TrimSpace(aws.ToString(page.NextContinuationToken)) + if continuationToken == "" { + err := fmt.Errorf("list s3 objects for %s/%s stopped at page %d after %d objects: provider returned truncated page without next continuation token", bucket, strings.Trim(strings.TrimSpace(prefix), "/"), pageNumber, len(out)) + stats.FailedPage = pageNumber + return out, stats, firstKeys, err + } + baseInput.StartAfter = nil + } +} + +func replayS3TerminalPage(ctx context.Context, client s3ListObjectsV2Client, baseInput *awss3.ListObjectsV2Input, continuationToken string, firstPage *awss3.ListObjectsV2Output, bucket, prefix, requestPrefix string, policy s3ListPageRetryPolicy, pageNumber, objectCount int, lastKey string, logEnabled bool) (int, int, error) { + maxAttempts := intEnvOrDefault(envS3InventoryTerminalReplays, defaultS3InventoryTerminalReplays, 2) + firstFingerprint := s3ListPageFingerprint(firstPage) + replays := 0 + retries := 0 + for attempt := 2; attempt <= maxAttempts; attempt++ { + page, tokenID, pageRetries, err := listS3PrefixPageWithRetry(ctx, client, baseInput, continuationToken, bucket, prefix, requestPrefix, policy, pageNumber, objectCount, lastKey, logEnabled) + replays++ + if err != nil { + return replays, retries + pageRetries, incompleteS3ListingError(bucket, prefix, lastKey, "terminal replay failed", err) + } + retries += pageRetries + if logEnabled { + log.Printf("INFO: syfon_s3_inventory_terminal_replay request_id=%s bucket=%s requested_prefix=%q page=%d token=%s replay=%d max_replays=%d objects=%d truncated=%t", common.GetRequestID(ctx), bucket, prefix, pageNumber, tokenID, attempt-1, maxAttempts-1, len(page.Contents), aws.ToBool(page.IsTruncated)) + } + if s3ListPageFingerprint(page) != firstFingerprint { + return replays, retries, incompleteS3ListingError(bucket, prefix, lastKey, "terminal replay returned different page content", nil) + } + } + return replays, retries, nil +} + +func s3ListPageFingerprint(page *awss3.ListObjectsV2Output) string { + if page == nil { + return "nil" + } + objects := append([]types.Object(nil), page.Contents...) + sort.Slice(objects, func(i, j int) bool { + return aws.ToString(objects[i].Key) < aws.ToString(objects[j].Key) + }) + var builder strings.Builder + fmt.Fprintf(&builder, "truncated=%t\n", aws.ToBool(page.IsTruncated)) + for _, object := range objects { + lastModified := "" + if object.LastModified != nil { + lastModified = object.LastModified.UTC().Format(time.RFC3339Nano) + } + fmt.Fprintf(&builder, "%s\x00%d\x00%s\x00%s\n", aws.ToString(object.Key), aws.ToInt64(object.Size), aws.ToString(object.ETag), lastModified) + } + sum := sha256.Sum256([]byte(builder.String())) + return hex.EncodeToString(sum[:])[:16] +} + +func incompleteS3ListingError(bucket, prefix, lastKey, reason string, cause error) error { + message := fmt.Sprintf("provider returned an incomplete listing for s3://%s/%s after key %q: %s", bucket, strings.Trim(strings.TrimSpace(prefix), "/"), lastKey, reason) + if cause != nil { + message += ": " + cause.Error() + } + return &StorageInspectError{Kind: StorageInspectListingIncomplete, Message: message} +} + +func listS3PrefixPageWithRetry(ctx context.Context, client s3ListObjectsV2Client, baseInput *awss3.ListObjectsV2Input, continuationToken string, bucket string, prefix string, requestPrefix string, policy s3ListPageRetryPolicy, pageNumber int, objectCount int, lastKey string, logEnabled bool) (*awss3.ListObjectsV2Output, string, int, error) { + tokenID := s3ContinuationTokenFingerprint(continuationToken) + retries := 0 + for attempt := 1; ; attempt++ { + release, acquireErr := acquireS3Probe(ctx, "list", bucket, prefix) + if acquireErr != nil { + return nil, tokenID, retries, acquireErr + } + page, err := client.ListObjectsV2(ctx, cloneListObjectsV2Input(baseInput, continuationToken)) + release() + if err == nil { + if page == nil { + err = errors.New("provider returned an empty list page") + } else if aws.ToBool(page.IsTruncated) && strings.TrimSpace(aws.ToString(page.NextContinuationToken)) == "" { + err = errors.New("provider returned a malformed truncated list page without next continuation token") + } else if aws.ToBool(page.IsTruncated) && len(page.Contents) == 0 { + err = errors.New("provider returned an empty malformed truncated list page") + } else { + return page, tokenID, retries, nil + } + } + if !isRetryableS3ListPageError(err) || attempt >= policy.MaxAttempts { + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_page_failed request_id=%s bucket=%s requested_prefix=%q input_prefix=%q page=%d token=%s objects=%d attempt=%d max_attempts=%d last_key=%q retryable=%t error=%q", common.GetRequestID(ctx), bucket, prefix, requestPrefix, pageNumber, tokenID, objectCount, attempt, policy.MaxAttempts, lastKey, isRetryableS3ListPageError(err), err.Error()) + } + return nil, tokenID, retries, fmt.Errorf("list s3 objects for %s/%s failed at page %d after %d objects and %d attempts: %w", bucket, strings.Trim(strings.TrimSpace(prefix), "/"), pageNumber, objectCount, attempt, err) + } + retries++ + backoff := policy.backoff(attempt) + if logEnabled { + log.Printf("INFO: syfon_s3_prefix_list_page_retry request_id=%s bucket=%s requested_prefix=%q input_prefix=%q page=%d token=%s objects=%d attempt=%d max_attempts=%d backoff_ms=%d last_key=%q error=%q", common.GetRequestID(ctx), bucket, prefix, requestPrefix, pageNumber, tokenID, objectCount, attempt, policy.MaxAttempts, backoff.Milliseconds(), lastKey, err.Error()) + } + if err := sleepS3ListPageRetry(ctx, backoff); err != nil { + return nil, tokenID, retries, err + } + } +} + +func appendS3ListPageObjects(out *[]StorageBucketObject, page *awss3.ListObjectsV2Output, bucket string, firstKeys *[]string, seenKeys map[string]struct{}) int { + before := len(*out) + for _, item := range page.Contents { + key := strings.Trim(strings.TrimSpace(aws.ToString(item.Key)), "/") + if key == "" { + continue + } + if _, seen := seenKeys[key]; seen { + continue + } + seenKeys[key] = struct{}{} + if len(*firstKeys) < cap(*firstKeys) { + *firstKeys = append(*firstKeys, key) + } + var size int64 + if item.Size != nil { + size = *item.Size + } + lastMod := time.Time{} + if item.LastModified != nil { + lastMod = *item.LastModified + } + *out = append(*out, StorageBucketObject{ + Provider: common.S3Provider, + Bucket: bucket, + Key: key, + ObjectURL: common.BucketToURL(bucket, key), + Path: path.Base(key), + SizeBytes: size, + ETag: strings.Trim(strings.TrimSpace(aws.ToString(item.ETag)), "\""), + LastModTime: lastMod, + }) + } + return len(*out) - before +} + +func cloneListObjectsV2Input(input *awss3.ListObjectsV2Input, continuationToken string) *awss3.ListObjectsV2Input { + clone := *input + clone.StartAfter = nil + if strings.TrimSpace(continuationToken) == "" { + clone.ContinuationToken = nil + } else { + clone.ContinuationToken = aws.String(continuationToken) + } + return &clone +} + +func isRetryableS3ListPageError(err error) bool { + if err == nil { + return false + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch strings.ToLower(strings.TrimSpace(apiErr.ErrorCode())) { + case "internalerror", "slowdown", "serviceunavailable", "requesttimeout", "requesttimeoutexception", "toomanyrequests", "throttling", "throttlingexception", "requestlimitexceeded": + return true + default: + return false + } + } + if errors.Is(err, io.EOF) { + return true + } + var netErr net.Error + if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "eof") || strings.Contains(msg, "connection reset") || strings.Contains(msg, "timeout") || + strings.Contains(msg, "malformed truncated list page") || strings.Contains(msg, "empty list page") +} + +func s3ListPageRetryPolicyFromEnv() s3ListPageRetryPolicy { + return s3ListPageRetryPolicy{ + MaxAttempts: intEnvOrDefault(envS3ListPageMaxAttempts, defaultS3ListPageMaxAttempts, 1), + InitialBackoff: millisEnvOrDefault(envS3ListPageInitialBackoffMillis, defaultS3ListPageInitialBackoff, 0), + MaxBackoff: millisEnvOrDefault(envS3ListPageMaxBackoffMillis, defaultS3ListPageMaxBackoff, 0), + } +} + +func intEnvOrDefault(name string, fallback int, minimum int) int { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + value, err := strconv.Atoi(raw) + if err != nil || value < minimum { + return fallback + } + return value +} + +func millisEnvOrDefault(name string, fallback time.Duration, minimumMillis int) time.Duration { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + value, err := strconv.Atoi(raw) + if err != nil || value < minimumMillis { + return fallback + } + return time.Duration(value) * time.Millisecond +} + +func (policy s3ListPageRetryPolicy) backoff(attempt int) time.Duration { + backoff := policy.InitialBackoff + for i := 1; i < attempt; i++ { + backoff *= 2 + if backoff >= policy.MaxBackoff { + backoff = policy.MaxBackoff + break + } + } + if backoff <= 0 { + return 0 + } + jitterMax := backoff / 4 + if jitterMax <= 0 { + return backoff + } + return backoff + time.Duration(rand.Int63n(int64(jitterMax)+1)) +} + +func s3ContinuationTokenFingerprint(token string) string { + token = strings.TrimSpace(token) + if token == "" { + return "start" + } + sum := sha1.Sum([]byte(token)) + return hex.EncodeToString(sum[:])[:12] +} + +var sleepS3ListPageRetry = func(ctx context.Context, duration time.Duration) error { + if duration <= 0 { + return nil + } + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func classifyS3ListError(bucket string, prefix string, err error) error { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch strings.ToLower(strings.TrimSpace(apiErr.ErrorCode())) { + case "forbidden", "accessdenied", "permissiondenied": + return &StorageInspectError{ + Kind: StorageInspectBucketUnavailable, + Message: fmt.Sprintf( + "provider rejected bucket inventory request for s3://%s/%s; mapped bucket target may be missing or inaccessible", + bucket, + strings.Trim(strings.TrimSpace(prefix), "/"), + ), + } + case "nosuchbucket": + return &StorageInspectError{Kind: StorageInspectBucketUnavailable, Message: fmt.Sprintf("provider could not find bucket %q", bucket)} + case "notfound": + return &StorageInspectError{Kind: StorageInspectBucketUnavailable, Message: fmt.Sprintf("provider could not resolve bucket inventory target %q", bucket)} + } + } + return fmt.Errorf("list s3 objects for %s/%s: %w", bucket, strings.Trim(strings.TrimSpace(prefix), "/"), err) +} diff --git a/internal/core/object_storage_inventory_test.go b/internal/core/object_storage_inventory_test.go new file mode 100644 index 00000000..cdbf0796 --- /dev/null +++ b/internal/core/object_storage_inventory_test.go @@ -0,0 +1,339 @@ +package core + +import ( + "context" + "errors" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" +) + +type fakeS3ListPage struct { + output *awss3.ListObjectsV2Output + err error +} + +type fakeS3ListClient struct { + pages map[string][]fakeS3ListPage + startAfterPages map[string][]fakeS3ListPage + calls []string + startAfters []string +} + +func (client *fakeS3ListClient) ListObjectsV2(_ context.Context, input *awss3.ListObjectsV2Input, _ ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) { + token := strings.TrimSpace(aws.ToString(input.ContinuationToken)) + startAfter := strings.TrimSpace(aws.ToString(input.StartAfter)) + client.calls = append(client.calls, token) + client.startAfters = append(client.startAfters, startAfter) + if startAfter != "" { + queue := client.startAfterPages[startAfter] + if len(queue) == 0 { + return finalListPage(), nil + } + next := queue[0] + client.startAfterPages[startAfter] = queue[1:] + return next.output, next.err + } + queue := client.pages[token] + if len(queue) == 0 { + return nil, errors.New("unexpected token " + token) + } + next := queue[0] + client.pages[token] = queue[1:] + return next.output, next.err +} + +func TestListS3PrefixPagesRetriesTransientPageFailure(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "4") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: listPage("token-2", "prefix/one.txt")}, + }, + "token-2": { + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {output: finalListPage("prefix/two.txt")}, + {output: finalListPage("prefix/two.txt")}, + {output: finalListPage("prefix/two.txt")}, + }, + }} + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err != nil { + t.Fatalf("list pages: %v", err) + } + if got, want := keys(items), []string{"prefix/one.txt", "prefix/two.txt"}; strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("unexpected keys: got %v want %v", got, want) + } + if got := countString(client.calls, "token-2"); got != 5 { + t.Fatalf("expected page retry and terminal replays to reuse continuation token five times, got calls %q", strings.Join(client.calls, ",")) + } + if stats.Pages != 2 { + t.Fatalf("expected 2 successful pages, got %+v", stats) + } +} + +func TestListS3PrefixPagesRecoversObservedPageSeventeenInternalError(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "4") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{}} + for page := 1; page < 17; page++ { + token := "" + if page > 1 { + token = tokenForPage(page) + } + client.pages[token] = []fakeS3ListPage{ + {output: listPage(tokenForPage(page+1), "prefix/page-"+strconv.Itoa(page)+".txt")}, + } + } + client.pages[tokenForPage(17)] = []fakeS3ListPage{ + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {output: finalListPage("prefix/page-17.txt")}, + {output: finalListPage("prefix/page-17.txt")}, + {output: finalListPage("prefix/page-17.txt")}, + } + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err != nil { + t.Fatalf("list pages: %v", err) + } + if len(items) != 17 { + t.Fatalf("expected 17 inventory objects after recovered page 17, got %d", len(items)) + } + if stats.Pages != 17 || stats.Retries != 2 { + t.Fatalf("expected recovered page 17 with 2 retries, got %+v", stats) + } + if got := countString(client.calls, tokenForPage(17)); got != 5 { + t.Fatalf("expected page 17 token to be retried and replayed five times, got calls %v", client.calls) + } +} + +func TestListS3PrefixPagesExhaustsTransientRetriesWithProgress(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "3") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: listPage("token-2", "prefix/one.txt")}, + }, + "token-2": { + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + {err: &smithy.GenericAPIError{Code: "InternalError", Message: "try again"}}, + }, + }} + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err == nil { + t.Fatal("expected retry exhaustion error") + } + if len(items) != 1 || items[0].Key != "prefix/one.txt" { + t.Fatalf("expected collected page to be preserved for error reporting, got %+v", items) + } + if stats.FailedPage != 2 { + t.Fatalf("expected failed page 2, got %+v", stats) + } + if !strings.Contains(err.Error(), "page 2") || !strings.Contains(err.Error(), "after 1 objects and 3 attempts") { + t.Fatalf("expected progress in error, got %q", err.Error()) + } +} + +func TestListS3PrefixPagesDoesNotRetryAccessDenied(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "4") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {err: &smithy.GenericAPIError{Code: "AccessDenied", Message: "no"}}, + }, + }} + + _, _, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err == nil { + t.Fatal("expected access denied error") + } + if len(client.calls) != 1 { + t.Fatalf("expected no retries for access denied, got calls %v", client.calls) + } +} + +func TestListS3PrefixPagesErrorsOnMissingContinuationToken(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "1") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: &awss3.ListObjectsV2Output{IsTruncated: aws.Bool(true), Contents: []types.Object{{Key: aws.String("prefix/one.txt")}}}}, + }, + }} + + _, _, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err == nil || !strings.Contains(err.Error(), "without next continuation token") { + t.Fatalf("expected missing token error, got %v", err) + } +} + +func TestListS3PrefixPagesRetriesMalformedTruncatedPage(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ListPageMaxAttempts, "2") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: &awss3.ListObjectsV2Output{IsTruncated: aws.Bool(true)}}, + {output: finalListPage("prefix/one.txt")}, + {output: finalListPage("prefix/one.txt")}, + {output: finalListPage("prefix/one.txt")}, + }, + }} + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err != nil { + t.Fatalf("list pages: %v", err) + } + if len(items) != 1 || stats.Retries != 1 { + t.Fatalf("expected malformed page retry, items=%v stats=%+v", items, stats) + } +} + +func TestListS3PrefixPagesRetriesExactProbeMiss(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + t.Setenv(envS3ExactProbeMaxAttempts, "2") + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": { + {output: finalListPage("prefix/other.txt")}, + {output: finalListPage("prefix/target.txt")}, + }, + }} + + items, _, _, err := listS3PrefixPagesWithExactProbeRetry(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/target.txt")}, "bucket", "prefix/target.txt", "prefix/target.txt", StoragePrefixListOptions{ExactPrefix: true, MaxKeys: 1}, false) + if err != nil { + t.Fatalf("exact probe: %v", err) + } + if len(items) != 1 || items[0].Key != "prefix/target.txt" { + t.Fatalf("expected exact probe retry to find target, got %v", items) + } +} + +func TestListS3PrefixPagesTerminalReplayUsesContinuationTokenOnly(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": {{output: listPage("token-2", "prefix/one.txt")}}, + "token-2": { + {output: finalListPage("prefix/two.txt")}, + {output: finalListPage("prefix/two.txt")}, + {output: finalListPage("prefix/two.txt")}, + }, + }} + + items, stats, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err != nil { + t.Fatalf("list pages: %v", err) + } + if got, want := strings.Join(keys(items), ","), "prefix/one.txt,prefix/two.txt"; got != want { + t.Fatalf("unexpected keys: got %q want %q", got, want) + } + if stats.TerminalReplayAttempts != 2 { + t.Fatalf("expected two terminal replays, got %+v", stats) + } + if len(client.startAfters) != 4 || containsNonEmpty(client.startAfters) { + t.Fatalf("expected no StartAfter requests, got %q", client.startAfters) + } +} + +func TestListS3PrefixPagesRejectsContradictoryTerminalReplay(t *testing.T) { + restore := disableS3ListRetrySleep(t) + defer restore() + client := &fakeS3ListClient{pages: map[string][]fakeS3ListPage{ + "": {{output: listPage("token-2", "prefix/one.txt")}}, + "token-2": { + {output: finalListPage("prefix/two.txt")}, + {output: listPage("token-3", "prefix/two.txt")}, + }, + }} + + _, _, _, err := listS3PrefixPages(context.Background(), client, &awss3.ListObjectsV2Input{Bucket: aws.String("bucket"), Prefix: aws.String("prefix/")}, "bucket", "prefix", "prefix/", StoragePrefixListOptions{}, false) + if err == nil || !strings.Contains(err.Error(), "terminal replay") { + t.Fatalf("expected contradictory terminal replay error, got %v", err) + } +} + +func listPage(nextToken string, keys ...string) *awss3.ListObjectsV2Output { + return &awss3.ListObjectsV2Output{ + IsTruncated: aws.Bool(true), + NextContinuationToken: aws.String(nextToken), + Contents: objects(keys...), + } +} + +func finalListPage(keys ...string) *awss3.ListObjectsV2Output { + return &awss3.ListObjectsV2Output{ + IsTruncated: aws.Bool(false), + Contents: objects(keys...), + } +} + +func objects(keys ...string) []types.Object { + out := make([]types.Object, 0, len(keys)) + for _, key := range keys { + out = append(out, types.Object{Key: aws.String(key), Size: aws.Int64(12)}) + } + return out +} + +func tokenForPage(page int) string { + return "token-" + strconv.Itoa(page) +} + +func keys(items []StorageBucketObject) []string { + out := make([]string, 0, len(items)) + for _, item := range items { + out = append(out, item.Key) + } + return out +} + +func countString(values []string, target string) int { + count := 0 + for _, value := range values { + if value == target { + count++ + } + } + return count +} + +func containsString(values []string, target string) bool { + return countString(values, target) > 0 +} + +func containsNonEmpty(values []string) bool { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return true + } + } + return false +} + +func disableS3ListRetrySleep(t *testing.T) func() { + t.Helper() + previous := sleepS3ListPageRetry + sleepS3ListPageRetry = func(context.Context, time.Duration) error { + return nil + } + return func() { + sleepS3ListPageRetry = previous + } +} diff --git a/internal/core/s3_probe_limiter.go b/internal/core/s3_probe_limiter.go new file mode 100644 index 00000000..625202b6 --- /dev/null +++ b/internal/core/s3_probe_limiter.go @@ -0,0 +1,71 @@ +package core + +import ( + "context" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/calypr/syfon/internal/common" +) + +const ( + defaultS3ProbeConcurrency = 8 + envS3ProbeConcurrency = "SYFON_S3_PROBE_CONCURRENCY" +) + +var s3ProbeLimiterContextKey contextKey = "s3ProbeLimiter" + +type s3ProbeLimiter struct { + permits chan struct{} +} + +func newS3ProbeLimiter(limit int) *s3ProbeLimiter { + if limit < 1 { + limit = defaultS3ProbeConcurrency + } + return &s3ProbeLimiter{permits: make(chan struct{}, limit)} +} + +func newS3ProbeLimiterFromEnv() *s3ProbeLimiter { + return newS3ProbeLimiter(s3ProbeConcurrencyFromEnv()) +} + +func s3ProbeConcurrencyFromEnv() int { + raw := strings.TrimSpace(os.Getenv(envS3ProbeConcurrency)) + if raw == "" { + return defaultS3ProbeConcurrency + } + limit, err := strconv.Atoi(raw) + if err != nil || limit < 1 { + return defaultS3ProbeConcurrency + } + return limit +} + +func withS3ProbeLimiter(ctx context.Context, limiter *s3ProbeLimiter) context.Context { + if limiter == nil || ctx.Value(s3ProbeLimiterContextKey) != nil { + return ctx + } + return context.WithValue(ctx, s3ProbeLimiterContextKey, limiter) +} + +func acquireS3Probe(ctx context.Context, operation, bucket, key string) (func(), error) { + limiter, _ := ctx.Value(s3ProbeLimiterContextKey).(*s3ProbeLimiter) + if limiter == nil { + return func() {}, nil + } + started := time.Now() + select { + case limiter.permits <- struct{}{}: + waited := time.Since(started) + if waited >= 100*time.Millisecond { + log.Printf("INFO: syfon_s3_probe_limiter_wait request_id=%s operation=%s bucket=%s key=%q wait_ms=%d", common.GetRequestID(ctx), operation, bucket, key, waited.Milliseconds()) + } + return func() { <-limiter.permits }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} diff --git a/internal/core/service.go b/internal/core/service.go index cb218b8b..b525228c 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -37,6 +37,8 @@ type ObjectManager struct { uM urlmanager.UrlManager bucketScopeCache *bucketScopeCache inspectS3Object func(context.Context, models.S3Credential, string, string) (*StorageObjectMetadata, error) + listS3Prefix func(context.Context, models.S3Credential, string, string, StoragePrefixListOptions) ([]StorageBucketObject, error) + s3ProbeLimiter *s3ProbeLimiter } type VisibleBucket struct { @@ -50,5 +52,7 @@ func NewObjectManager(db db.DatabaseInterface, uM urlmanager.UrlManager) *Object uM: uM, bucketScopeCache: newBucketScopeCache(30 * time.Second), inspectS3Object: defaultS3ObjectInspector, + listS3Prefix: defaultS3PrefixLister, + s3ProbeLimiter: newS3ProbeLimiterFromEnv(), } } diff --git a/internal/core/service_test.go b/internal/core/service_test.go index ab985d54..fb6d7a87 100644 --- a/internal/core/service_test.go +++ b/internal/core/service_test.go @@ -17,8 +17,10 @@ import ( type coreTestDB struct { *testutils.MockDatabase - aliases map[string]string - creds []models.S3Credential + aliases map[string]string + creds []models.S3Credential + getS3CredentialCalls int + listS3CredentialsCalls int } func (d *coreTestDB) ResolveObjectAlias(ctx context.Context, aliasID string) (string, error) { @@ -31,6 +33,7 @@ func (d *coreTestDB) ResolveObjectAlias(ctx context.Context, aliasID string) (st } func (d *coreTestDB) ListS3Credentials(ctx context.Context) ([]models.S3Credential, error) { + d.listS3CredentialsCalls++ if d.creds != nil { out := make([]models.S3Credential, len(d.creds)) copy(out, d.creds) @@ -42,6 +45,14 @@ func (d *coreTestDB) ListS3Credentials(ctx context.Context) ([]models.S3Credenti return d.MockDatabase.ListS3Credentials(ctx) } +func (d *coreTestDB) GetS3Credential(ctx context.Context, bucket string) (*models.S3Credential, error) { + d.getS3CredentialCalls++ + if d.MockDatabase == nil { + return nil, nil + } + return d.MockDatabase.GetS3Credential(ctx, bucket) +} + type capturingURLManager struct { signURLBucket string signURLAccessURL string @@ -953,6 +964,51 @@ func TestObjectManagerDeleteResolveAndSignDelegation(t *testing.T) { }) }) + t.Run("root bucket scope preserves existing physical storage key", func(t *testing.T) { + db := &coreTestDB{ + MockDatabase: &testutils.MockDatabase{ + BucketScopes: map[string]models.BucketScope{ + "gdc_mirror|gdc_mirror": { + Organization: "gdc_mirror", + ProjectID: "gdc_mirror", + Bucket: "gdcdata", + PathPrefix: "", + }, + }, + }, + } + om := NewObjectManager(db, &capturingURLManager{}) + obj := &models.InternalObject{ + DrsObject: drs.DrsObject{ + Id: "00664eeb-830c-5fe4-b48c-054cd9c8e02f", + Checksums: []drs.Checksum{{ + Type: "sha256", + Checksum: "239f8402efd37b62bfb892aa4becb0692b3ca5f58015083d8567e8d7fbdd1843", + }}, + ControlledAccess: &[]string{"/organization/gdc_mirror/project/gdc_mirror"}, + }, + } + sourceURL := "s3://gdcdata/00664eeb-830c-5fe4-b48c-054cd9c8e02f/239f8402efd37b62bfb892aa4becb0692b3ca5f58015083d8567e8d7fbdd1843" + + target, err := om.ResolveCanonicalStorageTarget(context.Background(), CanonicalStorageTargetRequest{ + Object: obj, + AccessURL: sourceURL, + PreferChecksum: true, + }) + if err != nil { + t.Fatalf("ResolveCanonicalStorageTarget failed: %v", err) + } + if target.URL != sourceURL { + t.Fatalf("expected root scoped physical URL to remain %q, got %q", sourceURL, target.URL) + } + if target.Bucket != "gdcdata" { + t.Fatalf("expected bucket gdcdata, got %q", target.Bucket) + } + if target.Key != "00664eeb-830c-5fe4-b48c-054cd9c8e02f/239f8402efd37b62bfb892aa4becb0692b3ca5f58015083d8567e8d7fbdd1843" { + t.Fatalf("unexpected target key: %q", target.Key) + } + }) + t.Run("create bucket scope updates signing cache", func(t *testing.T) { mockDB := &testutils.MockDatabase{} db := &coreTestDB{MockDatabase: mockDB} @@ -979,4 +1035,5 @@ func TestObjectManagerDeleteResolveAndSignDelegation(t *testing.T) { t.Fatalf("expected scoped storage url %q, got %q", want, um.signURLAccessURL) } }) + } diff --git a/internal/core/storage_signing.go b/internal/core/storage_signing.go index ac085729..6be8acd0 100644 --- a/internal/core/storage_signing.go +++ b/internal/core/storage_signing.go @@ -69,6 +69,9 @@ func (m *ObjectManager) ResolveCanonicalStorageTarget(ctx context.Context, req C return CanonicalStorageTarget{}, fmt.Errorf("unable to resolve scoped storage bucket for object %s", obj.Id) } targetKey := m.canonicalObjectKey(obj, req.Key, existingKey, req.PreferChecksum) + if existingOK && strings.EqualFold(strings.TrimSpace(existingBucket), targetBucket) && len(normalizedScopePrefixes(scopes)) == 0 && strings.TrimSpace(existingKey) != "" { + targetKey = existingKey + } targetKey = normalizeScopedStorageKey(targetKey, scopes) if strings.TrimSpace(targetKey) == "" { return CanonicalStorageTarget{}, fmt.Errorf("unable to resolve scoped storage key for object %s", obj.Id) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 482cec58..69cc4b89 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -169,19 +169,7 @@ func newAWSKMSKeyManagerFromEnv() (CredentialKeyManager, error) { }, nil } -func RegisterCredentialKeyManager(name string, factory func() (CredentialKeyManager, error)) error { - name = strings.ToLower(strings.TrimSpace(name)) - if name == "" { - return errors.New("credential key manager name is required") - } - if factory == nil { - return errors.New("credential key manager factory is required") - } - credentialKeyManagerRegistryMu.Lock() - defer credentialKeyManagerRegistryMu.Unlock() - credentialKeyManagerRegistry[name] = factory - return nil -} + func resolveCredentialKeyManager(name string) (CredentialKeyManager, error) { managerName := strings.ToLower(strings.TrimSpace(name)) diff --git a/internal/db/interface.go b/internal/db/interface.go index 30791773..c03dd652 100644 --- a/internal/db/interface.go +++ b/internal/db/interface.go @@ -21,6 +21,7 @@ type ObjectStore interface { CreateObject(ctx context.Context, obj *models.InternalObject) error GetObjectsByChecksum(ctx context.Context, checksum string) ([]models.InternalObject, error) GetObjectsByChecksums(ctx context.Context, checksums []string) (map[string][]models.InternalObject, error) + ListScopedObjectIDsByChecksums(ctx context.Context, organization, project string, checksums []string) (map[string][]string, error) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) CreateObjectAlias(ctx context.Context, aliasID, canonicalObjectID string) error ResolveObjectAlias(ctx context.Context, aliasID string) (string, error) @@ -41,10 +42,6 @@ type ObjectIDPageLister interface { ListObjectIDsPageByResources(ctx context.Context, resources []string, includeUnscoped bool, startAfter string, limit, offset int) ([]string, error) } -type ObjectPathPageLister interface { - ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) -} - type ObjectChecksumPageLister interface { ListObjectIDsPageByChecksum(ctx context.Context, checksum, checksumType, organization, project, startAfter string, limit, offset int, resources []string, includeUnscoped, restrictToResources bool) ([]string, error) } @@ -63,6 +60,7 @@ type FileUsageScopedLister interface { ListFileUsagePageByResources(ctx context.Context, resources []string, includeUnscoped bool, limit, offset int, inactiveSince *time.Time) ([]models.FileUsage, error) GetFileUsageSummaryByScope(ctx context.Context, organization, project string, inactiveSince *time.Time) (models.FileUsageSummary, error) GetFileUsageSummaryByResources(ctx context.Context, resources []string, includeUnscoped bool, inactiveSince *time.Time) (models.FileUsageSummary, error) + GetProjectRecordSummaryByScope(ctx context.Context, organization, project string) (models.FileUsageSummary, error) } type TransferAttributionScopedStore interface { diff --git a/internal/db/postgres/browse_index.go b/internal/db/postgres/browse_index.go deleted file mode 100644 index 041673ca..00000000 --- a/internal/db/postgres/browse_index.go +++ /dev/null @@ -1,163 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - "fmt" - "sort" - "strings" - - sycommon "github.com/calypr/syfon/common" - icommon "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" - "github.com/lib/pq" -) - -type browseIndexRow struct { - ObjectID string - Resource string - NormalizedPath string - ParentPath string - EntryName string -} - -func postgresBrowseRowsForObject(objectID, objectName string, resources []string) []browseIndexRow { - info, ok, err := icommon.BrowsePathInfoFromName(objectName) - if err != nil || !ok { - return nil - } - normalizedResources := sycommon.NormalizeAccessResources(resources) - rows := make([]browseIndexRow, 0, len(normalizedResources)) - for _, resource := range normalizedResources { - rows = append(rows, browseIndexRow{ - ObjectID: objectID, - Resource: resource, - NormalizedPath: info.Normalized, - ParentPath: info.ParentPath, - EntryName: info.EntryName, - }) - } - return rows -} - -func postgresDeleteBrowseRowsByIDsTx(ctx context.Context, tx *sql.Tx, ids []string) error { - if len(ids) == 0 { - return nil - } - _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = ANY($1)`, pq.Array(ids)) - return err -} - -func postgresDeleteBrowseRowTx(ctx context.Context, tx *sql.Tx, objectID, resource string) error { - _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = $1 AND resource = $2`, objectID, resource) - return err -} - -func postgresInsertBrowseRowsTx(ctx context.Context, tx *sql.Tx, rows []browseIndexRow) error { - if len(rows) == 0 { - return nil - } - objectIDs := make([]string, 0, len(rows)) - resources := make([]string, 0, len(rows)) - normalizedPaths := make([]string, 0, len(rows)) - parentPaths := make([]string, 0, len(rows)) - entryNames := make([]string, 0, len(rows)) - for _, row := range rows { - objectIDs = append(objectIDs, row.ObjectID) - resources = append(resources, row.Resource) - normalizedPaths = append(normalizedPaths, row.NormalizedPath) - parentPaths = append(parentPaths, row.ParentPath) - entryNames = append(entryNames, row.EntryName) - } - _, err := tx.ExecContext(ctx, ` - INSERT INTO drs_object_browse_index (object_id, resource, normalized_path, parent_path, entry_name) - SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::text[], $5::text[]) - ON CONFLICT (resource, object_id) DO UPDATE SET - normalized_path = EXCLUDED.normalized_path, - parent_path = EXCLUDED.parent_path, - entry_name = EXCLUDED.entry_name`, - pq.Array(objectIDs), pq.Array(resources), pq.Array(normalizedPaths), pq.Array(parentPaths), pq.Array(entryNames), - ) - return err -} - -func postgresRebuildBrowseRowsForObjectTx(ctx context.Context, tx *sql.Tx, objectID, objectName string, resources []string) error { - if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = $1`, objectID); err != nil { - return err - } - return postgresInsertBrowseRowsTx(ctx, tx, postgresBrowseRowsForObject(objectID, objectName, resources)) -} - -func (db *PostgresDB) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { - resource, err := sycommon.ResourcePath(organization, project) - if err != nil { - return nil, nil, err - } - normalizedPath, _, err := icommon.NormalizeBrowsePath(path) - if err != nil { - return nil, nil, err - } - if offset < 0 { - offset = 0 - } - - args := []any{resource, normalizedPath} - query := ` - SELECT object_id - FROM drs_object_browse_index - WHERE resource = $1 AND parent_path = $2` - if strings.TrimSpace(startAfter) != "" { - args = append(args, strings.TrimSpace(startAfter)) - query += fmt.Sprintf(" AND object_id > $%d", len(args)) - } - args = append(args, limit, offset) - query += fmt.Sprintf(" ORDER BY object_id LIMIT $%d OFFSET $%d", len(args)-1, len(args)) - rows, err := db.db.QueryContext(ctx, query, args...) - if err != nil { - return nil, nil, err - } - defer rows.Close() - ids, err := scanObjectIDs(rows) - if err != nil { - return nil, nil, err - } - - dirQuery := `SELECT normalized_path FROM drs_object_browse_index WHERE resource = $1 AND parent_path <> ''` - dirArgs := []any{resource} - if normalizedPath != "" { - dirQuery = `SELECT normalized_path FROM drs_object_browse_index WHERE resource = $1 AND normalized_path LIKE $2 AND parent_path <> $3` - dirArgs = append(dirArgs, normalizedPath+"/%", normalizedPath) - } - dirRows, err := db.db.QueryContext(ctx, dirQuery+" ORDER BY normalized_path", dirArgs...) - if err != nil { - return nil, nil, err - } - defer dirRows.Close() - - directoriesByPath := make(map[string]models.BrowseDirectory) - for dirRows.Next() { - var candidatePath string - if err := dirRows.Scan(&candidatePath); err != nil { - return nil, nil, err - } - dirInfo, ok := icommon.ImmediateBrowseDirectory(normalizedPath, candidatePath) - if !ok { - continue - } - directoriesByPath[dirInfo.Normalized] = models.BrowseDirectory{Name: dirInfo.EntryName, Path: dirInfo.Normalized} - } - if err := dirRows.Err(); err != nil { - return nil, nil, err - } - directories := make([]models.BrowseDirectory, 0, len(directoriesByPath)) - for _, directory := range directoriesByPath { - directories = append(directories, directory) - } - sort.Slice(directories, func(i, j int) bool { - if directories[i].Name == directories[j].Name { - return directories[i].Path < directories[j].Path - } - return directories[i].Name < directories[j].Name - }) - return ids, directories, nil -} diff --git a/internal/db/postgres/credentials.go b/internal/db/postgres/credentials.go index 36ce4461..dddf02b7 100644 --- a/internal/db/postgres/credentials.go +++ b/internal/db/postgres/credentials.go @@ -91,6 +91,10 @@ func (db *PostgresDB) SaveS3Credential(ctx context.Context, cred *models.S3Crede common.AuditS3CredentialAccess(ctx, "write", bucket, wrapped) return wrapped } + if err := db.ensureUniquePhysicalBucket(ctx, stored.CredentialID, stored.Bucket); err != nil { + common.AuditS3CredentialAccess(ctx, "write", stored.Bucket, err) + return err + } _, err = db.db.ExecContext(ctx, ` INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) @@ -113,6 +117,29 @@ func (db *PostgresDB) SaveS3Credential(ctx context.Context, cred *models.S3Crede return nil } +func (db *PostgresDB) ensureUniquePhysicalBucket(ctx context.Context, credentialID, bucket string) error { + credentialID = strings.TrimSpace(credentialID) + bucket = strings.TrimSpace(bucket) + if bucket == "" { + return nil + } + + var existingCredentialID string + err := db.db.QueryRowContext(ctx, ` + SELECT credential_id + FROM s3_credential + WHERE bucket = $1 AND credential_id <> $2 + LIMIT 1 + `, bucket, credentialID).Scan(&existingCredentialID) + if err == nil { + return fmt.Errorf("physical bucket %q is already configured under credential %q; reuse that credential and add a bucket scope instead", bucket, existingCredentialID) + } + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return fmt.Errorf("failed to validate physical bucket uniqueness: %w", err) +} + func (db *PostgresDB) DeleteS3Credential(ctx context.Context, credentialID string) error { resolvedID, err := db.resolveCredentialID(ctx, credentialID) if err != nil { diff --git a/internal/db/postgres/credentials_sqlmock_test.go b/internal/db/postgres/credentials_sqlmock_test.go index 005dbe96..11ee9654 100644 --- a/internal/db/postgres/credentials_sqlmock_test.go +++ b/internal/db/postgres/credentials_sqlmock_test.go @@ -104,6 +104,14 @@ func TestSaveS3Credential(t *testing.T) { defer rawDB.Close() derivedID := common.DeriveCredentialID("b1", "", "us-east-1", "https://s3.example", "ak") + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT credential_id + FROM s3_credential + WHERE bucket = $1 AND credential_id <> $2 + LIMIT 1 + `)). + WithArgs("b1", derivedID). + WillReturnError(sql.ErrNoRows) mock.ExpectExec(regexp.QuoteMeta(` INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) VALUES ($1, $2, $3, $4, $5, $6, $7) @@ -130,6 +138,33 @@ func TestSaveS3Credential(t *testing.T) { } } +func TestSaveS3CredentialRejectsDuplicatePhysicalBucket(t *testing.T) { + t.Setenv(crypto.CredentialMasterKeyEnv, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") + pg, mock, rawDB := newMockPostgresDB(t) + defer rawDB.Close() + + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT credential_id + FROM s3_credential + WHERE bucket = $1 AND credential_id <> $2 + LIMIT 1 + `)). + WithArgs("shared-bucket", "org-b/default"). + WillReturnRows(sqlmock.NewRows([]string{"credential_id"}).AddRow("org-a/default")) + + err := pg.SaveS3Credential(context.Background(), &models.S3Credential{ + CredentialID: "org-b/default", + Bucket: "shared-bucket", + Provider: "s3", + Region: "us-east-1", + AccessKey: "ak", + SecretKey: "sk", + }) + if err == nil || err.Error() != `physical bucket "shared-bucket" is already configured under credential "org-a/default"; reuse that credential and add a bucket scope instead` { + t.Fatalf("expected duplicate physical bucket error, got %v", err) + } +} + func TestDeleteS3Credential(t *testing.T) { t.Run("success", func(t *testing.T) { pg, mock, rawDB := newMockPostgresDB(t) diff --git a/internal/db/postgres/metrics.go b/internal/db/postgres/metrics.go index 8e7138b0..1fb75481 100644 --- a/internal/db/postgres/metrics.go +++ b/internal/db/postgres/metrics.go @@ -363,6 +363,27 @@ func (db *PostgresDB) GetFileUsageSummaryByResources(ctx context.Context, resour return db.getScopedFileUsageSummary(ctx, resources, includeUnscoped, inactiveSince) } +func (db *PostgresDB) GetProjectRecordSummaryByScope(ctx context.Context, organization, project string) (models.FileUsageSummary, error) { + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return models.FileUsageSummary{}, err + } + var summary models.FileUsageSummary + var latest sql.NullTime + if err := db.db.QueryRowContext(ctx, ` + SELECT COUNT(DISTINCT o.id), MAX(o.updated_time) + FROM drs_object o + INNER JOIN drs_object_controlled_access ca ON ca.object_id = o.id + WHERE ca.resource = $1`, resource).Scan(&summary.RecordCount, &latest); err != nil { + return models.FileUsageSummary{}, err + } + if latest.Valid { + t := latest.Time.UTC() + summary.RecordLatestUpdatedTime = &t + } + return summary, nil +} + func scanFileUsageRows(rows *sql.Rows, capacity int) ([]models.FileUsage, error) { out := make([]models.FileUsage, 0, capacity) for rows.Next() { @@ -615,13 +636,6 @@ func nullableInt64(v *int64) any { return *v } -func nullableTime(v *time.Time) any { - if v == nil { - return nil - } - return v.UTC() -} - func (db *PostgresDB) backfillAccessGrants(ctx context.Context) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -1033,79 +1047,6 @@ func postgresTransferResourceClause(resources []string, startIndex int) (string, return strings.Join(clauses, " OR "), args } -func providerTransferWhere(filter models.TransferAttributionFilter) (string, []any) { - parts := make([]string, 0) - args := make([]any, 0) - add := func(column string, value any) { - args = append(args, value) - parts = append(parts, fmt.Sprintf("%s = $%d", column, len(args))) - } - status := strings.TrimSpace(filter.ReconciliationStatus) - if status == "" { - status = models.ProviderTransferMatched - } - if status != "all" { - add("reconciliation_status", status) - } - if strings.TrimSpace(filter.Organization) != "" { - add("organization", strings.TrimSpace(filter.Organization)) - } - if strings.TrimSpace(filter.Project) != "" { - add("project", strings.TrimSpace(filter.Project)) - } - direction := strings.TrimSpace(filter.Direction) - if direction == "" { - switch strings.TrimSpace(filter.EventType) { - case models.ProviderTransferDirectionDownload: - direction = models.ProviderTransferDirectionDownload - case models.ProviderTransferDirectionUpload: - direction = models.ProviderTransferDirectionUpload - } - } - if direction != "" && direction != "all" { - add("direction", direction) - } - if filter.From != nil { - args = append(args, filter.From.UTC()) - parts = append(parts, fmt.Sprintf("event_time >= $%d", len(args))) - } - if filter.To != nil { - args = append(args, filter.To.UTC()) - parts = append(parts, fmt.Sprintf("event_time <= $%d", len(args))) - } - if strings.TrimSpace(filter.Provider) != "" { - add("provider", strings.TrimSpace(filter.Provider)) - } - if strings.TrimSpace(filter.Bucket) != "" { - add("bucket", strings.TrimSpace(filter.Bucket)) - } - if strings.TrimSpace(filter.SHA256) != "" { - add("sha256", strings.TrimSpace(filter.SHA256)) - } - if strings.TrimSpace(filter.User) != "" { - user := strings.TrimSpace(filter.User) - args = append(args, user, user) - parts = append(parts, fmt.Sprintf("(actor_email = $%d OR actor_subject = $%d)", len(args)-1, len(args))) - } - if len(parts) == 0 { - return "", args - } - return " WHERE " + strings.Join(parts, " AND "), args -} - -func providerTransferGroupExpr(groupBy string) (string, string) { - switch strings.ToLower(strings.TrimSpace(groupBy)) { - case "user": - return "COALESCE(NULLIF(actor_email, ''), actor_subject), actor_email, actor_subject", "COALESCE(NULLIF(actor_email, ''), actor_subject) AS key, '' AS organization, '' AS project, '' AS provider, '' AS bucket, '' AS sha256, actor_email, actor_subject" - case "provider": - return "provider, bucket", "provider || ':' || bucket AS key, '' AS organization, '' AS project, provider, bucket, '' AS sha256, '' AS actor_email, '' AS actor_subject" - case "object": - return "sha256", "sha256 AS key, '' AS organization, '' AS project, '' AS provider, '' AS bucket, sha256, '' AS actor_email, '' AS actor_subject" - default: - return "organization, project", "organization || '/' || project AS key, organization, project, '' AS provider, '' AS bucket, '' AS sha256, '' AS actor_email, '' AS actor_subject" - } -} - func transferAttributionGroupExpr(groupBy string) (string, string) { switch strings.ToLower(strings.TrimSpace(groupBy)) { case "user": diff --git a/internal/db/postgres/metrics_sqlmock_test.go b/internal/db/postgres/metrics_sqlmock_test.go index b1f92628..4b43cd62 100644 --- a/internal/db/postgres/metrics_sqlmock_test.go +++ b/internal/db/postgres/metrics_sqlmock_test.go @@ -70,17 +70,6 @@ func TestTransferAttributionWhereHelpers(t *testing.T) { } func TestProviderTransferHelpers(t *testing.T) { - where, args := providerTransferWhere(models.TransferAttributionFilter{User: "alice", ReconciliationStatus: "all"}) - if !strings.Contains(where, "actor_email") { - t.Fatalf("expected user filter in provider where clause: %q", where) - } - if len(args) < 2 { - t.Fatalf("expected user args in provider where clause") - } - - if key, _ := providerTransferGroupExpr("user"); !strings.Contains(key, "actor_email") { - t.Fatalf("unexpected provider group expression for user: %q", key) - } if key, _ := transferAttributionGroupExpr("provider"); !strings.Contains(key, "provider") { t.Fatalf("unexpected attribution group expression for provider: %q", key) } @@ -108,6 +97,3 @@ func TestRecordProviderTransferEvents_EmptyInput(t *testing.T) { t.Fatalf("expected nil for empty provider transfer events, got %v", err) } } - - - diff --git a/internal/db/postgres/object_schema.sql b/internal/db/postgres/object_schema.sql index c5eb31d1..617e4ff7 100644 --- a/internal/db/postgres/object_schema.sql +++ b/internal/db/postgres/object_schema.sql @@ -4,13 +4,12 @@ CREATE TABLE IF NOT EXISTS drs_object ( created_time TIMESTAMPTZ, updated_time TIMESTAMPTZ, name TEXT, - file_name TEXT, version TEXT, description TEXT ); ALTER TABLE drs_object - ADD COLUMN IF NOT EXISTS file_name TEXT; + DROP COLUMN IF EXISTS file_name; CREATE TABLE IF NOT EXISTS drs_object_access_method ( object_id TEXT NOT NULL, @@ -38,16 +37,15 @@ CREATE TABLE IF NOT EXISTS drs_object_alias ( FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE ); -CREATE TABLE IF NOT EXISTS drs_object_browse_index ( +CREATE TABLE IF NOT EXISTS drs_object_name_alias ( object_id TEXT NOT NULL, - resource TEXT NOT NULL, - normalized_path TEXT NOT NULL, - parent_path TEXT NOT NULL, - entry_name TEXT NOT NULL, - PRIMARY KEY (resource, object_id), + name_alias TEXT NOT NULL, + PRIMARY KEY (object_id, name_alias), FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE ); +DROP TABLE IF EXISTS drs_object_browse_index; + CREATE INDEX IF NOT EXISTS drs_object_access_method_object_id_idx ON drs_object_access_method(object_id); @@ -75,11 +73,5 @@ CREATE INDEX IF NOT EXISTS drs_object_controlled_access_object_id_resource_idx CREATE INDEX IF NOT EXISTS drs_object_alias_object_id_idx ON drs_object_alias(object_id); -CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_object_id_idx - ON drs_object_browse_index(resource, parent_path, object_id); - -CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_entry_name_idx - ON drs_object_browse_index(resource, parent_path, entry_name); - -CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_normalized_path_idx - ON drs_object_browse_index(resource, normalized_path); +CREATE INDEX IF NOT EXISTS drs_object_name_alias_object_id_idx + ON drs_object_name_alias(object_id); diff --git a/internal/db/postgres/objects.go b/internal/db/postgres/objects.go index c67f71a7..ee0c301d 100644 --- a/internal/db/postgres/objects.go +++ b/internal/db/postgres/objects.go @@ -5,7 +5,6 @@ import ( "database/sql" "errors" "fmt" - "path" "sort" "strings" "time" @@ -17,34 +16,6 @@ import ( "github.com/lib/pq" ) -func postgresStoredFileName(obj *models.InternalObject) string { - if obj != nil && obj.Properties != nil { - if raw, ok := obj.Properties["file_name"].(string); ok && strings.TrimSpace(raw) != "" { - return strings.TrimSpace(raw) - } - } - if obj == nil { - return "" - } - return common.StringVal(obj.Name) -} - -func postgresLoadedNames(name, fileName sql.NullString) (string, string) { - loadedName := strings.TrimSpace(name.String) - loadedFileName := strings.TrimSpace(fileName.String) - if loadedFileName == "" { - loadedFileName = loadedName - loadedName = "" - } - if loadedName == "" && loadedFileName != "" { - loadedName = path.Base(strings.Trim(loadedFileName, "/")) - if loadedName == "." || loadedName == "/" || loadedName == "" { - loadedName = loadedFileName - } - } - return loadedName, loadedFileName -} - func (db *PostgresDB) DeleteObject(ctx context.Context, id string) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -139,11 +110,11 @@ func (db *PostgresDB) GetObject(ctx context.Context, id string) (*models.Interna retryLookup: // 1. Fetch main record var r models.DrsObjectRecord - var name, fileName, version, description sql.NullString + var name, version, description sql.NullString err := db.db.QueryRowContext(ctx, ` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = $1`, lookupID).Scan( - &r.ID, &r.Size, &r.CreatedTime, &r.UpdatedTime, &name, &fileName, &version, &description, + &r.ID, &r.Size, &r.CreatedTime, &r.UpdatedTime, &name, &version, &description, ) if err == sql.ErrNoRows { if !resolvedAlias { @@ -162,9 +133,13 @@ retryLookup: if err != nil { return nil, fmt.Errorf("failed to fetch record: %w", err) } - r.Name, r.FileName = postgresLoadedNames(name, fileName) + r.Name = strings.TrimSpace(name.String) r.Version = version.String r.Description = description.String + nameAliases, err := db.nameAliasesForObject(ctx, lookupID) + if err != nil { + return nil, err + } objectID := r.ID if resolvedAlias && requestID != "" { objectID = requestID @@ -181,10 +156,8 @@ retryLookup: Name: common.Ptr(r.Name), SelfUri: "drs://" + objectID, }, - Properties: map[string]interface{}{}, - } - if strings.TrimSpace(r.FileName) != "" { - obj.Properties["file_name"] = r.FileName + NameAliases: nameAliases, + Properties: map[string]interface{}{}, } // 2. Fetch storage access methods. urlRows, err := db.db.QueryContext(ctx, "SELECT url, type FROM drs_object_access_method WHERE object_id = $1", lookupID) @@ -258,9 +231,9 @@ func (db *PostgresDB) CreateObject(ctx context.Context, obj *models.InternalObje // Insert main record _, err = tx.ExecContext(ctx, ` - INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), postgresStoredFileName(obj), common.StringVal(obj.Version), common.StringVal(obj.Description), + INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.CleanToBasename(common.StringVal(obj.Name)), common.StringVal(obj.Version), common.StringVal(obj.Description), ) if err != nil { return fmt.Errorf("failed to insert drs_object: %w", err) @@ -269,8 +242,11 @@ func (db *PostgresDB) CreateObject(ctx context.Context, obj *models.InternalObje if err := insertControlledAccessTx(ctx, tx, obj.Id, objectAccessResources(obj)); err != nil { return err } - if err := postgresRebuildBrowseRowsForObjectTx(ctx, tx, obj.Id, postgresStoredFileName(obj), objectAccessResources(obj)); err != nil { - return fmt.Errorf("failed to update browse index: %w", err) + for _, alias := range normalizeObjectNameAliases(obj) { + _, err = tx.ExecContext(ctx, `INSERT INTO drs_object_name_alias (object_id, name_alias) VALUES ($1, $2)`, obj.Id, alias) + if err != nil { + return fmt.Errorf("failed to insert name alias: %w", err) + } } // Insert storage access methods. @@ -322,26 +298,25 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte names := make([]string, 0, len(objects)) versions := make([]string, 0, len(objects)) descriptions := make([]string, 0, len(objects)) - fileNames := make([]string, 0, len(objects)) accessObjectIDs := make([]string, 0) accessURLs := make([]string, 0) accessTypes := make([]string, 0) controlledObjectIDs := make([]string, 0) controlledResources := make([]string, 0) - browseRows := make([]browseIndexRow, 0) checksumObjectIDs := make([]string, 0) checksumTypes := make([]string, 0) checksumValues := make([]string, 0) + nameAliasObjectIDs := make([]string, 0) + nameAliasValues := make([]string, 0) for _, obj := range objects { ids = append(ids, obj.Id) sizes = append(sizes, obj.Size) createdTimes = append(createdTimes, obj.CreatedTime) updatedTimes = append(updatedTimes, common.TimeVal(obj.UpdatedTime)) - names = append(names, common.StringVal(obj.Name)) - fileNames = append(fileNames, postgresStoredFileName(&obj)) + names = append(names, common.CleanToBasename(common.StringVal(obj.Name))) versions = append(versions, common.StringVal(obj.Version)) descriptions = append(descriptions, common.StringVal(obj.Description)) @@ -350,7 +325,10 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte controlledObjectIDs = append(controlledObjectIDs, obj.Id) controlledResources = append(controlledResources, resource) } - browseRows = append(browseRows, postgresBrowseRowsForObject(obj.Id, postgresStoredFileName(&obj), objectAccessResources(&obj))...) + for _, alias := range normalizeObjectNameAliases(&obj) { + nameAliasObjectIDs = append(nameAliasObjectIDs, obj.Id) + nameAliasValues = append(nameAliasValues, alias) + } if obj.AccessMethods != nil { for _, am := range *obj.AccessMethods { if am.AccessUrl == nil || am.AccessUrl.Url == "" { @@ -381,18 +359,17 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte } if _, err := tx.ExecContext(ctx, ` - INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) - SELECT * FROM UNNEST($1::text[], $2::bigint[], $3::timestamp[], $4::timestamp[], $5::text[], $6::text[], $7::text[], $8::text[]) + INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) + SELECT * FROM UNNEST($1::text[], $2::bigint[], $3::timestamp[], $4::timestamp[], $5::text[], $6::text[], $7::text[]) ON CONFLICT (id) DO UPDATE SET size = EXCLUDED.size, created_time = EXCLUDED.created_time, updated_time = EXCLUDED.updated_time, name = EXCLUDED.name, - file_name = EXCLUDED.file_name, version = EXCLUDED.version, description = EXCLUDED.description`, pq.Array(ids), pq.Array(sizes), pq.Array(createdTimes), pq.Array(updatedTimes), - pq.Array(names), pq.Array(fileNames), pq.Array(versions), pq.Array(descriptions), + pq.Array(names), pq.Array(versions), pq.Array(descriptions), ); err != nil { return fmt.Errorf("failed bulk upsert drs_object: %w", err) } @@ -406,8 +383,8 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_checksum WHERE object_id = ANY($1)`, pq.Array(ids)); err != nil { return fmt.Errorf("failed bulk clear checksums: %w", err) } - if err := postgresDeleteBrowseRowsByIDsTx(ctx, tx, ids); err != nil { - return fmt.Errorf("failed bulk clear browse index: %w", err) + if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_name_alias WHERE object_id = ANY($1)`, pq.Array(ids)); err != nil { + return fmt.Errorf("failed bulk clear name aliases: %w", err) } if len(accessObjectIDs) > 0 { @@ -428,6 +405,15 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte return fmt.Errorf("failed bulk insert controlled access: %w", err) } } + if len(nameAliasObjectIDs) > 0 { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO drs_object_name_alias (object_id, name_alias) + SELECT * FROM UNNEST($1::text[], $2::text[])`, + pq.Array(nameAliasObjectIDs), pq.Array(nameAliasValues), + ); err != nil { + return fmt.Errorf("failed bulk insert name aliases: %w", err) + } + } if len(checksumObjectIDs) > 0 { if _, err := tx.ExecContext(ctx, ` @@ -438,9 +424,6 @@ func (db *PostgresDB) RegisterObjects(ctx context.Context, objects []models.Inte return fmt.Errorf("failed bulk insert checksums: %w", err) } } - if err := postgresInsertBrowseRowsTx(ctx, tx, browseRows); err != nil { - return fmt.Errorf("failed bulk insert browse index: %w", err) - } if err := db.flushObjectUsageEventsForIDsTx(ctx, tx, ids); err != nil { return fmt.Errorf("failed to apply object usage events: %w", err) @@ -522,6 +505,61 @@ func (db *PostgresDB) GetObjectsByChecksums(ctx context.Context, checksums []str return result, nil } +func (db *PostgresDB) ListScopedObjectIDsByChecksums(ctx context.Context, organization, project string, checksums []string) (map[string][]string, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" || project == "" || len(checksums) == 0 { + return map[string][]string{}, nil + } + resource, err := sycommon.ResourcePath(organization, project) + if err != nil { + return nil, err + } + normalized := uniqueNonEmptyStrings(checksums) + if len(normalized) == 0 { + return map[string][]string{}, nil + } + rows, err := db.db.QueryContext(ctx, ` + SELECT DISTINCT c.checksum, c.object_id + FROM drs_object_checksum c + INNER JOIN drs_object_controlled_access ca ON ca.object_id = c.object_id + WHERE ca.resource = $1 AND c.type = $2 AND c.checksum = ANY($3) + ORDER BY c.checksum, c.object_id`, resource, "sha256", pq.Array(normalized)) + if err != nil { + return nil, err + } + defer rows.Close() + out := make(map[string][]string, len(normalized)) + for _, checksum := range normalized { + out[checksum] = []string{} + } + seen := make(map[string]map[string]struct{}, len(normalized)) + for rows.Next() { + var checksum string + var objectID string + if err := rows.Scan(&checksum, &objectID); err != nil { + return nil, err + } + checksum = strings.TrimSpace(checksum) + objectID = strings.TrimSpace(objectID) + if checksum == "" || objectID == "" { + continue + } + if seen[checksum] == nil { + seen[checksum] = make(map[string]struct{}) + } + if _, ok := seen[checksum][objectID]; ok { + continue + } + seen[checksum][objectID] = struct{}{} + out[checksum] = append(out[checksum], objectID) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func (db *PostgresDB) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) { organization = strings.TrimSpace(organization) project = strings.TrimSpace(project) @@ -1140,7 +1178,6 @@ func (db *PostgresDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []st o.created_time, o.updated_time, o.name, - o.file_name, o.version, o.description FROM drs_object o @@ -1167,35 +1204,30 @@ func (db *PostgresDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []st for rows.Next() { var ( - id string - name, fileName sql.NullString - version, description sql.NullString - size int64 - createdTime, updatedTime time.Time + id string + name sql.NullString + version, description sql.NullString + size int64 + createdTime, updatedTime time.Time ) if err := rows.Scan( - &id, &size, &createdTime, &updatedTime, &name, &fileName, &version, &description, + &id, &size, &createdTime, &updatedTime, &name, &version, &description, ); err != nil { return nil, err } - - loadedName, loadedFileName := postgresLoadedNames(name, fileName) objectsByID[id] = &models.InternalObject{ DrsObject: drs.DrsObject{ Id: id, Size: size, CreatedTime: createdTime, UpdatedTime: common.Ptr(updatedTime), - Name: common.Ptr(loadedName), + Name: common.Ptr(strings.TrimSpace(name.String)), Version: common.Ptr(version.String), Description: common.Ptr(description.String), SelfUri: "drs://" + id, }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(loadedFileName) != "" { - objectsByID[id].Properties["file_name"] = loadedFileName - } } if err := rows.Err(); err != nil { @@ -1213,6 +1245,9 @@ func (db *PostgresDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []st if err := db.attachControlledAccess(ctx, objectsByID); err != nil { return nil, err } + if err := db.attachNameAliases(ctx, objectsByID); err != nil { + return nil, err + } return objectsByID, nil } @@ -1305,6 +1340,23 @@ func sortedObjectIDs(objectsByID map[string]*models.InternalObject) []string { return ids } +func uniqueNonEmptyStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + normalized := strings.TrimSpace(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + func (db *PostgresDB) UpdateObjectAccessMethods(ctx context.Context, objectID string, accessMethods []drs.AccessMethod) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -1353,9 +1405,6 @@ func (db *PostgresDB) RemoveObjectControlledAccess(ctx context.Context, objectID if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_controlled_access WHERE object_id = $1 AND resource = $2`, objectID, resource); err != nil { return err } - if err := postgresDeleteBrowseRowTx(ctx, tx, objectID, resource); err != nil { - return err - } return tx.Commit() } @@ -1394,6 +1443,13 @@ func objectAccessResources(obj *models.InternalObject) []string { return sycommon.AuthzMapToList(obj.Authorizations) } +func normalizeObjectNameAliases(obj *models.InternalObject) []string { + if obj == nil { + return nil + } + return common.NormalizeNameAliases(common.StringVal(obj.Name), obj.NameAliases) +} + func insertControlledAccessTx(ctx context.Context, tx *sql.Tx, objectID string, resources []string) error { for _, resource := range sycommon.NormalizeAccessResources(resources) { if _, err := tx.ExecContext(ctx, `INSERT INTO drs_object_controlled_access (object_id, resource) VALUES ($1, $2)`, objectID, resource); err != nil { @@ -1423,6 +1479,27 @@ func (db *PostgresDB) controlledAccessForObject(ctx context.Context, objectID st return sycommon.NormalizeAccessResources(resources), nil } +func (db *PostgresDB) nameAliasesForObject(ctx context.Context, objectID string) ([]string, error) { + rows, err := db.db.QueryContext(ctx, `SELECT name_alias FROM drs_object_name_alias WHERE object_id = $1 ORDER BY name_alias`, objectID) + if err != nil { + return nil, err + } + defer rows.Close() + + aliases := make([]string, 0) + for rows.Next() { + var alias string + if err := rows.Scan(&alias); err != nil { + return nil, err + } + aliases = append(aliases, alias) + } + if err := rows.Err(); err != nil { + return nil, err + } + return common.NormalizeNameAliases("", aliases), nil +} + func (db *PostgresDB) attachControlledAccess(ctx context.Context, objectsByID map[string]*models.InternalObject) error { if len(objectsByID) == 0 { return nil @@ -1468,3 +1545,39 @@ func (db *PostgresDB) attachControlledAccess(ctx context.Context, objectsByID ma } return nil } + +func (db *PostgresDB) attachNameAliases(ctx context.Context, objectsByID map[string]*models.InternalObject) error { + if len(objectsByID) == 0 { + return nil + } + rows, err := db.db.QueryContext(ctx, ` + SELECT object_id, name_alias + FROM drs_object_name_alias + WHERE object_id = ANY($1) + ORDER BY object_id, name_alias`, pq.Array(sortedObjectIDs(objectsByID))) + if err != nil { + return fmt.Errorf("failed to fetch bulk object name aliases: %w", err) + } + defer rows.Close() + + byObject := make(map[string][]string, len(objectsByID)) + for rows.Next() { + var objectID, alias string + if err := rows.Scan(&objectID, &alias); err != nil { + return err + } + byObject[objectID] = append(byObject[objectID], alias) + } + if err := rows.Err(); err != nil { + return err + } + + for objectID, aliases := range byObject { + obj := objectsByID[objectID] + if obj == nil { + continue + } + obj.NameAliases = common.NormalizeNameAliases(common.StringVal(obj.Name), aliases) + } + return nil +} diff --git a/internal/db/postgres/objects_sqlmock_test.go b/internal/db/postgres/objects_sqlmock_test.go index 24935b44..a262703f 100644 --- a/internal/db/postgres/objects_sqlmock_test.go +++ b/internal/db/postgres/objects_sqlmock_test.go @@ -87,7 +87,7 @@ func TestGetObject_NotFound(t *testing.T) { defer rawDB.Close() mock.ExpectQuery(regexp.QuoteMeta(` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = $1`)). WithArgs("missing"). WillReturnError(sql.ErrNoRows) @@ -108,12 +108,16 @@ func TestGetObject_DeduplicatesAndPropagatesAuthz(t *testing.T) { created := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) updated := created.Add(2 * time.Hour) mock.ExpectQuery(regexp.QuoteMeta(` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = $1`)). WithArgs("obj-1"). WillReturnRows(sqlmock.NewRows([]string{ - "id", "size", "created_time", "updated_time", "name", "file_name", "version", "description", - }).AddRow("obj-1", int64(123), created, updated, "file.txt", "nested/file.txt", "v1", "desc")) + "id", "size", "created_time", "updated_time", "name", "version", "description", + }).AddRow("obj-1", int64(123), created, updated, "file.txt", "v1", "desc")) + mock.ExpectQuery(regexp.QuoteMeta("SELECT name_alias FROM drs_object_name_alias WHERE object_id = $1 ORDER BY name_alias")). + WithArgs("obj-1"). + WillReturnRows(sqlmock.NewRows([]string{"name_alias"}). + AddRow("file-old.txt")) mock.ExpectQuery(regexp.QuoteMeta("SELECT url, type FROM drs_object_access_method WHERE object_id = $1")). WithArgs("obj-1"). @@ -145,6 +149,9 @@ func TestGetObject_DeduplicatesAndPropagatesAuthz(t *testing.T) { if obj.AccessMethods == nil || len(*obj.AccessMethods) != 2 { t.Fatalf("expected 2 deduplicated access methods, got %+v", obj.AccessMethods) } + if len(obj.NameAliases) != 1 || obj.NameAliases[0] != "file-old.txt" { + t.Fatalf("expected propagated name aliases, got %+v", obj.NameAliases) + } if len(obj.Checksums) != 2 { t.Fatalf("expected 2 deduplicated checksums, got %d", len(obj.Checksums)) } @@ -162,12 +169,15 @@ func TestGetObject_IgnoresAuthContext(t *testing.T) { now := time.Date(2026, time.March, 1, 10, 0, 0, 0, time.UTC) mock.ExpectQuery(regexp.QuoteMeta(` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = $1`)). WithArgs("obj-2"). WillReturnRows(sqlmock.NewRows([]string{ - "id", "size", "created_time", "updated_time", "name", "file_name", "version", "description", - }).AddRow("obj-2", int64(1), now, now, "n", "nested/n", "v", "d")) + "id", "size", "created_time", "updated_time", "name", "version", "description", + }).AddRow("obj-2", int64(1), now, now, "n", "v", "d")) + mock.ExpectQuery(regexp.QuoteMeta("SELECT name_alias FROM drs_object_name_alias WHERE object_id = $1 ORDER BY name_alias")). + WithArgs("obj-2"). + WillReturnRows(sqlmock.NewRows([]string{"name_alias"})) mock.ExpectQuery(regexp.QuoteMeta("SELECT url, type FROM drs_object_access_method WHERE object_id = $1")). WithArgs("obj-2"). WillReturnRows(sqlmock.NewRows([]string{"url", "type"}). @@ -191,41 +201,6 @@ func TestGetObject_IgnoresAuthContext(t *testing.T) { } } -func TestGetObject_LegacyNameFallsBackToFileName(t *testing.T) { - pg, mock, rawDB := newMockPostgresDB(t) - defer rawDB.Close() - - now := time.Date(2026, time.March, 1, 10, 0, 0, 0, time.UTC) - mock.ExpectQuery(regexp.QuoteMeta(` - SELECT id, size, created_time, updated_time, name, file_name, version, description - FROM drs_object WHERE id = $1`)). - WithArgs("obj-legacy"). - WillReturnRows(sqlmock.NewRows([]string{ - "id", "size", "created_time", "updated_time", "name", "file_name", "version", "description", - }).AddRow("obj-legacy", int64(1), now, now, "nested/dir/file.txt", nil, "v", "d")) - mock.ExpectQuery(regexp.QuoteMeta("SELECT url, type FROM drs_object_access_method WHERE object_id = $1")). - WithArgs("obj-legacy"). - WillReturnRows(sqlmock.NewRows([]string{"url", "type"}). - AddRow("s3://bucket/key", "s3")) - mock.ExpectQuery(regexp.QuoteMeta("SELECT resource FROM drs_object_controlled_access WHERE object_id = $1 ORDER BY resource")). - WithArgs("obj-legacy"). - WillReturnRows(sqlmock.NewRows([]string{"resource"})) - mock.ExpectQuery(regexp.QuoteMeta("SELECT type, checksum FROM drs_object_checksum WHERE object_id = $1")). - WithArgs("obj-legacy"). - WillReturnRows(sqlmock.NewRows([]string{"type", "checksum"})) - - obj, err := pg.GetObject(context.Background(), "obj-legacy") - if err != nil { - t.Fatalf("GetObject returned error: %v", err) - } - if got := common.StringVal(obj.Name); got != "file.txt" { - t.Fatalf("expected basename name, got %q", got) - } - if got, _ := obj.Properties["file_name"].(string); got != "nested/dir/file.txt" { - t.Fatalf("expected legacy path to populate file_name, got %#v", obj.Properties["file_name"]) - } -} - func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { pg, mock, rawDB := newMockPostgresDB(t) defer rawDB.Close() @@ -240,7 +215,6 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { o.created_time, o.updated_time, o.name, - o.file_name, o.version, o.description FROM drs_object o @@ -258,10 +232,10 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { )`)). WithArgs(pq.Array([]string{"obj-2", "obj-1", "obj-2"}), pq.Array([]string(nil))). WillReturnRows(sqlmock.NewRows([]string{ - "id", "size", "created_time", "updated_time", "name", "file_name", "version", "description", + "id", "size", "created_time", "updated_time", "name", "version", "description", }). - AddRow("obj-1", int64(10), created, updated, "file-1", "nested/file-1", "v1", "desc-1"). - AddRow("obj-2", int64(20), created, updated, "file-2", "nested/file-2", "v2", "desc-2")) + AddRow("obj-1", int64(10), created, updated, "file-1", "v1", "desc-1"). + AddRow("obj-2", int64(20), created, updated, "file-2", "v2", "desc-2")) mock.ExpectQuery(regexp.QuoteMeta(` SELECT object_id, url, type @@ -295,6 +269,16 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { AddRow("obj-1", "/organization/org/project/p1"). AddRow("obj-2", "/organization/org/project/p1")) + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT object_id, name_alias + FROM drs_object_name_alias + WHERE object_id = ANY($1) + ORDER BY object_id, name_alias`)). + WithArgs(pq.Array([]string{"obj-1", "obj-2"})). + WillReturnRows(sqlmock.NewRows([]string{"object_id", "name_alias"}). + AddRow("obj-1", "file-1-old"). + AddRow("obj-2", "file-2-old")) + objects, err := pg.GetBulkObjects(context.Background(), []string{"obj-2", "obj-1", "obj-2"}) if err != nil { t.Fatalf("GetBulkObjects returned error: %v", err) @@ -305,6 +289,9 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { if objects[0].AccessMethods == nil || len(*objects[0].AccessMethods) != 1 { t.Fatalf("expected one access method on obj-2, got %+v", objects[0].AccessMethods) } + if len(objects[0].NameAliases) != 1 || objects[0].NameAliases[0] != "file-2-old" { + t.Fatalf("expected propagated aliases on obj-2, got %+v", objects[0].NameAliases) + } if len(objects[1].Checksums) != 1 || objects[1].Checksums[0].Checksum != "aaa" { t.Fatalf("expected deduplicated checksum on obj-1, got %+v", objects[1].Checksums) } @@ -313,30 +300,34 @@ func TestGetBulkObjects_UsesSplitHydrationQueries(t *testing.T) { } } -func TestListObjectIDsPageByPath(t *testing.T) { +func TestListScopedObjectIDsByChecksums(t *testing.T) { pg, mock, rawDB := newMockPostgresDB(t) defer rawDB.Close() mock.ExpectQuery(regexp.QuoteMeta(` - SELECT object_id - FROM drs_object_browse_index - WHERE resource = $1 AND parent_path = $2 AND object_id > $3 ORDER BY object_id LIMIT $4 OFFSET $5`)). - WithArgs("/organization/org/project/proj", "nested", "obj-a", 10, 0). - WillReturnRows(sqlmock.NewRows([]string{"object_id"}).AddRow("obj-c")) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT normalized_path FROM drs_object_browse_index WHERE resource = $1 AND normalized_path LIKE $2 AND parent_path <> $3 ORDER BY normalized_path`)). - WithArgs("/organization/org/project/proj", "nested/%", "nested"). - WillReturnRows(sqlmock.NewRows([]string{"normalized_path"}).AddRow("nested/deep/b.txt")) - - ids, directories, err := pg.ListObjectIDsPageByPath(context.Background(), "org", "proj", "nested", "obj-a", 10, 0) + SELECT DISTINCT c.checksum, c.object_id + FROM drs_object_checksum c + INNER JOIN drs_object_controlled_access ca ON ca.object_id = c.object_id + WHERE ca.resource = $1 AND c.type = $2 AND c.checksum = ANY($3) + ORDER BY c.checksum, c.object_id`)). + WithArgs("/organization/org/project/p1", "sha256", pq.Array([]string{"sha-a", "sha-b", "missing"})). + WillReturnRows(sqlmock.NewRows([]string{"checksum", "object_id"}). + AddRow("sha-a", "obj-1"). + AddRow("sha-a", "obj-2"). + AddRow("sha-b", "obj-3")) + + res, err := pg.ListScopedObjectIDsByChecksums(context.Background(), "org", "p1", []string{"sha-a", "sha-a", "sha-b", "missing", ""}) if err != nil { - t.Fatalf("ListObjectIDsPageByPath returned error: %v", err) + t.Fatalf("ListScopedObjectIDsByChecksums returned error: %v", err) + } + if got := res["sha-a"]; len(got) != 2 || got[0] != "obj-1" || got[1] != "obj-2" { + t.Fatalf("unexpected ids for sha-a: %+v", got) } - if !slices.Equal(ids, []string{"obj-c"}) { - t.Fatalf("unexpected IDs: %v", ids) + if got := res["sha-b"]; len(got) != 1 || got[0] != "obj-3" { + t.Fatalf("unexpected ids for sha-b: %+v", got) } - if len(directories) != 1 || directories[0].Path != "nested/deep" { - t.Fatalf("unexpected directories: %+v", directories) + if got := res["missing"]; len(got) != 0 { + t.Fatalf("expected empty ids for missing, got %+v", got) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatalf("unmet expectations: %v", err) diff --git a/internal/db/postgres/postgres.go b/internal/db/postgres/postgres.go index c60a80d8..d53becf9 100644 --- a/internal/db/postgres/postgres.go +++ b/internal/db/postgres/postgres.go @@ -138,6 +138,23 @@ func (db *PostgresDB) ensureS3CredentialSchema() error { `ALTER TABLE s3_credential DROP CONSTRAINT IF EXISTS s3_credential_pkey`, `ALTER TABLE s3_credential ADD PRIMARY KEY (credential_id)`, `CREATE INDEX IF NOT EXISTS idx_s3_credential_bucket ON s3_credential(bucket)`, + `CREATE OR REPLACE FUNCTION enforce_s3_credential_unique_bucket() RETURNS trigger AS $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM s3_credential + WHERE bucket = NEW.bucket AND credential_id <> NEW.credential_id + ) THEN + RAISE EXCEPTION 'physical bucket "%" is already configured under another credential', NEW.bucket; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql`, + `DROP TRIGGER IF EXISTS s3_credential_unique_bucket_trigger ON s3_credential`, + `CREATE TRIGGER s3_credential_unique_bucket_trigger + BEFORE INSERT OR UPDATE OF bucket, credential_id ON s3_credential + FOR EACH ROW + EXECUTE FUNCTION enforce_s3_credential_unique_bucket()`, } for _, q := range queries { if _, err := db.db.Exec(q); err != nil { diff --git a/internal/db/postgres/postgres_schema_test.go b/internal/db/postgres/postgres_schema_test.go index 5469a874..738f0b49 100644 --- a/internal/db/postgres/postgres_schema_test.go +++ b/internal/db/postgres/postgres_schema_test.go @@ -14,12 +14,13 @@ func TestSchemaEnsurers(t *testing.T) { defer rawDB.Close() mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object").WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("ALTER TABLE drs_object ADD COLUMN IF NOT EXISTS file_name TEXT")).WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta("ALTER TABLE drs_object DROP COLUMN IF EXISTS file_name")).WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_access_method").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_controlled_access").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_checksum").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_alias").WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_browse_index").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE TABLE IF NOT EXISTS drs_object_name_alias").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("DROP TABLE IF EXISTS drs_object_browse_index").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_access_method_object_id_idx ON drs_object_access_method(object_id)")). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_checksum_object_id_idx ON drs_object_checksum(object_id)")). @@ -38,11 +39,7 @@ func TestSchemaEnsurers(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_alias_object_id_idx ON drs_object_alias(object_id)")). WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_object_id_idx ON drs_object_browse_index(resource, parent_path, object_id)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_entry_name_idx ON drs_object_browse_index(resource, parent_path, entry_name)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_normalized_path_idx ON drs_object_browse_index(resource, normalized_path)")). + mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS drs_object_name_alias_object_id_idx ON drs_object_name_alias(object_id)")). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectQuery("information_schema\\.columns"). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) @@ -102,6 +99,23 @@ func TestSchemaEnsurers(t *testing.T) { `ALTER TABLE s3_credential DROP CONSTRAINT IF EXISTS s3_credential_pkey`, `ALTER TABLE s3_credential ADD PRIMARY KEY (credential_id)`, `CREATE INDEX IF NOT EXISTS idx_s3_credential_bucket ON s3_credential(bucket)`, + `CREATE OR REPLACE FUNCTION enforce_s3_credential_unique_bucket() RETURNS trigger AS $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM s3_credential + WHERE bucket = NEW.bucket AND credential_id <> NEW.credential_id + ) THEN + RAISE EXCEPTION 'physical bucket "%" is already configured under another credential', NEW.bucket; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql`, + `DROP TRIGGER IF EXISTS s3_credential_unique_bucket_trigger ON s3_credential`, + `CREATE TRIGGER s3_credential_unique_bucket_trigger + BEFORE INSERT OR UPDATE OF bucket, credential_id ON s3_credential + FOR EACH ROW + EXECUTE FUNCTION enforce_s3_credential_unique_bucket()`, } { mock.ExpectExec(regexp.QuoteMeta(query)).WillReturnResult(sqlmock.NewResult(0, 0)) } diff --git a/internal/db/scripts/normalize_object_names_to_basename_postgres.sql b/internal/db/scripts/normalize_object_names_to_basename_postgres.sql new file mode 100644 index 00000000..de075253 --- /dev/null +++ b/internal/db/scripts/normalize_object_names_to_basename_postgres.sql @@ -0,0 +1,85 @@ +-- Normalize legacy drs_object.name values from full paths to basenames. +-- +-- Usage: +-- psql "$DATABASE_URL" -f internal/db/scripts/normalize_object_names_to_basename_postgres.sql +-- +-- The update only touches non-empty names whose slash/backslash-normalized +-- basename differs from the stored value. Access URLs, aliases, checksums, and +-- controlled_access rows are intentionally untouched. + +BEGIN; + +WITH candidates AS ( + SELECT + id, + name, + regexp_replace(replace(btrim(name), E'\\', '/'), '^.*/', '') AS basename + FROM drs_object + WHERE name IS NOT NULL + AND btrim(name) <> '' +), +changed AS ( + SELECT id, name, basename + FROM candidates + WHERE basename <> '' + AND name <> basename +) +SELECT COUNT(*) AS rows_to_update +FROM changed; + +WITH candidates AS ( + SELECT + id, + name, + regexp_replace(replace(btrim(name), E'\\', '/'), '^.*/', '') AS basename + FROM drs_object + WHERE name IS NOT NULL + AND btrim(name) <> '' +), +changed AS ( + SELECT id, name, basename + FROM candidates + WHERE basename <> '' + AND name <> basename +) +SELECT id, name AS old_name, basename AS new_name +FROM changed +ORDER BY id +LIMIT 20; + +WITH candidates AS ( + SELECT + id, + regexp_replace(replace(btrim(name), E'\\', '/'), '^.*/', '') AS basename + FROM drs_object + WHERE name IS NOT NULL + AND btrim(name) <> '' +), +changed AS ( + SELECT id, basename + FROM candidates + WHERE basename <> '' +) +UPDATE drs_object AS o +SET + name = changed.basename, + updated_time = now() +FROM changed +WHERE o.id = changed.id + AND o.name <> changed.basename; + +WITH candidates AS ( + SELECT + id, + name, + regexp_replace(replace(btrim(name), E'\\', '/'), '^.*/', '') AS basename + FROM drs_object + WHERE name IS NOT NULL + AND btrim(name) <> '' +) +SELECT COUNT(*) AS rows_still_path_like +FROM candidates +WHERE basename <> '' + AND name <> basename; + +COMMIT; diff --git a/internal/db/sqlite/browse_index.go b/internal/db/sqlite/browse_index.go deleted file mode 100644 index 32d6b622..00000000 --- a/internal/db/sqlite/browse_index.go +++ /dev/null @@ -1,142 +0,0 @@ -package sqlite - -import ( - "context" - "database/sql" - "sort" - "strings" - - sycommon "github.com/calypr/syfon/common" - icommon "github.com/calypr/syfon/internal/common" - "github.com/calypr/syfon/internal/models" -) - -type browseIndexRow struct { - ObjectID string - Resource string - NormalizedPath string - ParentPath string - EntryName string -} - -func sqliteBrowseRowsForObject(objectID, objectName string, resources []string) []browseIndexRow { - info, ok, err := icommon.BrowsePathInfoFromName(objectName) - if err != nil || !ok { - return nil - } - normalizedResources := sycommon.NormalizeAccessResources(resources) - rows := make([]browseIndexRow, 0, len(normalizedResources)) - for _, resource := range normalizedResources { - rows = append(rows, browseIndexRow{ - ObjectID: objectID, - Resource: resource, - NormalizedPath: info.Normalized, - ParentPath: info.ParentPath, - EntryName: info.EntryName, - }) - } - return rows -} - -func sqliteDeleteBrowseRowsByIDsTx(tx *sql.Tx, ids []string) error { - return execSQLiteDeleteByIDs(tx, "drs_object_browse_index", ids) -} - -func sqliteInsertBrowseRowsTx(tx *sql.Tx, rows []browseIndexRow) error { - if len(rows) == 0 { - return nil - } - args := make([]interface{}, 0, len(rows)*5) - for _, row := range rows { - args = append(args, row.ObjectID, row.Resource, row.NormalizedPath, row.ParentPath, row.EntryName) - } - return execSQLiteBulkInsert( - tx, - "INSERT INTO drs_object_browse_index (object_id, resource, normalized_path, parent_path, entry_name) VALUES ", - "(?, ?, ?, ?, ?)", - 5, - args, - " ON CONFLICT(resource, object_id) DO UPDATE SET normalized_path=excluded.normalized_path, parent_path=excluded.parent_path, entry_name=excluded.entry_name", - ) -} - -func sqliteRebuildBrowseRowsForObjectTx(ctx context.Context, tx *sql.Tx, objectID, objectName string, resources []string) error { - if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = ?`, objectID); err != nil { - return err - } - return sqliteInsertBrowseRowsTx(tx, sqliteBrowseRowsForObject(objectID, objectName, resources)) -} - -func (db *SqliteDB) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { - resource, err := sycommon.ResourcePath(organization, project) - if err != nil { - return nil, nil, err - } - normalizedPath, _, err := icommon.NormalizeBrowsePath(path) - if err != nil { - return nil, nil, err - } - if offset < 0 { - offset = 0 - } - - args := []any{resource, normalizedPath} - query := ` - SELECT object_id - FROM drs_object_browse_index - WHERE resource = ? AND parent_path = ?` - if strings.TrimSpace(startAfter) != "" { - args = append(args, strings.TrimSpace(startAfter)) - query += " AND object_id > ?" - } - args = append(args, limit, offset) - query += " ORDER BY object_id LIMIT ? OFFSET ?" - rows, err := db.db.QueryContext(ctx, query, args...) - if err != nil { - return nil, nil, err - } - defer rows.Close() - ids, err := scanObjectIDs(rows) - if err != nil { - return nil, nil, err - } - - dirQuery := `SELECT normalized_path FROM drs_object_browse_index WHERE resource = ? AND parent_path <> '' ORDER BY normalized_path` - dirArgs := []any{resource} - if normalizedPath != "" { - dirQuery = `SELECT normalized_path FROM drs_object_browse_index WHERE resource = ? AND normalized_path LIKE ? AND parent_path <> ? ORDER BY normalized_path` - dirArgs = append(dirArgs, normalizedPath+"/%", normalizedPath) - } - dirRows, err := db.db.QueryContext(ctx, dirQuery, dirArgs...) - if err != nil { - return nil, nil, err - } - defer dirRows.Close() - - directoriesByPath := make(map[string]models.BrowseDirectory) - for dirRows.Next() { - var candidatePath string - if err := dirRows.Scan(&candidatePath); err != nil { - return nil, nil, err - } - dirInfo, ok := icommon.ImmediateBrowseDirectory(normalizedPath, candidatePath) - if !ok { - continue - } - directoriesByPath[dirInfo.Normalized] = models.BrowseDirectory{Name: dirInfo.EntryName, Path: dirInfo.Normalized} - } - if err := dirRows.Err(); err != nil { - return nil, nil, err - } - directories := make([]models.BrowseDirectory, 0, len(directoriesByPath)) - for _, directory := range directoriesByPath { - directories = append(directories, directory) - } - sort.Slice(directories, func(i, j int) bool { - if directories[i].Name == directories[j].Name { - return directories[i].Path < directories[j].Path - } - return directories[i].Name < directories[j].Name - }) - return ids, directories, nil -} diff --git a/internal/db/sqlite/credentials.go b/internal/db/sqlite/credentials.go index 457e8060..ce550bcb 100644 --- a/internal/db/sqlite/credentials.go +++ b/internal/db/sqlite/credentials.go @@ -91,6 +91,10 @@ func (db *SqliteDB) SaveS3Credential(ctx context.Context, cred *models.S3Credent common.AuditS3CredentialAccess(ctx, "write", bucket, wrapped) return wrapped } + if err := db.ensureUniquePhysicalBucket(ctx, stored.CredentialID, stored.Bucket); err != nil { + common.AuditS3CredentialAccess(ctx, "write", stored.Bucket, err) + return err + } // SQLite UPSERT syntax: INSERT INTO ... ON CONFLICT (...) DO UPDATE SET ... _, err = db.db.ExecContext(ctx, ` @@ -114,6 +118,29 @@ func (db *SqliteDB) SaveS3Credential(ctx context.Context, cred *models.S3Credent return nil } +func (db *SqliteDB) ensureUniquePhysicalBucket(ctx context.Context, credentialID, bucket string) error { + credentialID = strings.TrimSpace(credentialID) + bucket = strings.TrimSpace(bucket) + if bucket == "" { + return nil + } + + var existingCredentialID string + err := db.db.QueryRowContext(ctx, ` + SELECT credential_id + FROM s3_credential + WHERE bucket = ? AND credential_id <> ? + LIMIT 1 + `, bucket, credentialID).Scan(&existingCredentialID) + if err == nil { + return fmt.Errorf("physical bucket %q is already configured under credential %q; reuse that credential and add a bucket scope instead", bucket, existingCredentialID) + } + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return fmt.Errorf("failed to validate physical bucket uniqueness: %w", err) +} + func (db *SqliteDB) DeleteS3Credential(ctx context.Context, credentialID string) error { resolvedID, err := db.resolveCredentialID(ctx, credentialID) if err != nil { diff --git a/internal/db/sqlite/metrics.go b/internal/db/sqlite/metrics.go index b331a43f..ff4b0af4 100644 --- a/internal/db/sqlite/metrics.go +++ b/internal/db/sqlite/metrics.go @@ -462,6 +462,27 @@ func (db *SqliteDB) GetFileUsageSummaryByResources(ctx context.Context, resource return db.getScopedFileUsageSummary(ctx, resources, includeUnscoped, inactiveSince) } +func (db *SqliteDB) GetProjectRecordSummaryByScope(ctx context.Context, organization, project string) (models.FileUsageSummary, error) { + resource, err := sycommon.ResourcePath(strings.TrimSpace(organization), strings.TrimSpace(project)) + if err != nil { + return models.FileUsageSummary{}, err + } + var summary models.FileUsageSummary + var latest any + if err := db.db.QueryRowContext(ctx, ` + SELECT COUNT(DISTINCT o.id), MAX(o.updated_time) + FROM drs_object o + INNER JOIN drs_object_controlled_access ca ON ca.object_id = o.id + WHERE ca.resource = ?`, resource).Scan(&summary.RecordCount, &latest); err != nil { + return models.FileUsageSummary{}, err + } + if parsed, ok := parseSQLiteTransferTime(latest); ok { + t := parsed.UTC() + summary.RecordLatestUpdatedTime = &t + } + return summary, nil +} + func scanFileUsageRows(rows *sql.Rows, capacity int) ([]models.FileUsage, error) { out := make([]models.FileUsage, 0, capacity) for rows.Next() { @@ -677,13 +698,6 @@ func nullableInt64(v *int64) any { return *v } -func nullableTime(v *time.Time) any { - if v == nil { - return nil - } - return v.UTC() -} - func (db *SqliteDB) backfillAccessGrants(ctx context.Context) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -1073,77 +1087,6 @@ func sqliteTransferResourceClause(resources []string) (string, []any) { return strings.Join(clauses, " OR "), args } -func providerTransferWhere(filter models.TransferAttributionFilter) (string, []any) { - parts := make([]string, 0) - args := make([]any, 0) - add := func(clause string, value any) { - parts = append(parts, clause) - args = append(args, value) - } - status := strings.TrimSpace(filter.ReconciliationStatus) - if status == "" { - status = models.ProviderTransferMatched - } - if status != "all" { - add("reconciliation_status = ?", status) - } - if strings.TrimSpace(filter.Organization) != "" { - add("organization = ?", strings.TrimSpace(filter.Organization)) - } - if strings.TrimSpace(filter.Project) != "" { - add("project = ?", strings.TrimSpace(filter.Project)) - } - direction := strings.TrimSpace(filter.Direction) - if direction == "" { - switch strings.TrimSpace(filter.EventType) { - case models.ProviderTransferDirectionDownload: - direction = models.ProviderTransferDirectionDownload - case models.ProviderTransferDirectionUpload: - direction = models.ProviderTransferDirectionUpload - } - } - if direction != "" && direction != "all" { - add("direction = ?", direction) - } - if filter.From != nil { - add("event_time >= ?", filter.From.UTC()) - } - if filter.To != nil { - add("event_time <= ?", filter.To.UTC()) - } - if strings.TrimSpace(filter.Provider) != "" { - add("provider = ?", strings.TrimSpace(filter.Provider)) - } - if strings.TrimSpace(filter.Bucket) != "" { - add("bucket = ?", strings.TrimSpace(filter.Bucket)) - } - if strings.TrimSpace(filter.SHA256) != "" { - add("sha256 = ?", strings.TrimSpace(filter.SHA256)) - } - if strings.TrimSpace(filter.User) != "" { - user := strings.TrimSpace(filter.User) - parts = append(parts, "(actor_email = ? OR actor_subject = ?)") - args = append(args, user, user) - } - if len(parts) == 0 { - return "", args - } - return " WHERE " + strings.Join(parts, " AND "), args -} - -func providerTransferGroupExpr(groupBy string) (string, string) { - switch strings.ToLower(strings.TrimSpace(groupBy)) { - case "user": - return "COALESCE(NULLIF(actor_email, ''), actor_subject), actor_email, actor_subject", "COALESCE(NULLIF(actor_email, ''), actor_subject) AS key, '' AS organization, '' AS project, '' AS provider, '' AS bucket, '' AS sha256, actor_email, actor_subject" - case "provider": - return "provider, bucket", "provider || ':' || bucket AS key, '' AS organization, '' AS project, provider, bucket, '' AS sha256, '' AS actor_email, '' AS actor_subject" - case "object": - return "sha256", "sha256 AS key, '' AS organization, '' AS project, '' AS provider, '' AS bucket, sha256, '' AS actor_email, '' AS actor_subject" - default: - return "organization, project", "organization || '/' || project AS key, organization, project, '' AS provider, '' AS bucket, '' AS sha256, '' AS actor_email, '' AS actor_subject" - } -} - func transferAttributionGroupExpr(groupBy string) (string, string) { switch strings.ToLower(strings.TrimSpace(groupBy)) { case "user": diff --git a/internal/db/sqlite/objects.go b/internal/db/sqlite/objects.go index f92fa9c9..1a1750c6 100644 --- a/internal/db/sqlite/objects.go +++ b/internal/db/sqlite/objects.go @@ -5,7 +5,6 @@ import ( "database/sql" "errors" "fmt" - "path" "sort" "strings" "time" @@ -16,34 +15,6 @@ import ( "github.com/calypr/syfon/internal/models" ) -func sqliteStoredFileName(obj *models.InternalObject) string { - if obj != nil && obj.Properties != nil { - if raw, ok := obj.Properties["file_name"].(string); ok && strings.TrimSpace(raw) != "" { - return strings.TrimSpace(raw) - } - } - if obj == nil { - return "" - } - return common.StringVal(obj.Name) -} - -func sqliteLoadedNames(name, fileName sql.NullString) (string, string) { - loadedName := strings.TrimSpace(name.String) - loadedFileName := strings.TrimSpace(fileName.String) - if loadedFileName == "" { - loadedFileName = loadedName - loadedName = "" - } - if loadedName == "" && loadedFileName != "" { - loadedName = path.Base(strings.Trim(loadedFileName, "/")) - if loadedName == "." || loadedName == "/" || loadedName == "" { - loadedName = loadedFileName - } - } - return loadedName, loadedFileName -} - func (db *SqliteDB) DeleteObject(ctx context.Context, id string) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -140,11 +111,11 @@ func (db *SqliteDB) GetObject(ctx context.Context, id string) (*models.InternalO retryLookup: // 1. Fetch main record var r models.DrsObjectRecord - var name, fileName, version, description sql.NullString + var name, version, description sql.NullString err := db.db.QueryRowContext(ctx, ` - SELECT id, size, created_time, updated_time, name, file_name, version, description + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object WHERE id = ?`, lookupID).Scan( - &r.ID, &r.Size, &r.CreatedTime, &r.UpdatedTime, &name, &fileName, &version, &description, + &r.ID, &r.Size, &r.CreatedTime, &r.UpdatedTime, &name, &version, &description, ) if err == sql.ErrNoRows { if !resolvedAlias { @@ -163,9 +134,13 @@ retryLookup: if err != nil { return nil, fmt.Errorf("failed to fetch record: %w", err) } - r.Name, r.FileName = sqliteLoadedNames(name, fileName) + r.Name = strings.TrimSpace(name.String) r.Version = version.String r.Description = description.String + nameAliases, err := db.nameAliasesForObject(ctx, lookupID) + if err != nil { + return nil, err + } objectID := r.ID if resolvedAlias && requestID != "" { objectID = requestID @@ -182,10 +157,8 @@ retryLookup: Name: common.Ptr(r.Name), SelfUri: "drs://" + objectID, }, - Properties: map[string]interface{}{}, - } - if strings.TrimSpace(r.FileName) != "" { - obj.Properties["file_name"] = r.FileName + NameAliases: nameAliases, + Properties: map[string]interface{}{}, } // 2. Fetch storage access methods. @@ -260,9 +233,9 @@ func (db *SqliteDB) CreateObject(ctx context.Context, obj *models.InternalObject // Insert main record _, err = tx.ExecContext(ctx, ` - INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), sqliteStoredFileName(obj), common.StringVal(obj.Version), common.StringVal(obj.Description), + INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.CleanToBasename(common.StringVal(obj.Name)), common.StringVal(obj.Version), common.StringVal(obj.Description), ) if err != nil { return fmt.Errorf("failed to insert drs_object: %w", err) @@ -271,8 +244,11 @@ func (db *SqliteDB) CreateObject(ctx context.Context, obj *models.InternalObject if err := insertControlledAccessTx(ctx, tx, obj.Id, objectAccessResources(obj)); err != nil { return err } - if err := sqliteRebuildBrowseRowsForObjectTx(ctx, tx, obj.Id, sqliteStoredFileName(obj), objectAccessResources(obj)); err != nil { - return fmt.Errorf("failed to update browse index: %w", err) + for _, alias := range normalizeObjectNameAliases(obj) { + _, err = tx.ExecContext(ctx, `INSERT INTO drs_object_name_alias (object_id, name_alias) VALUES (?, ?)`, obj.Id, alias) + if err != nil { + return fmt.Errorf("failed to insert name alias: %w", err) + } } // Insert storage access methods. @@ -327,17 +303,19 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern accessArgs := make([]interface{}, 0) controlledArgs := make([]interface{}, 0) checksumArgs := make([]interface{}, 0) - browseRows := make([]browseIndexRow, 0) + nameAliasArgs := make([]interface{}, 0) for _, obj := range objects { ids = append(ids, obj.Id) - mainArgs = append(mainArgs, obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.StringVal(obj.Name), sqliteStoredFileName(&obj), common.StringVal(obj.Version), common.StringVal(obj.Description)) + mainArgs = append(mainArgs, obj.Id, obj.Size, obj.CreatedTime, common.TimeVal(obj.UpdatedTime), common.CleanToBasename(common.StringVal(obj.Name)), common.StringVal(obj.Version), common.StringVal(obj.Description)) seenAccess := make(map[string]struct{}) for _, resource := range objectAccessResources(&obj) { controlledArgs = append(controlledArgs, obj.Id, resource) } - browseRows = append(browseRows, sqliteBrowseRowsForObject(obj.Id, sqliteStoredFileName(&obj), objectAccessResources(&obj))...) + for _, alias := range normalizeObjectNameAliases(&obj) { + nameAliasArgs = append(nameAliasArgs, obj.Id, alias) + } if obj.AccessMethods != nil { for _, am := range *obj.AccessMethods { if am.AccessUrl == nil || am.AccessUrl.Url == "" { @@ -363,16 +341,15 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern } } - mainPrefix := `INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) VALUES ` + mainPrefix := `INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) VALUES ` mainSuffix := ` ON CONFLICT(id) DO UPDATE SET size=excluded.size, created_time=excluded.created_time, updated_time=excluded.updated_time, name=excluded.name, - file_name=excluded.file_name, version=excluded.version, description=excluded.description` - if err := execSQLiteBulkInsert(tx, mainPrefix, "(?, ?, ?, ?, ?, ?, ?, ?)", 8, mainArgs, mainSuffix); err != nil { + if err := execSQLiteBulkInsert(tx, mainPrefix, "(?, ?, ?, ?, ?, ?, ?)", 7, mainArgs, mainSuffix); err != nil { return fmt.Errorf("failed bulk upsert drs_object: %w", err) } @@ -385,8 +362,8 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern if err := execSQLiteDeleteByIDs(tx, "drs_object_checksum", ids); err != nil { return fmt.Errorf("failed bulk clear checksums: %w", err) } - if err := sqliteDeleteBrowseRowsByIDsTx(tx, ids); err != nil { - return fmt.Errorf("failed bulk clear browse index: %w", err) + if err := execSQLiteDeleteByIDs(tx, "drs_object_name_alias", ids); err != nil { + return fmt.Errorf("failed bulk clear name aliases: %w", err) } if len(accessArgs) > 0 { @@ -413,6 +390,18 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern return fmt.Errorf("failed bulk insert controlled access: %w", err) } } + if len(nameAliasArgs) > 0 { + if err := execSQLiteBulkInsert( + tx, + "INSERT INTO drs_object_name_alias (object_id, name_alias) VALUES ", + "(?, ?)", + 2, + nameAliasArgs, + "", + ); err != nil { + return fmt.Errorf("failed bulk insert name aliases: %w", err) + } + } if len(checksumArgs) > 0 { if err := execSQLiteBulkInsert( tx, @@ -425,9 +414,6 @@ func (db *SqliteDB) RegisterObjects(ctx context.Context, objects []models.Intern return fmt.Errorf("failed bulk insert checksums: %w", err) } } - if err := sqliteInsertBrowseRowsTx(tx, browseRows); err != nil { - return fmt.Errorf("failed bulk insert browse index: %w", err) - } if err := db.flushObjectUsageEventsForIDsTx(ctx, tx, ids); err != nil { return fmt.Errorf("failed to apply object usage events: %w", err) } @@ -489,6 +475,87 @@ func (db *SqliteDB) GetObjectsByChecksums(ctx context.Context, checksums []strin return result, nil } +func (db *SqliteDB) ListScopedObjectIDsByChecksums(ctx context.Context, organization, project string, checksums []string) (map[string][]string, error) { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" || project == "" || len(checksums) == 0 { + return map[string][]string{}, nil + } + resource, err := sycommon.ResourcePath(organization, project) + if err != nil { + return nil, err + } + normalized := uniqueNonEmptyStrings(checksums) + if len(normalized) == 0 { + return map[string][]string{}, nil + } + // Keep the checksum predicate below within SQLite's bound-parameter limit. + // The two fixed parameters are the project resource and checksum type. + maxChecksums := sqliteMaxParams - 2 + if len(normalized) > maxChecksums { + out := make(map[string][]string, len(normalized)) + for start := 0; start < len(normalized); start += maxChecksums { + end := start + maxChecksums + if end > len(normalized) { + end = len(normalized) + } + part, err := db.ListScopedObjectIDsByChecksums(ctx, organization, project, normalized[start:end]) + if err != nil { + return nil, err + } + for checksum, ids := range part { + out[checksum] = append(out[checksum], ids...) + } + } + return out, nil + } + args := make([]any, 0, len(normalized)+2) + args = append(args, resource, "sha256") + placeholders := makePlaceholders(len(normalized)) + for _, checksum := range normalized { + args = append(args, checksum) + } + rows, err := db.db.QueryContext(ctx, fmt.Sprintf(` + SELECT DISTINCT c.checksum, c.object_id + FROM drs_object_checksum c + INNER JOIN drs_object_controlled_access ca ON ca.object_id = c.object_id + WHERE ca.resource = ? AND c.type = ? AND c.checksum IN (%s) + ORDER BY c.checksum, c.object_id`, placeholders), args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := make(map[string][]string, len(normalized)) + for _, checksum := range normalized { + out[checksum] = []string{} + } + seen := make(map[string]map[string]struct{}, len(normalized)) + for rows.Next() { + var checksum string + var objectID string + if err := rows.Scan(&checksum, &objectID); err != nil { + return nil, err + } + checksum = strings.TrimSpace(checksum) + objectID = strings.TrimSpace(objectID) + if checksum == "" || objectID == "" { + continue + } + if seen[checksum] == nil { + seen[checksum] = make(map[string]struct{}) + } + if _, ok := seen[checksum][objectID]; ok { + continue + } + seen[checksum][objectID] = struct{}{} + out[checksum] = append(out[checksum], objectID) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func (db *SqliteDB) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) { organization = strings.TrimSpace(organization) project = strings.TrimSpace(project) @@ -1163,7 +1230,6 @@ func (db *SqliteDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []stri o.created_time, o.updated_time, o.name, - o.file_name, o.version, o.description FROM drs_object o @@ -1179,35 +1245,30 @@ func (db *SqliteDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []stri for rows.Next() { var ( - id string - name, fileName sql.NullString - version, description sql.NullString - size int64 - createdTime, updatedTime time.Time + id string + name sql.NullString + version, description sql.NullString + size int64 + createdTime, updatedTime time.Time ) if err := rows.Scan( - &id, &size, &createdTime, &updatedTime, &name, &fileName, &version, &description, + &id, &size, &createdTime, &updatedTime, &name, &version, &description, ); err != nil { return nil, err } - - loadedName, loadedFileName := sqliteLoadedNames(name, fileName) objectsByID[id] = &models.InternalObject{ DrsObject: drs.DrsObject{ Id: id, Size: size, CreatedTime: createdTime, UpdatedTime: common.Ptr(updatedTime), - Name: common.Ptr(loadedName), + Name: common.Ptr(strings.TrimSpace(name.String)), Version: common.Ptr(version.String), Description: common.Ptr(description.String), SelfUri: "drs://" + id, }, Properties: map[string]interface{}{}, } - if strings.TrimSpace(loadedFileName) != "" { - objectsByID[id].Properties["file_name"] = loadedFileName - } } if err := rows.Err(); err != nil { @@ -1225,6 +1286,9 @@ func (db *SqliteDB) fetchObjectsByIDsOrChecksums(ctx context.Context, ids []stri if err := db.attachControlledAccess(ctx, objectsByID); err != nil { return nil, err } + if err := db.attachNameAliases(ctx, objectsByID); err != nil { + return nil, err + } return objectsByID, nil } @@ -1330,6 +1394,13 @@ func objectAccessResources(obj *models.InternalObject) []string { return sycommon.AuthzMapToList(obj.Authorizations) } +func normalizeObjectNameAliases(obj *models.InternalObject) []string { + if obj == nil { + return nil + } + return common.NormalizeNameAliases(common.StringVal(obj.Name), obj.NameAliases) +} + func sortedObjectIDs(objectsByID map[string]*models.InternalObject) []string { ids := make([]string, 0, len(objectsByID)) for id := range objectsByID { @@ -1339,6 +1410,23 @@ func sortedObjectIDs(objectsByID map[string]*models.InternalObject) []string { return ids } +func uniqueNonEmptyStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + normalized := strings.TrimSpace(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + func insertControlledAccessTx(ctx context.Context, tx *sql.Tx, objectID string, resources []string) error { for _, resource := range sycommon.NormalizeAccessResources(resources) { if _, err := tx.ExecContext(ctx, `INSERT INTO drs_object_controlled_access (object_id, resource) VALUES (?, ?)`, objectID, resource); err != nil { @@ -1368,6 +1456,27 @@ func (db *SqliteDB) controlledAccessForObject(ctx context.Context, objectID stri return sycommon.NormalizeAccessResources(resources), nil } +func (db *SqliteDB) nameAliasesForObject(ctx context.Context, objectID string) ([]string, error) { + rows, err := db.db.QueryContext(ctx, `SELECT name_alias FROM drs_object_name_alias WHERE object_id = ? ORDER BY name_alias`, objectID) + if err != nil { + return nil, err + } + defer rows.Close() + + aliases := make([]string, 0) + for rows.Next() { + var alias string + if err := rows.Scan(&alias); err != nil { + return nil, err + } + aliases = append(aliases, alias) + } + if err := rows.Err(); err != nil { + return nil, err + } + return common.NormalizeNameAliases("", aliases), nil +} + func (db *SqliteDB) attachControlledAccess(ctx context.Context, objectsByID map[string]*models.InternalObject) error { if len(objectsByID) == 0 { return nil @@ -1415,6 +1524,48 @@ func (db *SqliteDB) attachControlledAccess(ctx context.Context, objectsByID map[ return nil } +func (db *SqliteDB) attachNameAliases(ctx context.Context, objectsByID map[string]*models.InternalObject) error { + if len(objectsByID) == 0 { + return nil + } + ids := sortedObjectIDs(objectsByID) + query := fmt.Sprintf(` + SELECT object_id, name_alias + FROM drs_object_name_alias + WHERE object_id IN (%s) + ORDER BY object_id, name_alias`, makePlaceholders(len(ids))) + args := make([]any, 0, len(ids)) + for _, id := range ids { + args = append(args, id) + } + rows, err := db.db.QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("failed to fetch bulk object name aliases: %w", err) + } + defer rows.Close() + + byObject := make(map[string][]string, len(objectsByID)) + for rows.Next() { + var objectID, alias string + if err := rows.Scan(&objectID, &alias); err != nil { + return err + } + byObject[objectID] = append(byObject[objectID], alias) + } + if err := rows.Err(); err != nil { + return err + } + + for objectID, aliases := range byObject { + obj := objectsByID[objectID] + if obj == nil { + continue + } + obj.NameAliases = common.NormalizeNameAliases(common.StringVal(obj.Name), aliases) + } + return nil +} + func (db *SqliteDB) UpdateObjectAccessMethods(ctx context.Context, objectID string, accessMethods []drs.AccessMethod) error { tx, err := db.db.BeginTx(ctx, nil) if err != nil { @@ -1463,9 +1614,6 @@ func (db *SqliteDB) RemoveObjectControlledAccess(ctx context.Context, objectID, if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_controlled_access WHERE object_id = ? AND resource = ?`, objectID, resource); err != nil { return err } - if _, err := tx.ExecContext(ctx, `DELETE FROM drs_object_browse_index WHERE object_id = ? AND resource = ?`, objectID, resource); err != nil { - return err - } return tx.Commit() } diff --git a/internal/db/sqlite/sqlite.go b/internal/db/sqlite/sqlite.go index 87422192..8c0c135e 100644 --- a/internal/db/sqlite/sqlite.go +++ b/internal/db/sqlite/sqlite.go @@ -60,7 +60,6 @@ func (db *SqliteDB) initSchema() error { created_time TIMESTAMP, updated_time TIMESTAMP, name TEXT, - file_name TEXT, version TEXT, description TEXT )`, @@ -95,18 +94,13 @@ func (db *SqliteDB) initSchema() error { FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE )`, `CREATE INDEX IF NOT EXISTS idx_drs_object_alias_object_id ON drs_object_alias(object_id)`, - `CREATE TABLE IF NOT EXISTS drs_object_browse_index ( + `CREATE TABLE IF NOT EXISTS drs_object_name_alias ( object_id TEXT NOT NULL, - resource TEXT NOT NULL, - normalized_path TEXT NOT NULL, - parent_path TEXT NOT NULL, - entry_name TEXT NOT NULL, - PRIMARY KEY (resource, object_id), + name_alias TEXT NOT NULL, + PRIMARY KEY (object_id, name_alias), FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE )`, - `CREATE INDEX IF NOT EXISTS idx_drs_object_browse_index_resource_parent_object_id ON drs_object_browse_index(resource, parent_path, object_id)`, - `CREATE INDEX IF NOT EXISTS idx_drs_object_browse_index_resource_parent_entry_name ON drs_object_browse_index(resource, parent_path, entry_name)`, - `CREATE INDEX IF NOT EXISTS idx_drs_object_browse_index_resource_normalized_path ON drs_object_browse_index(resource, normalized_path)`, + `CREATE INDEX IF NOT EXISTS idx_drs_object_name_alias_object_id ON drs_object_name_alias(object_id)`, `CREATE TABLE IF NOT EXISTS s3_credential ( credential_id TEXT PRIMARY KEY, bucket TEXT NOT NULL, @@ -116,6 +110,28 @@ func (db *SqliteDB) initSchema() error { secret_key TEXT, endpoint TEXT )`, + `CREATE TRIGGER IF NOT EXISTS s3_credential_unique_bucket_insert + BEFORE INSERT ON s3_credential + FOR EACH ROW + WHEN EXISTS ( + SELECT 1 + FROM s3_credential + WHERE bucket = NEW.bucket AND credential_id <> NEW.credential_id + ) + BEGIN + SELECT RAISE(ABORT, 'physical bucket is already configured under another credential'); + END`, + `CREATE TRIGGER IF NOT EXISTS s3_credential_unique_bucket_update + BEFORE UPDATE OF bucket, credential_id ON s3_credential + FOR EACH ROW + WHEN EXISTS ( + SELECT 1 + FROM s3_credential + WHERE bucket = NEW.bucket AND credential_id <> NEW.credential_id + ) + BEGIN + SELECT RAISE(ABORT, 'physical bucket is already configured under another credential'); + END`, `CREATE TABLE IF NOT EXISTS bucket_scope ( organization TEXT NOT NULL, project_id TEXT NOT NULL, @@ -260,10 +276,11 @@ func (db *SqliteDB) initSchema() error { if err := db.ensureCredentialIdentitySchema(); err != nil { return err } - if _, err := db.db.Exec(`ALTER TABLE drs_object ADD COLUMN file_name TEXT NOT NULL DEFAULT ''`); err != nil { - if !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") { - return err - } + if err := db.ensureObjectTableShape(); err != nil { + return err + } + if _, err := db.db.Exec(`DROP TABLE IF EXISTS drs_object_browse_index`); err != nil { + return err } if _, err := db.db.Exec(`ALTER TABLE transfer_attribution_event ADD COLUMN access_grant_id TEXT NOT NULL DEFAULT ''`); err != nil { if !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") { @@ -284,6 +301,62 @@ func (db *SqliteDB) initSchema() error { return nil } +func (db *SqliteDB) ensureObjectTableShape() error { + rows, err := db.db.Query(`PRAGMA table_info(drs_object)`) + if err != nil { + return err + } + defer rows.Close() + + hasFileName := false + for rows.Next() { + var cid int + var name, typ string + var notNull, pk int + var dflt any + if err := rows.Scan(&cid, &name, &typ, ¬Null, &dflt, &pk); err != nil { + return err + } + if strings.EqualFold(strings.TrimSpace(name), "file_name") { + hasFileName = true + } + } + if err := rows.Err(); err != nil { + return err + } + if !hasFileName { + return nil + } + + tx, err := db.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + for _, stmt := range []string{ + `CREATE TABLE drs_object_new ( + id TEXT PRIMARY KEY, + size INTEGER, + created_time TIMESTAMP, + updated_time TIMESTAMP, + name TEXT, + version TEXT, + description TEXT + )`, + `INSERT INTO drs_object_new (id, size, created_time, updated_time, name, version, description) + SELECT id, size, created_time, updated_time, name, version, description FROM drs_object`, + `DROP TABLE drs_object`, + `ALTER TABLE drs_object_new RENAME TO drs_object`, + } { + if _, err := tx.Exec(stmt); err != nil { + return err + } + } + + return tx.Commit() +} + func (db *SqliteDB) ensureCredentialIdentitySchema() error { rows, err := db.db.Query(`PRAGMA table_info(s3_credential)`) if err != nil { diff --git a/internal/db/sqlite/sqlite_test.go b/internal/db/sqlite/sqlite_test.go index f1f62876..51f4b6f7 100644 --- a/internal/db/sqlite/sqlite_test.go +++ b/internal/db/sqlite/sqlite_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "path/filepath" "slices" "strings" @@ -243,7 +244,7 @@ func TestSqliteDB_GetObjectsByChecksum_WhenIDDiffers(t *testing.T) { } } -func TestSqliteDB_GetObject_LegacyNameFallsBackToFileName(t *testing.T) { +func TestSqliteDB_GetObjectPreservesStoredName(t *testing.T) { ctx := context.Background() db, err := NewSqliteDB(":memory:") if err != nil { @@ -252,11 +253,11 @@ func TestSqliteDB_GetObject_LegacyNameFallsBackToFileName(t *testing.T) { now := time.Now().UTC() if _, err := db.db.ExecContext(ctx, ` - INSERT INTO drs_object (id, size, created_time, updated_time, name, file_name, version, description) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - "legacy-1", int64(42), now, now, "nested/dir/file.txt", nil, "v1", "desc", + INSERT INTO drs_object (id, size, created_time, updated_time, name, version, description) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + "legacy-1", int64(42), now, now, "file.txt", "v1", "desc", ); err != nil { - t.Fatalf("insert legacy object: %v", err) + t.Fatalf("insert object: %v", err) } if _, err := db.db.ExecContext(ctx, ` INSERT INTO drs_object_access_method (object_id, url, type) @@ -271,10 +272,97 @@ func TestSqliteDB_GetObject_LegacyNameFallsBackToFileName(t *testing.T) { t.Fatalf("GetObject failed: %v", err) } if got := common.StringVal(obj.Name); got != "file.txt" { - t.Fatalf("expected basename name, got %q", got) + t.Fatalf("expected stored name, got %q", got) } - if got, _ := obj.Properties["file_name"].(string); got != "nested/dir/file.txt" { - t.Fatalf("expected legacy path to populate file_name, got %#v", obj.Properties["file_name"]) +} + +func TestSqliteDB_NormalizeNameToBasenameOnInsert(t *testing.T) { + ctx := context.Background() + db, err := NewSqliteDB(":memory:") + if err != nil { + t.Fatalf("failed to create db: %v", err) + } + + // 1. Test CreateObject with Unix and Windows paths + now := time.Now().UTC() + objUnix := &models.InternalObject{ + DrsObject: drs.DrsObject{ + Id: "unix-1", + Size: 100, + CreatedTime: now, + Name: common.Ptr("/path/to/some/unix_file.txt"), + }, + } + if err := db.CreateObject(ctx, objUnix); err != nil { + t.Fatalf("CreateObject failed: %v", err) + } + + gotUnix, err := db.GetObject(ctx, "unix-1") + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + if got := common.StringVal(gotUnix.Name); got != "unix_file.txt" { + t.Fatalf("expected name to be normalized to unix_file.txt, got %q", got) + } + + objWin := &models.InternalObject{ + DrsObject: drs.DrsObject{ + Id: "win-1", + Size: 200, + CreatedTime: now, + Name: common.Ptr(`C:\Windows\System32\win_file.txt`), + }, + } + if err := db.CreateObject(ctx, objWin); err != nil { + t.Fatalf("CreateObject failed: %v", err) + } + + gotWin, err := db.GetObject(ctx, "win-1") + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + if got := common.StringVal(gotWin.Name); got != "win_file.txt" { + t.Fatalf("expected name to be normalized to win_file.txt, got %q", got) + } + + // 2. Test RegisterObjects with paths + bulkObjs := []models.InternalObject{ + { + DrsObject: drs.DrsObject{ + Id: "bulk-unix", + Size: 300, + CreatedTime: now, + Name: common.Ptr("/var/log/syslog.log"), + }, + }, + { + DrsObject: drs.DrsObject{ + Id: "bulk-win", + Size: 400, + CreatedTime: now, + Name: common.Ptr(`D:\Data\config.json`), + }, + }, + } + + if err := db.RegisterObjects(ctx, bulkObjs); err != nil { + t.Fatalf("RegisterObjects failed: %v", err) + } + + gotBulkUnix, err := db.GetObject(ctx, "bulk-unix") + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + if got := common.StringVal(gotBulkUnix.Name); got != "syslog.log" { + t.Fatalf("expected name syslog.log, got %q", got) + } + + gotBulkWin, err := db.GetObject(ctx, "bulk-win") + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + if got := common.StringVal(gotBulkWin.Name); got != "config.json" { + t.Fatalf("expected name config.json, got %q", got) } } @@ -504,34 +592,116 @@ func TestSqliteDB_S3Credentials(t *testing.T) { } } -func TestSqliteDB_S3CredentialsCredentialIDAllowsSharedPhysicalBucket(t *testing.T) { +func TestSqliteDB_SaveS3CredentialRejectsDuplicatePhysicalBucket(t *testing.T) { + t.Setenv(crypto.CredentialMasterKeyEnv, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") + ctx := context.Background() + db, err := NewSqliteDB(":memory:") + if err != nil { + t.Fatalf("failed to create db: %v", err) + } + + first := models.S3Credential{ + CredentialID: "org-a/default", + Bucket: "shared-bucket", + Region: "us-east-1", + AccessKey: "key-a", + SecretKey: "secret-a", + } + if err := db.SaveS3Credential(ctx, &first); err != nil { + t.Fatalf("SaveS3Credential(first) failed: %v", err) + } + + second := models.S3Credential{ + CredentialID: "org-b/default", + Bucket: "shared-bucket", + Region: "us-east-1", + AccessKey: "key-b", + SecretKey: "secret-b", + } + err = db.SaveS3Credential(ctx, &second) + if err == nil || !strings.Contains(err.Error(), `physical bucket "shared-bucket" is already configured under credential "org-a/default"`) { + t.Fatalf("expected duplicate physical bucket error, got %v", err) + } +} + +func TestSqliteDB_GetS3CredentialRejectsAmbiguousLegacyPhysicalBucket(t *testing.T) { t.Setenv(crypto.CredentialMasterKeyEnv, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") ctx := context.Background() db, err := NewSqliteDB(":memory:") if err != nil { t.Fatalf("failed to create db: %v", err) } + if _, err := db.db.ExecContext(ctx, `DROP TRIGGER IF EXISTS s3_credential_unique_bucket_insert`); err != nil { + t.Fatalf("drop insert trigger: %v", err) + } + if _, err := db.db.ExecContext(ctx, `DROP TRIGGER IF EXISTS s3_credential_unique_bucket_update`); err != nil { + t.Fatalf("drop update trigger: %v", err) + } for _, cred := range []models.S3Credential{ {CredentialID: "org-a/default", Bucket: "shared-bucket", Region: "us-east-1", AccessKey: "key-a", SecretKey: "secret-a"}, {CredentialID: "org-b/default", Bucket: "shared-bucket", Region: "us-east-1", AccessKey: "key-b", SecretKey: "secret-b"}, } { - cred := cred - if err := db.SaveS3Credential(ctx, &cred); err != nil { - t.Fatalf("SaveS3Credential(%s) failed: %v", cred.CredentialID, err) + stored, err := crypto.PrepareS3CredentialForStorage(&cred) + if err != nil { + t.Fatalf("PrepareS3CredentialForStorage(%s) failed: %v", cred.CredentialID, err) + } + if _, err := db.db.ExecContext(ctx, ` + INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, stored.CredentialID, stored.Bucket, "s3", stored.Region, stored.AccessKey, stored.SecretKey, stored.Endpoint); err != nil { + t.Fatalf("legacy insert(%s) failed: %v", cred.CredentialID, err) } } - got, err := db.GetS3Credential(ctx, "org-b/default") - if err != nil { - t.Fatalf("GetS3Credential by credential_id failed: %v", err) + if _, err := db.GetS3Credential(ctx, "shared-bucket"); err == nil || !strings.Contains(err.Error(), "multiple credentials") { + t.Fatalf("expected ambiguous physical bucket lookup error, got %v", err) } - if got.Bucket != "shared-bucket" || got.AccessKey != "key-b" { - t.Fatalf("unexpected credential: %+v", got) +} + +func TestSqliteDB_DirectInsertRejectsDuplicatePhysicalBucket(t *testing.T) { + t.Setenv(crypto.CredentialMasterKeyEnv, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") + ctx := context.Background() + db, err := NewSqliteDB(":memory:") + if err != nil { + t.Fatalf("failed to create db: %v", err) } - if _, err := db.GetS3Credential(ctx, "shared-bucket"); err == nil || !strings.Contains(err.Error(), "multiple credentials") { - t.Fatalf("expected ambiguous physical bucket lookup error, got %v", err) + first, err := crypto.PrepareS3CredentialForStorage(&models.S3Credential{ + CredentialID: "org-a/default", + Bucket: "shared-bucket", + Provider: "s3", + Region: "us-east-1", + AccessKey: "key-a", + SecretKey: "secret-a", + }) + if err != nil { + t.Fatalf("PrepareS3CredentialForStorage(first) failed: %v", err) + } + if _, err := db.db.ExecContext(ctx, ` + INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, first.CredentialID, first.Bucket, "s3", first.Region, first.AccessKey, first.SecretKey, first.Endpoint); err != nil { + t.Fatalf("raw first insert failed: %v", err) + } + + second, err := crypto.PrepareS3CredentialForStorage(&models.S3Credential{ + CredentialID: "org-b/default", + Bucket: "shared-bucket", + Provider: "s3", + Region: "us-east-1", + AccessKey: "key-b", + SecretKey: "secret-b", + }) + if err != nil { + t.Fatalf("PrepareS3CredentialForStorage(second) failed: %v", err) + } + _, err = db.db.ExecContext(ctx, ` + INSERT INTO s3_credential (credential_id, bucket, provider, region, access_key, secret_key, endpoint) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, second.CredentialID, second.Bucket, "s3", second.Region, second.AccessKey, second.SecretKey, second.Endpoint) + if err == nil || !strings.Contains(err.Error(), "physical bucket is already configured under another credential") { + t.Fatalf("expected trigger rejection, got %v", err) } } @@ -755,6 +925,96 @@ func TestSqliteDB_GetObjectsByChecksumsAndListByPrefix(t *testing.T) { } } +func TestSqliteDB_ListScopedObjectIDsByChecksums(t *testing.T) { + ctx := context.Background() + db, _ := NewSqliteDB(":memory:") + now := time.Now() + objects := []models.InternalObject{ + { + DrsObject: drs.DrsObject{ + Id: "proj-a-1", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-a"}}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "proj-a-2", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-a"}}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "other-project", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-a"}}, + }, + Authorizations: map[string][]string{"org": {"p2"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "other-org", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-a"}}, + }, + Authorizations: map[string][]string{"other": {"p1"}}, + }, + { + DrsObject: drs.DrsObject{ + Id: "proj-b", + CreatedTime: now, + UpdatedTime: &now, + Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-b"}}, + }, + Authorizations: map[string][]string{"org": {"p1"}}, + }, + } + if err := db.RegisterObjects(ctx, objects); err != nil { + t.Fatalf("RegisterObjects failed: %v", err) + } + + res, err := db.ListScopedObjectIDsByChecksums(ctx, "org", "p1", []string{"sha-a", "sha-a", "sha-b", "missing", ""}) + if err != nil { + t.Fatalf("ListScopedObjectIDsByChecksums failed: %v", err) + } + if got := res["sha-a"]; len(got) != 2 || got[0] != "proj-a-1" || got[1] != "proj-a-2" { + t.Fatalf("unexpected scoped ids for sha-a: %+v", got) + } + if got := res["sha-b"]; len(got) != 1 || got[0] != "proj-b" { + t.Fatalf("unexpected scoped ids for sha-b: %+v", got) + } + if got := res["missing"]; len(got) != 0 { + t.Fatalf("expected empty ids for missing checksum, got %+v", got) + } + emptyRes, err := db.ListScopedObjectIDsByChecksums(ctx, "org", "p1", nil) + if err != nil { + t.Fatalf("ListScopedObjectIDsByChecksums empty failed: %v", err) + } + if len(emptyRes) != 0 { + t.Fatalf("expected empty map for empty checksum input, got %+v", emptyRes) + } + + large := make([]string, sqliteMaxParams) + for i := range large { + large[i] = fmt.Sprintf("large-%d", i) + } + large[len(large)-1] = "sha-b" + largeRes, err := db.ListScopedObjectIDsByChecksums(ctx, "org", "p1", large) + if err != nil { + t.Fatalf("ListScopedObjectIDsByChecksums large input failed: %v", err) + } + if got := largeRes["sha-b"]; len(got) != 1 || got[0] != "proj-b" { + t.Fatalf("unexpected large-input result for sha-b: %+v", got) + } +} + func TestSqliteDB_ListObjectIDsByScopeRootIncludesUnscoped(t *testing.T) { ctx := context.Background() db, _ := NewSqliteDB(":memory:") @@ -1282,6 +1542,17 @@ func TestSqliteDB_ScopedFileUsageQueries(t *testing.T) { if summary.TotalFiles != 2 || summary.TotalUploads != 1 || summary.TotalDownloads != 1 || summary.InactiveFileCount != 0 { t.Fatalf("unexpected scoped summary: %+v", summary) } + + recordSummary, err := db.GetProjectRecordSummaryByScope(ctx, "org", "p1") + if err != nil { + t.Fatalf("GetProjectRecordSummaryByScope failed: %v", err) + } + if recordSummary.RecordCount != 2 { + t.Fatalf("unexpected scoped record count: %+v", recordSummary) + } + if recordSummary.RecordLatestUpdatedTime == nil || !recordSummary.RecordLatestUpdatedTime.Equal(now) { + t.Fatalf("unexpected scoped record latest updated time: %+v", recordSummary.RecordLatestUpdatedTime) + } } func TestSqliteDB_TransferAttributionByResources(t *testing.T) { @@ -1786,54 +2057,3 @@ func TestSqliteDB_GetPendingLFSMeta(t *testing.T) { t.Fatalf("expected ErrNotFound for missing pending metadata, got: %v", err) } } - -func TestSqliteDB_ListObjectIDsPageByPath(t *testing.T) { - ctx := context.Background() - db, err := NewSqliteDB(":memory:") - if err != nil { - t.Fatalf("failed to create db: %v", err) - } - - now := time.Now().UTC() - for _, obj := range []models.InternalObject{ - { - DrsObject: drs.DrsObject{ - Id: "obj-a", - Name: common.Ptr("nested/a.txt"), - CreatedTime: now, - }, - Authorizations: map[string][]string{"org": {"proj"}}, - }, - { - DrsObject: drs.DrsObject{ - Id: "obj-b", - Name: common.Ptr("nested/deep/b.txt"), - CreatedTime: now, - }, - Authorizations: map[string][]string{"org": {"proj"}}, - }, - { - DrsObject: drs.DrsObject{ - Id: "obj-c", - Name: common.Ptr("nested/z.txt"), - CreatedTime: now, - }, - Authorizations: map[string][]string{"org": {"proj"}}, - }, - } { - if err := db.CreateObject(ctx, &obj); err != nil { - t.Fatalf("CreateObject failed: %v", err) - } - } - - ids, directories, err := db.ListObjectIDsPageByPath(ctx, "org", "proj", "nested", "obj-a", 10, 0) - if err != nil { - t.Fatalf("ListObjectIDsPageByPath failed: %v", err) - } - if !slices.Equal(ids, []string{"obj-c"}) { - t.Fatalf("unexpected IDs: %v", ids) - } - if len(directories) != 1 || directories[0].Path != "nested/deep" { - t.Fatalf("unexpected directories: %+v", directories) - } -} diff --git a/internal/models/object.go b/internal/models/object.go index 19ecd034..ea76288c 100644 --- a/internal/models/object.go +++ b/internal/models/object.go @@ -2,16 +2,20 @@ package models import ( "encoding/json" + "fmt" + "sort" + "strings" "time" "github.com/calypr/syfon/apigen/server/drs" - "github.com/calypr/syfon/common" + syfoncommon "github.com/calypr/syfon/common" ) // InternalObject is the primary DRS domain model. It wraps the GA4GH DrsObject // and adds Syfon-specific authorization metadata. type InternalObject struct { drs.DrsObject + NameAliases []string `json:"name_aliases,omitempty"` Authorizations map[string][]string `json:"-"` Properties map[string]interface{} `json:"-"` } @@ -30,13 +34,19 @@ func (o *InternalObject) UnmarshalJSON(data []byte) error { if raw == nil { raw = map[string]interface{}{} } + if _, ok := raw["file_name"]; ok { + return fmt.Errorf("file_name is no longer supported") + } + if _, ok := raw["path"]; ok { + return fmt.Errorf("path is no longer supported") + } o.Properties = raw type wireObject struct { drs.DrsObject - Did string `json:"did,omitempty"` - Hashes map[string]string `json:"hashes,omitempty"` - FileName *string `json:"file_name,omitempty"` + Did string `json:"did,omitempty"` + Hashes map[string]string `json:"hashes,omitempty"` + NameAliases []string `json:"name_aliases,omitempty"` } var wire wireObject @@ -65,13 +75,14 @@ func (o *InternalObject) UnmarshalJSON(data []byte) error { o.UpdatedTime = wire.UpdatedTime o.Size = wire.Size o.Name = wire.Name + o.NameAliases = normalizeNameAliases(stringValue(wire.Name), wire.NameAliases) o.Description = wire.Description o.MimeType = wire.MimeType o.SelfUri = wire.SelfUri o.Version = wire.Version if wire.ControlledAccess != nil { - o.Authorizations = common.ControlledAccessToAuthzMap(*wire.ControlledAccess) + o.Authorizations = syfoncommon.ControlledAccessToAuthzMap(*wire.ControlledAccess) } return nil @@ -109,6 +120,9 @@ func (o InternalObject) MarshalJSON() ([]byte, error) { if o.Name != nil { out["name"] = *o.Name } + if len(o.NameAliases) > 0 { + out["name_aliases"] = normalizeNameAliases(stringValue(o.Name), o.NameAliases) + } if o.Description != nil { out["description"] = *o.Description } @@ -117,12 +131,6 @@ func (o InternalObject) MarshalJSON() ([]byte, error) { } // Ensure Gen3 compatibility fields are also present. out["did"] = o.Id - if fileName, ok := out["file_name"].(string); ok && fileName != "" { - out["file_name"] = fileName - } else if o.Name != nil { - out["file_name"] = *o.Name - } - if len(o.Checksums) > 0 { hashes := make(map[string]string, len(o.Checksums)) for _, c := range o.Checksums { @@ -148,6 +156,46 @@ func isRetiredInternalAuthField(key string) bool { } } +func stringValue(v *string) string { + if v == nil { + return "" + } + return *v +} + +func normalizeNameAliases(primary string, aliases []string) []string { + primary = cleanToBasename(primary) + seen := make(map[string]struct{}, len(aliases)+1) + out := make([]string, 0, len(aliases)) + for _, alias := range aliases { + name := cleanToBasename(alias) + if name == "" || name == primary { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + out = append(out, name) + } + sort.Strings(out) + return out +} + +func cleanToBasename(name string) string { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return "" + } + trimmed = strings.ReplaceAll(trimmed, "\\", "/") + parts := strings.Split(trimmed, "/") + base := parts[len(parts)-1] + if base == "" { + return trimmed + } + return base +} + func (o InternalObject) External() drs.DrsObject { return o.DrsObject } diff --git a/internal/models/object_test.go b/internal/models/object_test.go index 5b355260..ac5dd428 100644 --- a/internal/models/object_test.go +++ b/internal/models/object_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func TestInternalObjectUnmarshalLegacyAndMarshalCompatibility(t *testing.T) { +func TestInternalObjectRejectsLegacyPathAliases(t *testing.T) { raw := []byte(`{ "did":"did-1", "file_name":"legacy-name.txt", @@ -14,41 +14,8 @@ func TestInternalObjectUnmarshalLegacyAndMarshalCompatibility(t *testing.T) { "unknown_field":"keep-me" }`) var obj InternalObject - if err := json.Unmarshal(raw, &obj); err != nil { - t.Fatalf("unmarshal failed: %v", err) - } - if obj.Id != "did-1" { - t.Fatalf("expected id from did, got %q", obj.Id) - } - if obj.Name != nil { - t.Fatalf("expected name to remain unset when only file_name is provided, got %+v", obj.Name) - } - if len(obj.Checksums) != 1 || obj.Checksums[0].Type != "sha256" || obj.Checksums[0].Checksum != "abc" { - t.Fatalf("unexpected checksums: %+v", obj.Checksums) - } - - encoded, err := json.Marshal(obj) - if err != nil { - t.Fatalf("marshal failed: %v", err) - } - var out map[string]interface{} - if err := json.Unmarshal(encoded, &out); err != nil { - t.Fatalf("decode marshaled payload: %v", err) - } - if out["did"] != "did-1" { - t.Fatalf("expected did in output, got %v", out["did"]) - } - if out["file_name"] != "legacy-name.txt" { - t.Fatalf("expected file_name in output, got %v", out["file_name"]) - } - if _, ok := out["name"]; ok { - t.Fatalf("did not expect name in output when only file_name was provided") - } - if _, ok := out["unknown_field"]; !ok { - t.Fatalf("expected unknown field preservation in output") - } - if _, retired := out["urls"]; retired { - t.Fatalf("expected retired auth fields removed from output") + if err := json.Unmarshal(raw, &obj); err == nil { + t.Fatal("expected file_name payload to be rejected") } } diff --git a/internal/models/storage.go b/internal/models/storage.go index 6d4b3e94..e1be2f64 100644 --- a/internal/models/storage.go +++ b/internal/models/storage.go @@ -48,10 +48,12 @@ type FileUsage struct { // FileUsageSummary aggregates transfer activity for a set of objects. type FileUsageSummary struct { - TotalFiles int64 - TotalUploads int64 - TotalDownloads int64 - InactiveFileCount int64 + TotalFiles int64 + TotalUploads int64 + TotalDownloads int64 + InactiveFileCount int64 + RecordCount int64 + RecordLatestUpdatedTime *time.Time } // BucketVisibilityRow is the minimum storage projection needed to build bucket @@ -62,11 +64,6 @@ type BucketVisibilityRow struct { Resource string } -type BrowseDirectory struct { - Name string - Path string -} - const ( TransferEventAccessIssued = "access_issued" @@ -214,12 +211,13 @@ type TransferAttributionBreakdown struct { // DrsObjectRecord mirrors the subset of drs_object columns returned by storage queries. type DrsObjectRecord struct { - ID string - Size int64 - CreatedTime time.Time - UpdatedTime time.Time - Name string - FileName string - Version string - Description string + ID string + Size int64 + CreatedTime time.Time + UpdatedTime time.Time + DownloadCount int64 + LastDownloadTime *time.Time + Name string + Version string + Description string } diff --git a/internal/repair/service.go b/internal/repair/service.go new file mode 100644 index 00000000..f4ecdd5b --- /dev/null +++ b/internal/repair/service.go @@ -0,0 +1,543 @@ +package repair + +import ( + "context" + "fmt" + "net/http" + "net/url" + "sort" + "strings" + + drsapi "github.com/calypr/syfon/apigen/client/drs" + "github.com/calypr/syfon/apigen/client/internalapi" + "github.com/calypr/syfon/client/request" + syfoncommon "github.com/calypr/syfon/common" + intcommon "github.com/calypr/syfon/internal/common" +) + +const defaultPageSize = 500 + +func (s *Service) Audit(ctx context.Context, opts Options) (Report, error) { + state, err := s.audit(ctx, opts) + if err != nil { + return Report{}, err + } + return state.report, nil +} + +func (s *Service) Apply(ctx context.Context, opts Options) (ApplyResult, error) { + opts.Organization = strings.TrimSpace(opts.Organization) + opts.Project = strings.TrimSpace(opts.Project) + if opts.Organization == "" || opts.Project == "" { + return ApplyResult{}, fmt.Errorf("apply requires --organization and --project") + } + + state, err := s.audit(ctx, opts) + if err != nil { + return ApplyResult{}, err + } + + result := ApplyResult{Report: state.report} + for _, obj := range state.objects { + if obj.updated == nil { + continue + } + result.AutoFixable++ + if _, err := s.index.Update(ctx, obj.record.Did, *obj.updated); err != nil { + result.Skipped++ + continue + } + result.Mutated++ + } + return result, nil +} + +func (s *Service) audit(ctx context.Context, opts Options) (*auditState, error) { + scopes, err := s.loadScopeTargets(ctx) + if err != nil { + return nil, err + } + records, scanned, err := s.listRecords(ctx, opts) + if err != nil { + return nil, err + } + + state := &auditState{ + report: Report{ + Organization: strings.TrimSpace(opts.Organization), + Project: strings.TrimSpace(opts.Project), + Scanned: scanned, + }, + } + for _, rec := range records { + obj, include := s.auditRecord(ctx, rec, scopes, opts) + if !include { + continue + } + state.objects = append(state.objects, obj) + } + s.addDuplicateFindings(state.objects) + sort.Slice(state.objects, func(i, j int) bool { return state.objects[i].record.Did < state.objects[j].record.Did }) + for _, obj := range state.objects { + if len(obj.findings) == 0 { + continue + } + state.report.Objects = append(state.report.Objects, ObjectReport{ + ObjectID: obj.record.Did, + SHA256: obj.sha256, + Organization: obj.scope.Organization, + Project: obj.scope.Project, + CurrentAccessURLs: append([]string(nil), obj.currentURLs...), + ProposedCanonicalURL: obj.canonicalURL, + AutoFixable: obj.updated != nil, + Findings: append([]Finding(nil), obj.findings...), + }) + } + return state, nil +} + +func (s *Service) listRecords(ctx context.Context, opts Options) ([]internalapi.InternalRecord, int, error) { + pageSize := opts.PageSize + if pageSize <= 0 { + pageSize = defaultPageSize + } + var out []internalapi.InternalRecord + start := "" + scanned := 0 + for { + limit := pageSize + if opts.Limit > 0 && opts.Limit-scanned < limit { + limit = opts.Limit - scanned + } + if limit <= 0 && opts.Limit > 0 { + break + } + resp, err := s.index.List(ctx, ListRecordsOptions{ + Limit: limit, + Start: start, + Organization: opts.Organization, + Project: opts.Project, + }) + if err != nil { + return nil, scanned, err + } + if resp.Records == nil || len(*resp.Records) == 0 { + break + } + records := *resp.Records + out = append(out, records...) + scanned += len(records) + start = strings.TrimSpace(records[len(records)-1].Did) + if len(records) < limit || start == "" { + break + } + } + return out, scanned, nil +} + +func (s *Service) auditRecord(ctx context.Context, rec internalapi.InternalRecord, scopes map[string][]scopeTarget, opts Options) (*auditedObject, bool) { + sha := "" + if rec.Hashes != nil { + sha = syfoncommon.NormalizeChecksum((*rec.Hashes)["sha256"]) + } + obj := &auditedObject{ + record: rec, + sha256: sha, + currentURLs: accessMethodURLs(rec.AccessMethods), + } + + resource, known, ambiguous := inferRecordResource(rec, sha, scopes) + obj.scopeKnown = known + obj.scopeAmbiguous = ambiguous + obj.inferredScope = resource + if known { + targets := scopes[resource] + if len(targets) == 0 { + obj.scopeKnown = false + } else { + obj.scope = targets[0] + obj.canonicalURL = canonicalAccessURL(obj.scope, rec.Did, sha) + } + } + + targetResource := "" + if strings.TrimSpace(opts.Organization) != "" && strings.TrimSpace(opts.Project) != "" { + targetResource, _ = syfoncommon.ResourcePath(opts.Organization, opts.Project) + } + if targetResource != "" && !recordMatchesResource(rec, targetResource) && obj.inferredScope != targetResource { + return obj, false + } + + if targetResource != "" && !recordMatchesResource(rec, targetResource) && obj.inferredScope == targetResource && sha != "" { + obj.findings = append(obj.findings, newFinding(FindingMissingControlledAccess, SeverityWarn, rec, sha, obj.currentURLs, obj.canonicalURL, true, "missing controlled_access row recoverable from deterministic scope")) + updated := cloneRecord(rec) + updated.ControlledAccess = addControlledAccess(updated.ControlledAccess, targetResource) + obj.updated = &updated + } + if obj.scopeKnown && obj.canonicalURL != "" { + s.classifyAccessMethods(ctx, obj, opts.CheckStorage) + } + if opts.CheckStorage { + s.addStorageFindings(ctx, obj) + } + return obj, true +} + +func (s *Service) classifyAccessMethods(ctx context.Context, obj *auditedObject, checkStorage bool) { + if obj.record.AccessMethods == nil { + return + } + methods := cloneAccessMethods(*obj.record.AccessMethods) + + // Construct path-style URL + pathStyleURL := pathStyleAccessURL(obj.scope, intcommon.StringVal(obj.record.Name)) + + // Determine correct target URL based on storage checks (if checkStorage is enabled) + targetURL := obj.canonicalURL + if checkStorage { + canonicalExists := s.checkURLExists(ctx, obj, obj.canonicalURL) + pathStyleExists := s.checkURLExists(ctx, obj, pathStyleURL) + if !canonicalExists && pathStyleExists { + targetURL = pathStyleURL + } + } + + // Check if the record already has the target URL + hasTarget := false + for _, raw := range obj.currentURLs { + if raw == targetURL { + hasTarget = true + break + } + } + + changed := false + for i := range methods { + raw := accessMethodURL(methods[i]) + if raw == "" || raw == targetURL { + continue + } + + // Any URL that does not match the target URL is considered legacy/incorrect. + if hasTarget { + obj.findings = append(obj.findings, newFinding(FindingLegacyAccessURLRemovable, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, targetURL, true, fmt.Sprintf("redundant URL %q has target sibling %q", raw, targetURL))) + methods[i].AccessUrl = nil + changed = true + } else { + obj.findings = append(obj.findings, newFinding(FindingLegacyAccessURLRewritable, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, targetURL, true, fmt.Sprintf("URL %q can be rewritten to target URL %q", raw, targetURL))) + setAccessMethodURL(&methods[i], targetURL) + changed = true + hasTarget = true + } + } + + if !changed { + return + } + filtered := make([]drsapi.AccessMethod, 0, len(methods)) + for _, method := range methods { + if accessMethodURL(method) == "" { + continue + } + filtered = append(filtered, method) + } + updated := cloneRecord(obj.record) + updated.AccessMethods = &filtered + if obj.updated != nil && obj.updated.ControlledAccess != nil { + updated.ControlledAccess = obj.updated.ControlledAccess + } + obj.updated = &updated +} + +func (s *Service) addStorageFindings(ctx context.Context, obj *auditedObject) { + for _, raw := range obj.currentURLs { + req := storageInspectRequest{ + Organization: obj.scope.Organization, + Project: obj.scope.Project, + ObjectURL: raw, + } + var resp storageInspectResponse + err := s.requestor.Do(ctx, http.MethodPost, "/data/inspect", req, &resp) + if err == nil { + continue + } + kind := FindingStorageProbeError + severity := SeverityWarn + message := err.Error() + if respErr, ok := err.(*request.ResponseError); ok && respErr.Status == http.StatusNotFound { + kind = FindingStorageObjectMissing + severity = SeverityError + message = "storage object not found" + } + obj.findings = append(obj.findings, newFinding(kind, severity, obj.record, obj.sha256, []string{raw}, obj.canonicalURL, false, message)) + } +} + +func (s *Service) loadScopeTargets(ctx context.Context) (map[string][]scopeTarget, error) { + buckets, err := s.buckets.List(ctx) + if err != nil { + return nil, fmt.Errorf("list buckets: %w", err) + } + out := map[string][]scopeTarget{} + for bucket := range buckets.S3BUCKETS { + scopes, err := s.buckets.ListScopes(ctx, bucket) + if err != nil { + return nil, fmt.Errorf("list bucket scopes for %s: %w", bucket, err) + } + for _, scope := range scopes { + resource, err := syfoncommon.ResourcePath(strings.TrimSpace(scope.Organization), strings.TrimSpace(scope.ProjectId)) + if err != nil || resource == "" { + continue + } + target := scopeTarget{ + Resource: resource, + Organization: strings.TrimSpace(scope.Organization), + Project: strings.TrimSpace(scope.ProjectId), + Bucket: bucket, + } + if scope.Path != nil { + parsed, err := url.Parse(strings.TrimSpace(*scope.Path)) + if err == nil { + if host := strings.TrimSpace(parsed.Host); host != "" { + target.Bucket = host + } + target.Prefix = strings.Trim(strings.TrimSpace(parsed.Path), "/") + } + } + out[resource] = append(out[resource], target) + } + } + return out, nil +} + +func (s *Service) addDuplicateFindings(objects []*auditedObject) { + byKey := map[string][]*auditedObject{} + for _, obj := range objects { + if obj.sha256 == "" { + continue + } + for _, resource := range recordProjectResources(obj.record, obj.inferredScope) { + byKey[resource+"|"+obj.sha256] = append(byKey[resource+"|"+obj.sha256], obj) + } + } + for key, group := range byKey { + if len(group) < 2 { + continue + } + resource := strings.SplitN(key, "|", 2)[0] + scope := intcommon.ParseResourcePath(resource) + for _, obj := range group { + obj.findings = append(obj.findings, newFinding(FindingDuplicateSHA256Sibling, SeverityWarn, obj.record, obj.sha256, obj.currentURLs, obj.canonicalURL, false, "same sha256 appears in multiple DIDs for this scope")) + if obj.scope.Organization == "" { + obj.scope.Organization = scope.Organization + obj.scope.Project = scope.Project + } + } + } +} + +func inferRecordResource(rec internalapi.InternalRecord, sha string, scopes map[string][]scopeTarget) (string, bool, bool) { + projectResources := recordProjectResources(rec, "") + if len(projectResources) == 1 { + resource := projectResources[0] + if len(scopes[resource]) == 0 { + return resource, false, false + } + return resource, true, false + } + if len(projectResources) > 1 { + return "", false, true + } + if strings.TrimSpace(sha) == "" { + return "", false, false + } + matches := []string{} + for resource, targets := range scopes { + if len(targets) == 0 || strings.TrimSpace(targets[0].Project) == "" { + continue + } + minted, err := intcommon.MintObjectIDFromChecksum(sha, []string{resource}) + if err != nil { + continue + } + if minted == strings.TrimSpace(rec.Did) { + matches = append(matches, resource) + } + } + if len(matches) == 1 { + return matches[0], true, false + } + if len(matches) > 1 { + return "", false, true + } + return "", false, false +} + +func recordProjectResources(rec internalapi.InternalRecord, inferred string) []string { + seen := map[string]struct{}{} + var out []string + if rec.ControlledAccess != nil { + for _, resource := range syfoncommon.NormalizeAccessResources(*rec.ControlledAccess) { + org, project, ok := syfoncommon.ResourceScope(resource) + if !ok || org == "" || project == "" { + continue + } + if _, exists := seen[resource]; exists { + continue + } + seen[resource] = struct{}{} + out = append(out, resource) + } + } + if inferred != "" { + if _, exists := seen[inferred]; !exists { + out = append(out, inferred) + } + } + sort.Strings(out) + return out +} + +func recordMatchesResource(rec internalapi.InternalRecord, resource string) bool { + if rec.ControlledAccess == nil { + return false + } + for _, candidate := range syfoncommon.NormalizeAccessResources(*rec.ControlledAccess) { + if candidate == resource { + return true + } + } + return false +} + +func accessMethodURLs(methods *[]drsapi.AccessMethod) []string { + if methods == nil { + return nil + } + var out []string + for _, method := range *methods { + if raw := accessMethodURL(method); raw != "" { + out = append(out, raw) + } + } + return out +} + +func accessMethodURL(method drsapi.AccessMethod) string { + if method.AccessUrl == nil { + return "" + } + return strings.TrimSpace(method.AccessUrl.Url) +} + +func setAccessMethodURL(method *drsapi.AccessMethod, raw string) { + if method.AccessUrl == nil { + method.AccessUrl = &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{} + } + method.AccessUrl.Url = raw +} + +func cloneAccessMethods(in []drsapi.AccessMethod) []drsapi.AccessMethod { + out := make([]drsapi.AccessMethod, len(in)) + copy(out, in) + return out +} + +func cloneRecord(rec internalapi.InternalRecord) internalapi.InternalRecord { + out := rec + if rec.AccessMethods != nil { + methods := cloneAccessMethods(*rec.AccessMethods) + out.AccessMethods = &methods + } + if rec.ControlledAccess != nil { + controlled := append([]string(nil), (*rec.ControlledAccess)...) + out.ControlledAccess = &controlled + } + if rec.Hashes != nil { + hashes := make(internalapi.HashInfo, len(*rec.Hashes)) + for k, v := range *rec.Hashes { + hashes[k] = v + } + out.Hashes = &hashes + } + return out +} + +func addControlledAccess(controlled *[]string, resource string) *[]string { + resources := append([]string(nil), syfoncommon.NormalizeAccessResources(derefStrings(controlled))...) + resources = append(resources, resource) + normalized := syfoncommon.NormalizeAccessResources(resources) + return &normalized +} + +func derefStrings(values *[]string) []string { + if values == nil { + return nil + } + return *values +} + +func canonicalAccessURL(target scopeTarget, did, sha string) string { + if strings.TrimSpace(target.Bucket) == "" || strings.TrimSpace(did) == "" || strings.TrimSpace(sha) == "" { + return "" + } + parts := []string{} + if prefix := strings.Trim(target.Prefix, "/"); prefix != "" { + parts = append(parts, prefix) + } + parts = append(parts, strings.TrimSpace(did), strings.TrimSpace(sha)) + return "s3://" + strings.TrimSpace(target.Bucket) + "/" + strings.Join(parts, "/") +} + +func pathStyleAccessURL(target scopeTarget, name string) string { + if strings.TrimSpace(target.Bucket) == "" || strings.TrimSpace(name) == "" { + return "" + } + parts := []string{} + if prefix := strings.Trim(target.Prefix, "/"); prefix != "" { + parts = append(parts, prefix) + } + parts = append(parts, strings.Trim(name, "/")) + return "s3://" + strings.TrimSpace(target.Bucket) + "/" + strings.Join(parts, "/") +} + +func (s *Service) checkURLExists(ctx context.Context, obj *auditedObject, url string) bool { + if s.requestor == nil { + return false + } + req := storageInspectRequest{ + Organization: obj.scope.Organization, + Project: obj.scope.Project, + ObjectURL: url, + } + var resp storageInspectResponse + err := s.requestor.Do(ctx, http.MethodPost, "/data/inspect", req, &resp) + return err == nil +} + +func newFinding(kind FindingKind, severity Severity, rec internalapi.InternalRecord, sha string, currentURLs []string, canonical string, autoFixable bool, msg string) Finding { + finding := Finding{ + Kind: kind, + Severity: severity, + ObjectID: rec.Did, + SHA256: sha, + CurrentAccessURLs: append([]string(nil), currentURLs...), + ProposedCanonicalURL: canonical, + AutoFixable: autoFixable, + Message: msg, + } + if rec.ControlledAccess != nil { + for _, resource := range syfoncommon.NormalizeAccessResources(*rec.ControlledAccess) { + org, project, ok := syfoncommon.ResourceScope(resource) + if ok && org != "" { + finding.Organization = org + finding.Project = project + break + } + } + } + return finding +} diff --git a/internal/repair/service_test.go b/internal/repair/service_test.go new file mode 100644 index 00000000..b4aedbf0 --- /dev/null +++ b/internal/repair/service_test.go @@ -0,0 +1,370 @@ +package repair + +import ( + "context" + "net/http" + "testing" + + "github.com/calypr/syfon/apigen/client/bucketapi" + drsapi "github.com/calypr/syfon/apigen/client/drs" + "github.com/calypr/syfon/apigen/client/internalapi" + "github.com/calypr/syfon/client/request" + syfoncommon "github.com/calypr/syfon/common" + intcommon "github.com/calypr/syfon/internal/common" +) + +func TestLegacyAccessURLWithCanonicalSiblingIsRemovable(t *testing.T) { + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{ + recordWithMethods(t, "did-1", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{ + "s3://bforepc/META/file.ndjson", + "s3://bforepc/bforepc-prod/did-1/abc", + }), + }}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 1 { + t.Fatalf("expected one object report, got %d", len(report.Objects)) + } + obj := report.Objects[0] + if !obj.AutoFixable { + t.Fatal("expected autofixable object") + } + if obj.Findings[0].Kind != FindingLegacyAccessURLRemovable { + t.Fatalf("expected removable finding, got %s", obj.Findings[0].Kind) + } +} + +func TestLegacyAccessURLWithoutCanonicalSiblingIsRewritable(t *testing.T) { + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{ + recordWithMethods(t, "did-1", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{ + "s3://bforepc/META/file.ndjson", + }), + }}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if got := report.Objects[0].Findings[0].Kind; got != FindingLegacyAccessURLRewritable { + t.Fatalf("expected rewritable finding, got %s", got) + } +} + +func TestPathStyleAccessURLWithCanonicalSiblingIsRemovable(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + // did is a real UUID + did := "01208c9f-e188-5aec-a026-51cc31fabd23" + // sha is a 64-char hex string + sha := "52aa452e5a90a2c641f20fffbac0e899d79d9cc40a560fe76e931b7f37566bb2" + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{ + recordWithMethods(t, did, sha, []string{resource}, []string{ + "s3://bforepc-prod/JHU/ashley_kiemen/hematoxylin_eosin_stain/Level_2/HTA201_3/HTA201_3_ndpis/HTA201_3_1_0145.ndpi", + "s3://bforepc-prod/" + did + "/" + sha, + }), + }}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 1 { + t.Fatalf("expected one object report, got %d", len(report.Objects)) + } + obj := report.Objects[0] + if !obj.AutoFixable { + t.Fatal("expected autofixable object") + } + if obj.Findings[0].Kind != FindingLegacyAccessURLRemovable { + t.Fatalf("expected removable finding, got %s", obj.Findings[0].Kind) + } +} + +func TestPathStyleAccessURLWithoutCanonicalSiblingIsRewritable(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + did := "01208c9f-e188-5aec-a026-51cc31fabd23" + sha := "52aa452e5a90a2c641f20fffbac0e899d79d9cc40a560fe76e931b7f37566bb2" + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{ + recordWithMethods(t, did, sha, []string{resource}, []string{ + "s3://bforepc-prod/JHU/ashley_kiemen/hematoxylin_eosin_stain/Level_2/HTA201_3/HTA201_3_ndpis/HTA201_3_1_0145.ndpi", + }), + }}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 1 { + t.Fatalf("expected one object report, got %d", len(report.Objects)) + } + obj := report.Objects[0] + if !obj.AutoFixable { + t.Fatal("expected autofixable object") + } + if obj.Findings[0].Kind != FindingLegacyAccessURLRewritable { + t.Fatalf("expected rewritable finding, got %s", obj.Findings[0].Kind) + } +} + +func TestMissingControlledAccessRecoverableFromDeterministicScope(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + did, err := intcommon.MintObjectIDFromChecksum("abc", []string{resource}) + if err != nil { + t.Fatalf("mint did: %v", err) + } + rec := recordWithMethods(t, did, "abc", nil, []string{"s3://bforepc/bforepc-prod/" + did + "/abc"}) + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{rec}}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if got := report.Objects[0].Findings[0].Kind; got != FindingMissingControlledAccess { + t.Fatalf("expected missing controlled access finding, got %s", got) + } + if !report.Objects[0].AutoFixable { + t.Fatal("expected autofixable object") + } +} + +func TestDuplicateSHAReportedWithoutAutofix(t *testing.T) { + rec1 := recordWithMethods(t, "did-1", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{"s3://bforepc/bforepc-prod/did-1/abc"}) + rec2 := recordWithMethods(t, "did-2", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{"s3://bforepc/bforepc-prod/did-2/abc"}) + svc := NewService( + &fakeIndex{records: []internalapi.InternalRecord{rec1, rec2}}, + fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 2 { + t.Fatalf("expected two object reports, got %d", len(report.Objects)) + } + for _, obj := range report.Objects { + found := false + for _, finding := range obj.Findings { + if finding.Kind == FindingDuplicateSHA256Sibling { + found = true + if finding.AutoFixable { + t.Fatal("duplicate sibling should not be autofixable") + } + } + } + if !found { + t.Fatalf("expected duplicate finding for %s", obj.ObjectID) + } + } +} + +func TestApplyUpdatesAccessMethodsAndControlledAccess(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + did, err := intcommon.MintObjectIDFromChecksum("abc", []string{resource}) + if err != nil { + t.Fatalf("mint did: %v", err) + } + rec := recordWithMethods(t, did, "abc", nil, []string{"s3://bforepc/META/file.ndjson"}) + idx := &fakeIndex{records: []internalapi.InternalRecord{rec}} + svc := NewService(idx, fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), &fakeRequester{}) + + result, err := svc.Apply(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if result.Mutated != 1 { + t.Fatalf("expected one mutation, got %d", result.Mutated) + } + if len(idx.updated) != 1 { + t.Fatalf("expected one updated record, got %d", len(idx.updated)) + } + updated := idx.updated[0] + if updated.ControlledAccess == nil || len(*updated.ControlledAccess) != 1 || (*updated.ControlledAccess)[0] != resource { + t.Fatalf("expected controlled access backfill, got %#v", updated.ControlledAccess) + } + if got := accessMethodURLs(updated.AccessMethods); len(got) != 1 || got[0] != "s3://bforepc/bforepc-prod/"+did+"/abc" { + t.Fatalf("unexpected access methods: %#v", got) + } +} + +func TestCheckStorageAddsMissingObjectFinding(t *testing.T) { + rec := recordWithMethods(t, "did-1", "abc", []string{"/organization/HTAN_INT/project/BForePC"}, []string{"s3://bforepc/bforepc-prod/did-1/abc"}) + req := &fakeRequester{ + err: &request.ResponseError{Method: http.MethodPost, URL: "/data/inspect", Status: http.StatusNotFound}, + } + svc := NewService(&fakeIndex{records: []internalapi.InternalRecord{rec}}, fakeBucketsForScope(t, "HTAN_INT", "BForePC", "s3://bforepc/bforepc-prod"), req) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC", CheckStorage: true}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + found := false + for _, finding := range report.Objects[0].Findings { + if finding.Kind == FindingStorageObjectMissing { + found = true + } + } + if !found { + t.Fatal("expected storage object missing finding") + } + if req.calls == 0 { + t.Fatal("expected storage inspect call") + } +} + +func TestSingleControlledAccessWithoutScopeTargetDoesNotPanicOrMutate(t *testing.T) { + resource, _ := syfoncommon.ResourcePath("HTAN_INT", "BForePC") + rec := recordWithMethods(t, "did-1", "abc", []string{resource}, []string{"s3://bforepc/legacy/file.ndjson"}) + idx := &fakeIndex{records: []internalapi.InternalRecord{rec}} + svc := NewService( + idx, + &fakeBuckets{ + list: bucketapi.BucketsResponse{ + S3BUCKETS: map[string]bucketapi.BucketMetadata{ + "other-bucket": {}, + }, + }, + scopes: map[string][]bucketapi.BucketScopeResponse{ + "other-bucket": nil, + }, + }, + &fakeRequester{}, + ) + + report, err := svc.Audit(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("audit failed: %v", err) + } + if len(report.Objects) != 0 { + t.Fatalf("expected no findings without a canonical target, got %d", len(report.Objects)) + } + + result, err := svc.Apply(context.Background(), Options{Organization: "HTAN_INT", Project: "BForePC"}) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + if result.Mutated != 0 || result.AutoFixable != 0 { + t.Fatalf("expected no mutations without a canonical target, got mutated=%d autofixable=%d", result.Mutated, result.AutoFixable) + } + if len(idx.updated) != 0 { + t.Fatalf("expected no updates, got %d", len(idx.updated)) + } +} + +type fakeIndex struct { + records []internalapi.InternalRecord + updated []internalapi.InternalRecord + listErr error + updateErr error +} + +func (f *fakeIndex) List(ctx context.Context, opts ListRecordsOptions) (internalapi.ListRecordsResponse, error) { + if f.listErr != nil { + return internalapi.ListRecordsResponse{}, f.listErr + } + records := append([]internalapi.InternalRecord(nil), f.records...) + return internalapi.ListRecordsResponse{Records: &records}, nil +} + +func (f *fakeIndex) Update(ctx context.Context, did string, rec internalapi.InternalRecord) (internalapi.InternalRecordResponse, error) { + if f.updateErr != nil { + return internalapi.InternalRecordResponse{}, f.updateErr + } + f.updated = append(f.updated, rec) + return internalapi.InternalRecordResponse{Did: did}, nil +} + +type fakeBuckets struct { + list bucketapi.BucketsResponse + scopes map[string][]bucketapi.BucketScopeResponse +} + +func (f *fakeBuckets) List(ctx context.Context) (bucketapi.BucketsResponse, error) { + return f.list, nil +} + +func (f *fakeBuckets) ListScopes(ctx context.Context, bucket string) ([]bucketapi.BucketScopeResponse, error) { + return f.scopes[bucket], nil +} + +type fakeRequester struct { + err error + calls int +} + +func (f *fakeRequester) Do(ctx context.Context, method, path string, body, out any, opts ...request.RequestOption) error { + f.calls++ + if f.err != nil { + return f.err + } + return nil +} + +func recordWithMethods(t *testing.T, did, sha string, controlled []string, urls []string) internalapi.InternalRecord { + t.Helper() + hashes := internalapi.HashInfo{"sha256": sha} + methods := make([]drsapi.AccessMethod, 0, len(urls)) + for _, raw := range urls { + accessURL := raw + methods = append(methods, drsapi.AccessMethod{ + Type: drsapi.AccessMethodTypeS3, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: accessURL}, + }) + } + rec := internalapi.InternalRecord{ + Did: did, + Hashes: &hashes, + AccessMethods: &methods, + } + if controlled != nil { + c := append([]string(nil), controlled...) + rec.ControlledAccess = &c + } + return rec +} + +func fakeBucketsForScope(t *testing.T, org, project, rawPath string) *fakeBuckets { + t.Helper() + return &fakeBuckets{ + list: bucketapi.BucketsResponse{ + S3BUCKETS: map[string]bucketapi.BucketMetadata{ + "bforepc": {}, + }, + }, + scopes: map[string][]bucketapi.BucketScopeResponse{ + "bforepc": {{ + Organization: org, + ProjectId: project, + Path: &rawPath, + }}, + }, + } +} + +var _ request.Requester = (*fakeRequester)(nil) diff --git a/internal/repair/types.go b/internal/repair/types.go new file mode 100644 index 00000000..f63a6065 --- /dev/null +++ b/internal/repair/types.go @@ -0,0 +1,143 @@ +package repair + +import ( + "context" + + "github.com/calypr/syfon/apigen/client/bucketapi" + "github.com/calypr/syfon/apigen/client/internalapi" + "github.com/calypr/syfon/client/request" +) + +type IndexAPI interface { + List(ctx context.Context, opts ListRecordsOptions) (internalapi.ListRecordsResponse, error) + Update(ctx context.Context, did string, rec internalapi.InternalRecord) (internalapi.InternalRecordResponse, error) +} + +type ListRecordsOptions struct { + Limit int + Start string + Organization string + Project string +} + +type BucketsAPI interface { + List(ctx context.Context) (bucketapi.BucketsResponse, error) + ListScopes(ctx context.Context, bucket string) ([]bucketapi.BucketScopeResponse, error) +} + +type Options struct { + Organization string + Project string + CheckStorage bool + Format string + Limit int + PageSize int +} + +type FindingKind string + +const ( + FindingLegacyAccessURLRemovable FindingKind = "legacy_access_url_removable" + FindingLegacyAccessURLRewritable FindingKind = "legacy_access_url_rewritable" + FindingNonCanonicalAccessURL FindingKind = "non_canonical_access_url" + FindingMissingControlledAccess FindingKind = "missing_controlled_access" + FindingDuplicateSHA256Sibling FindingKind = "duplicate_sha256_sibling" + FindingStorageObjectMissing FindingKind = "storage_object_missing" + FindingStorageProbeError FindingKind = "storage_probe_error" +) + +type Severity string + +const ( + SeverityInfo Severity = "info" + SeverityWarn Severity = "warn" + SeverityError Severity = "error" +) + +type Finding struct { + Kind FindingKind `json:"kind"` + Severity Severity `json:"severity"` + ObjectID string `json:"object_id"` + SHA256 string `json:"sha256,omitempty"` + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + CurrentAccessURLs []string `json:"current_access_urls,omitempty"` + ProposedCanonicalURL string `json:"proposed_canonical_url,omitempty"` + AutoFixable bool `json:"auto_fixable"` + Message string `json:"message,omitempty"` +} + +type ObjectReport struct { + ObjectID string `json:"object_id"` + SHA256 string `json:"sha256,omitempty"` + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + CurrentAccessURLs []string `json:"current_access_urls,omitempty"` + ProposedCanonicalURL string `json:"proposed_canonical_url,omitempty"` + AutoFixable bool `json:"auto_fixable"` + Findings []Finding `json:"findings,omitempty"` +} + +type Report struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + Scanned int `json:"scanned"` + Objects []ObjectReport `json:"objects,omitempty"` +} + +type ApplyResult struct { + Report Report `json:"report"` + Mutated int `json:"mutated"` + Skipped int `json:"skipped"` + AutoFixable int `json:"auto_fixable"` +} + +func (r Report) FindingCount() int { + return len(r.Objects) +} + +type Service struct { + index IndexAPI + buckets BucketsAPI + requestor request.Requester +} + +func NewService(index IndexAPI, buckets BucketsAPI, requestor request.Requester) *Service { + return &Service{index: index, buckets: buckets, requestor: requestor} +} + +type scopeTarget struct { + Resource string + Organization string + Project string + Bucket string + Prefix string +} + +type storageInspectRequest struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + ObjectURL string `json:"object_url,omitempty"` +} + +type storageInspectResponse struct { + ObjectURL string `json:"object_url"` +} + +type auditState struct { + report Report + objects []*auditedObject +} + +type auditedObject struct { + record internalapi.InternalRecord + sha256 string + currentURLs []string + scope scopeTarget + scopeKnown bool + scopeAmbiguous bool + inferredScope string + canonicalURL string + findings []Finding + updated *internalapi.InternalRecord +} diff --git a/internal/testutils/mocks.go b/internal/testutils/mocks.go index d91b54e9..5595c91b 100644 --- a/internal/testutils/mocks.go +++ b/internal/testutils/mocks.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "net/http" - "sort" "strings" "time" @@ -120,6 +119,54 @@ func (m *MockDatabase) GetObjectsByChecksums(ctx context.Context, checksums []st return out, nil } +func (m *MockDatabase) ListScopedObjectIDsByChecksums(ctx context.Context, organization, project string, checksums []string) (map[string][]string, error) { + out := make(map[string][]string, len(checksums)) + for _, checksum := range checksums { + checksum = strings.TrimSpace(checksum) + if checksum == "" { + continue + } + out[checksum] = []string{} + matches, err := m.GetObjectsByChecksum(ctx, checksum) + if err != nil { + return nil, err + } + seen := map[string]struct{}{} + for _, match := range matches { + if !objectMatchesMockScope(m.ObjectAuthz[match.Id], organization, project) { + continue + } + if _, ok := seen[match.Id]; ok { + continue + } + seen[match.Id] = struct{}{} + out[checksum] = append(out[checksum], match.Id) + } + } + return out, nil +} + +func objectMatchesMockScope(authz map[string][]string, organization, project string) bool { + organization = strings.TrimSpace(organization) + project = strings.TrimSpace(project) + if organization == "" { + return true + } + projects, ok := authz[organization] + if !ok { + return false + } + if project == "" || len(projects) == 0 { + return true + } + for _, candidate := range projects { + if candidate == project { + return true + } + } + return false +} + func (m *MockDatabase) ListObjectIDsByScope(ctx context.Context, organization, project string) ([]string, error) { ids := make([]string, 0) for id := range m.Objects { @@ -151,90 +198,6 @@ func (m *MockDatabase) ListObjectIDsByScope(ctx context.Context, organization, p return ids, nil } -func (m *MockDatabase) ListObjectIDsPageByPath(ctx context.Context, organization, project, path, startAfter string, limit, offset int) ([]string, []models.BrowseDirectory, error) { - normalizedPath, pathSegments, err := common.NormalizeBrowsePath(path) - if err != nil { - return nil, nil, err - } - ids := make([]string, 0) - directoriesByPath := make(map[string]models.BrowseDirectory) - for id, obj := range m.Objects { - authz := map[string][]string{} - if m.ObjectAuthz != nil { - if v, ok := m.ObjectAuthz[id]; ok { - authz = v - } - } - projects, ok := authz[organization] - if !ok { - continue - } - matchesProject := false - for _, candidateProject := range projects { - if candidateProject == project { - matchesProject = true - break - } - } - if !matchesProject { - continue - } - if obj.Name == nil { - continue - } - info, ok, err := common.BrowsePathInfoFromName(*obj.Name) - if err != nil || !ok { - continue - } - if !common.HasBrowsePathPrefix(info.Segments, pathSegments) { - continue - } - remaining := info.Segments[len(pathSegments):] - if len(remaining) == 0 { - continue - } - if len(remaining) == 1 { - if info.ParentPath == normalizedPath { - ids = append(ids, id) - } - continue - } - dirInfo, ok := common.ImmediateBrowseDirectory(normalizedPath, info.Normalized) - if !ok { - continue - } - directoriesByPath[dirInfo.Normalized] = models.BrowseDirectory{Name: dirInfo.EntryName, Path: dirInfo.Normalized} - } - sort.Strings(ids) - if startAfter != "" { - offset = sort.SearchStrings(ids, startAfter) - for offset < len(ids) && ids[offset] <= startAfter { - offset++ - } - } - if offset < 0 { - offset = 0 - } - if offset > len(ids) { - offset = len(ids) - } - end := offset + limit - if end > len(ids) { - end = len(ids) - } - directories := make([]models.BrowseDirectory, 0, len(directoriesByPath)) - for _, directory := range directoriesByPath { - directories = append(directories, directory) - } - sort.Slice(directories, func(i, j int) bool { - if directories[i].Name == directories[j].Name { - return directories[i].Path < directories[j].Path - } - return directories[i].Name < directories[j].Name - }) - return ids[offset:end], directories, nil -} - func (m *MockDatabase) ListObjectIDsByResources(ctx context.Context, resources []string, includeUnscoped bool) ([]string, error) { allowed := map[string]struct{}{} for _, resource := range resources { @@ -849,64 +812,6 @@ func (m *MockDatabase) GetTransferAttributionBreakdown(ctx context.Context, filt return out, nil } -func providerTransferEventMatchesFilter(ev models.ProviderTransferEvent, filter models.TransferAttributionFilter) bool { - status := filter.ReconciliationStatus - if status == "" { - status = models.ProviderTransferMatched - } - if status != "all" && ev.ReconciliationStatus != status { - return false - } - if filter.Organization != "" && ev.Organization != filter.Organization { - return false - } - if filter.Project != "" && ev.Project != filter.Project { - return false - } - direction := filter.Direction - if direction == "" { - direction = filter.EventType - } - if direction != "" && direction != "all" && ev.Direction != direction { - return false - } - if filter.From != nil && ev.EventTime.Before(*filter.From) { - return false - } - if filter.To != nil && ev.EventTime.After(*filter.To) { - return false - } - if filter.Provider != "" && ev.Provider != filter.Provider { - return false - } - if filter.Bucket != "" && ev.Bucket != filter.Bucket { - return false - } - if filter.SHA256 != "" && ev.SHA256 != filter.SHA256 { - return false - } - if filter.User != "" && ev.ActorEmail != filter.User && ev.ActorSubject != filter.User { - return false - } - return true -} - -func providerTransferBreakdownKey(ev models.ProviderTransferEvent, groupBy string) string { - switch groupBy { - case "user": - if ev.ActorEmail != "" { - return ev.ActorEmail - } - return ev.ActorSubject - case "provider": - return ev.Provider + ":" + ev.Bucket - case "object": - return ev.SHA256 - default: - return ev.Organization + "/" + ev.Project - } -} - func transferEventMatchesFilter(ev models.TransferAttributionEvent, filter models.TransferAttributionFilter) bool { if filter.Organization != "" && ev.Organization != filter.Organization { return false diff --git a/internal/version/version.go b/internal/version/version.go index 884fe763..f908be39 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -22,14 +22,3 @@ version: %s func String() string { return fmt.Sprintf("Syfon — a Modern DRS Server ⚡️\n"+tpl, GitCommit, GitBranch, GitUpstream, BuildDate, Version) } - -// LogFields logs build and version information to the given logger. -func LogFields() []interface{} { - return []interface{}{ - "GitCommit", GitCommit, - "GitBranch", GitBranch, - "GitUpstream", GitUpstream, - "BuildDate", BuildDate, - "Version", Version, - } -} diff --git a/migrate/client.go b/migrate/client.go deleted file mode 100644 index e067d406..00000000 --- a/migrate/client.go +++ /dev/null @@ -1,379 +0,0 @@ -package migrate - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/url" - "strings" - "time" - - "github.com/calypr/syfon/apigen/server/drs" -) - -const maxHTTPAttempts = 3 - -type BasicAuth struct { - Username string - Password string -} - -type AuthConfig struct { - BearerToken string - Basic *BasicAuth -} - -type HTTPClient struct { - baseURL string - httpClient *http.Client - auth AuthConfig - userAgent string -} - -func NewHTTPClient(baseURL string, auth AuthConfig, httpClient *http.Client) (*HTTPClient, error) { - baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") - if baseURL == "" { - return nil, fmt.Errorf("base URL is required") - } - if httpClient == nil { - httpClient = &http.Client{Timeout: 60 * time.Second} - } - return &HTTPClient{ - baseURL: baseURL, - httpClient: httpClient, - auth: auth, - userAgent: "syfon-migrate", - }, nil -} - -func (c *HTTPClient) ListPage(ctx context.Context, limit int, start string) ([]IndexdRecord, error) { - q := url.Values{} - if limit > 0 { - q.Set("limit", fmt.Sprintf("%d", limit)) - } - if strings.TrimSpace(start) != "" { - q.Set("start", start) - } - endpoint := c.endpoint("/index") - if encoded := q.Encode(); encoded != "" { - endpoint += "?" + encoded - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return nil, err - } - c.applyHeaders(req) - resp, err := c.do(req) - if err != nil { - return nil, fmt.Errorf("GET %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return nil, fmt.Errorf("GET %s: read error response body: %w", endpoint, err) - } - return nil, fmt.Errorf("GET %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - - var page IndexdPage - if err := json.NewDecoder(resp.Body).Decode(&page); err != nil { - return nil, fmt.Errorf("decode %s: %w", endpoint, err) - } - if len(page.Records) > 0 { - return page.Records, nil - } - if len(page.IDs) > 0 { - return c.fetchDocuments(ctx, page.IDs) - } - return nil, nil -} - -func (c *HTTPClient) fetchDocuments(ctx context.Context, ids []string) ([]IndexdRecord, error) { - records, err := c.bulkDocuments(ctx, ids) - if err == nil { - return records, nil - } - - out := make([]IndexdRecord, 0, len(ids)) - for _, id := range ids { - rec, getErr := c.getRecord(ctx, id) - if getErr != nil { - return nil, fmt.Errorf("bulk documents failed: %v; detail fetch %s failed: %w", err, id, getErr) - } - out = append(out, rec) - } - return out, nil -} - -func (c *HTTPClient) bulkDocuments(ctx context.Context, ids []string) ([]IndexdRecord, error) { - body, err := json.Marshal(ids) - if err != nil { - return nil, err - } - endpoint := c.endpoint("/index/bulk/documents") - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if err != nil { - return nil, err - } - c.applyHeaders(req) - req.Header.Set("Content-Type", "application/json") - resp, err := c.do(req) - if err != nil { - return nil, fmt.Errorf("POST %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return nil, fmt.Errorf("POST %s: read error response body: %w", endpoint, err) - } - return nil, fmt.Errorf("POST %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - var records []IndexdRecord - if err := json.NewDecoder(resp.Body).Decode(&records); err != nil { - return nil, fmt.Errorf("decode %s: %w", endpoint, err) - } - return records, nil -} - -func (c *HTTPClient) getRecord(ctx context.Context, id string) (IndexdRecord, error) { - endpoint := c.endpoint("/index/") + url.PathEscape(id) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return IndexdRecord{}, err - } - c.applyHeaders(req) - resp, err := c.do(req) - if err != nil { - return IndexdRecord{}, fmt.Errorf("GET %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return IndexdRecord{}, fmt.Errorf("GET %s: read error response body: %w", endpoint, err) - } - return IndexdRecord{}, fmt.Errorf("GET %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - var record IndexdRecord - if err := json.NewDecoder(resp.Body).Decode(&record); err != nil { - return IndexdRecord{}, fmt.Errorf("decode %s: %w", endpoint, err) - } - return record, nil -} - -func (c *HTTPClient) LoadBatch(ctx context.Context, records []MigrationRecord) error { - body, err := json.Marshal(bulkCreateRequestFromMigration(records)) - if err != nil { - return err - } - endpoint := c.endpoint("/index/bulk") - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if err != nil { - return err - } - c.applyHeaders(req) - req.Header.Set("Content-Type", "application/json") - resp, err := c.do(req) - if err != nil { - return fmt.Errorf("POST %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return fmt.Errorf("POST %s: read error response body: %w", endpoint, err) - } - return fmt.Errorf("POST %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - return nil -} - -func (c *HTTPClient) UserPrivileges(ctx context.Context) (map[string]map[string]bool, error) { - endpoint := c.userEndpoint("/user/user") - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return nil, err - } - c.applyHeaders(req) - resp, err := c.do(req) - if err != nil { - return nil, fmt.Errorf("GET %s: %w", endpoint, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := readTrimmedBody(resp.Body) - if err != nil { - return nil, fmt.Errorf("GET %s: read error response body: %w", endpoint, err) - } - return nil, fmt.Errorf("GET %s: unexpected status %d: %s", endpoint, resp.StatusCode, body) - } - - var data map[string]any - if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { - return nil, fmt.Errorf("decode %s: %w", endpoint, err) - } - resourceAccess, ok := data["authz"].(map[string]any) - if !ok || len(resourceAccess) == 0 { - resourceAccess, ok = data["project_access"].(map[string]any) - if !ok { - return nil, fmt.Errorf("no authz/project_access found in user response") - } - } - return extractUserPrivileges(resourceAccess), nil -} - -type bulkCreateRequest struct { - Records []bulkInternalRecord `json:"records"` -} - -type bulkInternalRecord struct { - AccessMethods []drs.AccessMethod `json:"access_methods,omitempty"` - ControlledAccess []string `json:"controlled_access,omitempty"` - CreatedTime *string `json:"created_time,omitempty"` - Description *string `json:"description,omitempty"` - Did string `json:"did"` - FileName *string `json:"file_name,omitempty"` - Hashes *map[string]string `json:"hashes,omitempty"` - Size *int64 `json:"size,omitempty"` - UpdatedTime *string `json:"updated_time,omitempty"` - Version *string `json:"version,omitempty"` -} - -func bulkCreateRequestFromMigration(records []MigrationRecord) bulkCreateRequest { - out := bulkCreateRequest{Records: make([]bulkInternalRecord, 0, len(records))} - for _, record := range records { - size := record.Size - created := record.CreatedTime.Format(time.RFC3339Nano) - var updated *string - if record.UpdatedTime != nil { - value := record.UpdatedTime.Format(time.RFC3339Nano) - updated = &value - } - hashes := make(map[string]string, len(record.Checksums)) - for _, checksum := range record.Checksums { - if checksum.Type != "" && checksum.Checksum != "" { - hashes[checksum.Type] = checksum.Checksum - } - } - out.Records = append(out.Records, bulkInternalRecord{ - Did: record.ID, - Size: &size, - FileName: record.Name, - Version: record.Version, - Description: record.Description, - CreatedTime: &created, - UpdatedTime: updated, - Hashes: &hashes, - AccessMethods: record.AccessMethods, - ControlledAccess: migrationRecordControlledAccess(record), - }) - } - return out -} - -func migrationRecordControlledAccess(record MigrationRecord) []string { - if len(record.ControlledAccess) > 0 { - return record.ControlledAccess - } - return record.Authz -} - -func (c *HTTPClient) endpoint(path string) string { - path = "/" + strings.TrimLeft(path, "/") - if strings.HasSuffix(c.baseURL, "/index") { - if path == "/index" { - return c.baseURL - } - return c.baseURL + strings.TrimPrefix(path, "/index") - } - return c.baseURL + path -} - -func (c *HTTPClient) userEndpoint(path string) string { - path = "/" + strings.TrimLeft(path, "/") - base := c.baseURL - if strings.HasSuffix(base, "/index") { - base = strings.TrimSuffix(base, "/index") - } - return base + path -} - -func (c *HTTPClient) do(req *http.Request) (*http.Response, error) { - var lastErr error - for attempt := 1; attempt <= maxHTTPAttempts; attempt++ { - if attempt > 1 && req.GetBody != nil { - body, err := req.GetBody() - if err != nil { - return nil, err - } - req.Body = body - } - - resp, err := c.httpClient.Do(req) - if !shouldRetryHTTP(resp, err) || attempt == maxHTTPAttempts { - return resp, err - } - if err != nil { - lastErr = err - } - if resp != nil && resp.Body != nil { - if _, copyErr := io.Copy(io.Discard, resp.Body); copyErr != nil && lastErr == nil { - lastErr = fmt.Errorf("discard retry response body: %w", copyErr) - } - if closeErr := resp.Body.Close(); closeErr != nil && lastErr == nil { - lastErr = fmt.Errorf("close retry response body: %w", closeErr) - } - } - - timer := time.NewTimer(time.Duration(attempt) * 750 * time.Millisecond) - select { - case <-req.Context().Done(): - timer.Stop() - if lastErr != nil { - return nil, lastErr - } - return nil, req.Context().Err() - case <-timer.C: - } - } - return nil, lastErr -} - -func readTrimmedBody(r io.Reader) (string, error) { - body, err := io.ReadAll(r) - if err != nil { - return "", err - } - return strings.TrimSpace(string(body)), nil -} - -func shouldRetryHTTP(resp *http.Response, err error) bool { - if err != nil { - var netErr net.Error - return errors.As(err, &netErr) && netErr.Timeout() - } - if resp == nil { - return false - } - return resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= http.StatusInternalServerError -} - -func (c *HTTPClient) applyHeaders(req *http.Request) { - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", c.userAgent) - if strings.TrimSpace(c.auth.BearerToken) != "" { - req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(c.auth.BearerToken)) - return - } - if c.auth.Basic != nil && c.auth.Basic.Username != "" { - req.SetBasicAuth(c.auth.Basic.Username, c.auth.Basic.Password) - } -} diff --git a/migrate/dump.go b/migrate/dump.go deleted file mode 100644 index d471e2a8..00000000 --- a/migrate/dump.go +++ /dev/null @@ -1,155 +0,0 @@ -package migrate - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - _ "github.com/mattn/go-sqlite3" -) - -type SQLiteDump struct { - db *sql.DB -} - -func OpenSQLiteDump(path string) (*SQLiteDump, error) { - if path == "" { - return nil, fmt.Errorf("dump path is required") - } - if dir := filepath.Dir(path); dir != "." && dir != "" { - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, err - } - } - db, err := sql.Open("sqlite3", path) - if err != nil { - return nil, err - } - dump := &SQLiteDump{db: db} - if err := dump.init(); err != nil { - db.Close() - return nil, err - } - return dump, nil -} - -func OpenExistingSQLiteDump(path string) (*SQLiteDump, error) { - if path == "" { - return nil, fmt.Errorf("dump path is required") - } - if _, err := os.Stat(path); err != nil { - return nil, err - } - return OpenSQLiteDump(path) -} - -func (d *SQLiteDump) Close() error { - if d == nil || d.db == nil { - return nil - } - return d.db.Close() -} - -func (d *SQLiteDump) init() error { - _, err := d.db.Exec(` -CREATE TABLE IF NOT EXISTS migration_records ( - id TEXT PRIMARY KEY, - record_json TEXT NOT NULL, - exported_at TEXT NOT NULL -); -CREATE TABLE IF NOT EXISTS migration_metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL -); -`) - return err -} - -func (d *SQLiteDump) LoadBatch(ctx context.Context, records []MigrationRecord) error { - tx, err := d.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() - stmt, err := tx.PrepareContext(ctx, ` -INSERT INTO migration_records (id, record_json, exported_at) -VALUES (?, ?, ?) -ON CONFLICT(id) DO UPDATE SET - record_json=excluded.record_json, - exported_at=excluded.exported_at -`) - if err != nil { - return err - } - defer stmt.Close() - - now := time.Now().UTC().Format(time.RFC3339Nano) - for _, record := range records { - body, err := json.Marshal(record) - if err != nil { - return err - } - if _, err := stmt.ExecContext(ctx, record.ID, string(body), now); err != nil { - return err - } - } - return tx.Commit() -} - -func (d *SQLiteDump) ReadBatches(ctx context.Context, batchSize int, fn func([]MigrationRecord) error) error { - if batchSize <= 0 { - batchSize = 500 - } - offset := 0 - for { - rows, err := d.db.QueryContext(ctx, ` -SELECT record_json FROM migration_records -ORDER BY id -LIMIT ? OFFSET ? -`, batchSize, offset) - if err != nil { - return err - } - - batch := make([]MigrationRecord, 0, batchSize) - for rows.Next() { - var raw string - if err := rows.Scan(&raw); err != nil { - rows.Close() - return err - } - var record MigrationRecord - if err := json.Unmarshal([]byte(raw), &record); err != nil { - rows.Close() - return err - } - batch = append(batch, record) - } - if err := rows.Err(); err != nil { - rows.Close() - return err - } - rows.Close() - - if len(batch) == 0 { - return nil - } - if err := fn(batch); err != nil { - return err - } - if len(batch) < batchSize { - return nil - } - offset += len(batch) - } -} - -func (d *SQLiteDump) Count(ctx context.Context) (int, error) { - var count int - err := d.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM migration_records`).Scan(&count) - return count, err -} diff --git a/migrate/dump_test.go b/migrate/dump_test.go deleted file mode 100644 index 2d47e10e..00000000 --- a/migrate/dump_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package migrate - -import ( - "context" - "path/filepath" - "testing" - "time" - - "github.com/calypr/syfon/apigen/server/drs" -) - -func TestSQLiteDumpRoundTrip(t *testing.T) { - path := filepath.Join(t.TempDir(), "records.sqlite") - dump, err := OpenSQLiteDump(path) - if err != nil { - t.Fatalf("OpenSQLiteDump: %v", err) - } - defer dump.Close() - - record := MigrationRecord{ - ID: "dg.test/roundtrip", - Size: 1, - CreatedTime: time.Now().UTC(), - Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha"}}, - } - if err := dump.LoadBatch(context.Background(), []MigrationRecord{record}); err != nil { - t.Fatalf("LoadBatch: %v", err) - } - count, err := dump.Count(context.Background()) - if err != nil { - t.Fatalf("Count: %v", err) - } - if count != 1 { - t.Fatalf("expected 1 record, got %d", count) - } - - var got []MigrationRecord - if err := dump.ReadBatches(context.Background(), 10, func(batch []MigrationRecord) error { - got = append(got, batch...) - return nil - }); err != nil { - t.Fatalf("ReadBatches: %v", err) - } - if len(got) != 1 || got[0].ID != record.ID { - t.Fatalf("unexpected roundtrip records: %+v", got) - } -} diff --git a/migrate/preflight.go b/migrate/preflight.go deleted file mode 100644 index 83c5e8f4..00000000 --- a/migrate/preflight.go +++ /dev/null @@ -1,205 +0,0 @@ -package migrate - -import ( - "context" - "fmt" - "sort" - "strings" - - syfoncommon "github.com/calypr/syfon/common" -) - -const maxPreflightMissingScopes = 10 - -type ImportPreflightReport struct { - Records int - RequiredResources []string - MissingResources []string - MissingRecords int - FirstDeniedRecord string -} - -type ImportPreflightError struct { - Report ImportPreflightReport -} - -func (e *ImportPreflightError) Error() string { - report := e.Report - scopes := FormatPreflightScopes(report.MissingResources) - if len(scopes) > maxPreflightMissingScopes { - truncated := len(scopes) - maxPreflightMissingScopes - scopes = scopes[:maxPreflightMissingScopes] - scopes = append(scopes, fmt.Sprintf("(and %d more)", truncated)) - } - - var b strings.Builder - fmt.Fprintf(&b, "missing create access for %d/%d records", report.MissingRecords, report.Records) - if report.FirstDeniedRecord != "" { - fmt.Fprintf(&b, "; first denied record=%q", report.FirstDeniedRecord) - } - if len(scopes) > 0 { - fmt.Fprintf(&b, "; missing organization/project scopes: %s", strings.Join(scopes, ", ")) - } - return b.String() -} - -func PreflightImport(ctx context.Context, reader DumpReader, privileges PrivilegeLister, batchSize int) (ImportPreflightReport, error) { - if reader == nil { - return ImportPreflightReport{}, fmt.Errorf("reader is required") - } - if privileges == nil { - return ImportPreflightReport{}, fmt.Errorf("privilege lister is required") - } - userPrivileges, err := privileges.UserPrivileges(ctx) - if err != nil { - return ImportPreflightReport{}, fmt.Errorf("load target privileges: %w", err) - } - userPrivileges = normalizePrivileges(userPrivileges) - - report := ImportPreflightReport{} - required := map[string]struct{}{} - missing := map[string]struct{}{} - - err = reader.ReadBatches(ctx, batchSize, func(records []MigrationRecord) error { - for _, record := range records { - if err := Validate(record); err != nil { - continue - } - report.Records++ - resources := syfoncommon.NormalizeAccessResources(migrationRecordControlledAccess(record)) - for _, resource := range resources { - required[resource] = struct{}{} - } - if hasCreateAccess(userPrivileges, resources) { - continue - } - report.MissingRecords++ - if report.FirstDeniedRecord == "" { - report.FirstDeniedRecord = record.ID - } - for _, resource := range resources { - missing[resource] = struct{}{} - } - } - return nil - }) - if err != nil { - return report, err - } - - report.RequiredResources = sortedMapKeys(required) - report.MissingResources = sortedMapKeys(missing) - if report.MissingRecords > 0 { - return report, &ImportPreflightError{Report: report} - } - return report, nil -} - -func extractUserPrivileges(resourceAccess map[string]any) map[string]map[string]bool { - out := make(map[string]map[string]bool, len(resourceAccess)) - for rawPath, rawEntries := range resourceAccess { - path := syfoncommon.NormalizeAccessResource(rawPath) - if path == "" { - continue - } - methods := map[string]bool{} - entries, ok := rawEntries.([]any) - if !ok { - out[path] = methods - continue - } - for _, entry := range entries { - mm, ok := entry.(map[string]any) - if !ok { - continue - } - service, _ := mm["service"].(string) - if service != "" && service != "indexd" && service != "drs" && service != "*" { - continue - } - method, _ := mm["method"].(string) - method = strings.TrimSpace(method) - if method == "" { - continue - } - methods[method] = true - } - out[path] = methods - } - return out -} - -func hasCreateAccess(privileges map[string]map[string]bool, resources []string) bool { - resources = syfoncommon.NormalizeAccessResources(resources) - if len(resources) == 0 { - return false - } - for _, resource := range resources { - methods, ok := privileges[resource] - if !ok { - continue - } - if methods["create"] || methods["*"] { - return true - } - } - return false -} - -func normalizePrivileges(privileges map[string]map[string]bool) map[string]map[string]bool { - if len(privileges) == 0 { - return nil - } - out := make(map[string]map[string]bool, len(privileges)) - for rawResource, methods := range privileges { - resource := syfoncommon.NormalizeAccessResource(rawResource) - if resource == "" { - continue - } - if out[resource] == nil { - out[resource] = map[string]bool{} - } - for method, allowed := range methods { - if allowed { - out[resource][method] = true - } - } - } - return out -} - -func sortedMapKeys(in map[string]struct{}) []string { - out := make([]string, 0, len(in)) - for key := range in { - out = append(out, key) - } - sort.Strings(out) - return out -} - -func FormatPreflightScopes(resources []string) []string { - out := make([]string, 0, len(resources)) - for _, resource := range resources { - formatted := formatPreflightScope(resource) - if formatted == "" { - continue - } - out = append(out, formatted) - } - return out -} - -func formatPreflightScope(resource string) string { - resource = syfoncommon.NormalizeAccessResource(resource) - org, project, ok := syfoncommon.ResourceScope(resource) - if !ok { - return resource - } - if org != "" && project != "" { - return org + "/" + project - } - if strings.TrimSpace(org) != "" { - return org + "/*" - } - return resource -} diff --git a/migrate/run.go b/migrate/run.go deleted file mode 100644 index 92e8eb62..00000000 --- a/migrate/run.go +++ /dev/null @@ -1,153 +0,0 @@ -package migrate - -import ( - "context" - "fmt" - "log/slog" - "time" -) - -func Run(ctx context.Context, source SourceLister, loader Loader, cfg Config) (Stats, error) { - if source == nil { - return Stats{}, fmt.Errorf("source is required") - } - if loader == nil && !cfg.DryRun { - return Stats{}, fmt.Errorf("loader is required") - } - batchSize := cfg.BatchSize - if batchSize <= 0 { - batchSize = 500 - } - if batchSize > 1024 { - batchSize = 1024 - } - sweeps := cfg.Sweeps - if sweeps <= 0 { - sweeps = 1 - } - - var stats Stats - seen := map[string]struct{}{} - now := time.Now().UTC() - - for sweep := 0; sweep < sweeps; sweep++ { - start := "" - sweepNew := 0 - for { - if cfg.Limit > 0 && stats.CountOfUniqueIDs >= cfg.Limit { - return stats, nil - } - fetchLimit := batchSize - if cfg.Limit > 0 { - remaining := cfg.Limit - stats.CountOfUniqueIDs - if remaining < fetchLimit { - fetchLimit = remaining - } - } - records, err := source.ListPage(ctx, fetchLimit, start) - if err != nil { - return stats, err - } - if len(records) == 0 { - break - } - - stats.Fetched += len(records) - filtered := make([]IndexdRecord, 0, len(records)) - maxDID := start - for _, record := range records { - if record.DID > maxDID { - maxDID = record.DID - } - if record.DID == "" { - filtered = append(filtered, record) - continue - } - if _, ok := seen[record.DID]; ok { - continue - } - seen[record.DID] = struct{}{} - filtered = append(filtered, record) - sweepNew++ - } - stats.CountOfUniqueIDs = len(seen) - if cfg.Limit > 0 && len(filtered) > fetchLimit { - filtered = filtered[:fetchLimit] - } - - transformed, transformErrs := TransformBatch(filtered, cfg.DefaultAuthz, now) - stats.Errors += len(transformErrs) - for _, err := range transformErrs { - slog.Warn("migration transform failed", "did", err.DID, "err", err.Err) - } - - valid := make([]MigrationRecord, 0, len(transformed)) - for _, record := range transformed { - if err := Validate(record); err != nil { - stats.Skipped++ - slog.Warn("migration validation skipped record", "id", record.ID, "err", err) - continue - } - valid = append(valid, record) - } - stats.Transformed += len(valid) - - if len(valid) > 0 { - if cfg.DryRun { - stats.Loaded += len(valid) - } else { - if err := loader.LoadBatch(ctx, valid); err != nil { - return stats, err - } - stats.Loaded += len(valid) - } - } - - if maxDID == "" || maxDID == start || len(records) < fetchLimit { - break - } - start = maxDID - } - - if sweepNew == 0 { - break - } - } - - return stats, nil -} - -func Import(ctx context.Context, reader DumpReader, loader Loader, batchSize int) (Stats, error) { - if reader == nil { - return Stats{}, fmt.Errorf("reader is required") - } - if loader == nil { - return Stats{}, fmt.Errorf("loader is required") - } - if batchSize <= 0 { - batchSize = 500 - } - var stats Stats - err := reader.ReadBatches(ctx, batchSize, func(records []MigrationRecord) error { - valid := make([]MigrationRecord, 0, len(records)) - for _, record := range records { - stats.Fetched++ - if err := Validate(record); err != nil { - stats.Skipped++ - slog.Warn("migration import skipped record", "id", record.ID, "err", err) - continue - } - valid = append(valid, record) - } - stats.Transformed += len(valid) - if len(valid) == 0 { - return nil - } - if err := loader.LoadBatch(ctx, valid); err != nil { - return err - } - stats.Loaded += len(valid) - return nil - }) - return stats, err -} diff --git a/migrate/run_test.go b/migrate/run_test.go deleted file mode 100644 index 15b0752a..00000000 --- a/migrate/run_test.go +++ /dev/null @@ -1,287 +0,0 @@ -package migrate - -import ( - "context" - "encoding/json" - "errors" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/calypr/syfon/apigen/server/drs" -) - -func TestRunPaginatesAndLoadsAllRecords(t *testing.T) { - size := int64(1) - source := &fakeSource{pages: [][]IndexdRecord{ - { - {DID: "a", Size: &size, Hashes: map[string]string{"sha256": "sha-a"}}, - {DID: "b", Size: &size, Hashes: map[string]string{"sha256": "sha-b"}}, - }, - { - {DID: "c", Size: &size, Hashes: map[string]string{"sha256": "sha-c"}}, - }, - }} - loader := &fakeLoader{} - - stats, err := Run(context.Background(), source, loader, Config{BatchSize: 2, Sweeps: 1}) - if err != nil { - t.Fatalf("Run returned error: %v", err) - } - if stats.Fetched != 3 || stats.Loaded != 3 || stats.CountOfUniqueIDs != 3 { - t.Fatalf("unexpected stats: %+v", stats) - } - if len(source.starts) != 2 || source.starts[0] != "" || source.starts[1] != "b" { - t.Fatalf("unexpected pagination starts: %+v", source.starts) - } - if len(loader.records) != 3 { - t.Fatalf("expected 3 loaded records, got %d", len(loader.records)) - } -} - -func TestRunDryRunDoesNotRequireLoader(t *testing.T) { - size := int64(1) - source := &fakeSource{pages: [][]IndexdRecord{{ - {DID: "a", Size: &size, Hashes: map[string]string{"sha256": "sha-a"}}, - }}} - - stats, err := Run(context.Background(), source, nil, Config{BatchSize: 100, DryRun: true}) - if err != nil { - t.Fatalf("Run returned error: %v", err) - } - if stats.Fetched != 1 || stats.Loaded != 1 || stats.CountOfUniqueIDs != 1 { - t.Fatalf("unexpected stats: %+v", stats) - } -} - -func TestHTTPClientEndToEndWithMockIndexdAndSyfon(t *testing.T) { - size := int64(7) - var loaded struct { - Records []struct { - Did string `json:"did"` - ControlledAccess []string `json:"controlled_access"` - AccessMethods []struct { - Type string `json:"type"` - AccessUrl struct { - Url string `json:"url"` - } `json:"access_url"` - } `json:"access_methods"` - } `json:"records"` - } - source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/index" { - t.Fatalf("unexpected source path: %s", r.URL.Path) - } - if got := r.URL.Query().Get("form"); got != "" { - t.Fatalf("did not expect form query, got %q", got) - } - _ = json.NewEncoder(w).Encode(IndexdPage{Records: []IndexdRecord{{ - DID: "dg.test/1", - Size: &size, - URLs: []string{"s3://bucket/key"}, - Hashes: map[string]string{"sha256": "sha"}, - Authz: []string{"https://calypr.org/program/foo/project/bar"}, - }}}) - })) - defer source.Close() - - target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/index/bulk" { - t.Fatalf("unexpected target path: %s", r.URL.Path) - } - if err := json.NewDecoder(r.Body).Decode(&loaded); err != nil { - t.Fatalf("decode target body: %v", err) - } - w.WriteHeader(http.StatusOK) - })) - defer target.Close() - - sourceClient, err := NewHTTPClient(source.URL, AuthConfig{}, source.Client()) - if err != nil { - t.Fatalf("NewHTTPClient source: %v", err) - } - targetClient, err := NewHTTPClient(target.URL, AuthConfig{}, target.Client()) - if err != nil { - t.Fatalf("NewHTTPClient target: %v", err) - } - - stats, err := Run(context.Background(), sourceClient, targetClient, Config{BatchSize: 100}) - if err != nil { - t.Fatalf("Run returned error: %v", err) - } - if stats.Loaded != 1 || len(loaded.Records) != 1 { - t.Fatalf("expected one loaded record, stats=%+v body=%+v", stats, loaded) - } - if loaded.Records[0].Did != "dg.test/1" { - t.Fatalf("unexpected loaded payload: %+v", loaded.Records[0]) - } - if len(loaded.Records[0].ControlledAccess) != 1 || loaded.Records[0].ControlledAccess[0] != "/organization/foo/project/bar" { - t.Fatalf("expected controlled_access payload, got %+v", loaded.Records[0]) - } - if len(loaded.Records[0].AccessMethods) != 1 || loaded.Records[0].AccessMethods[0].AccessUrl.Url != "s3://bucket/key" { - t.Fatalf("expected access_methods payload, got %+v", loaded.Records[0]) - } -} - -func TestHTTPClientRetriesTimeout(t *testing.T) { - transport := &retryTransport{} - client, err := NewHTTPClient("https://example.org/index", AuthConfig{}, &http.Client{Transport: transport}) - if err != nil { - t.Fatalf("NewHTTPClient: %v", err) - } - - records, err := client.ListPage(context.Background(), 1, "") - if err != nil { - t.Fatalf("ListPage returned error: %v", err) - } - if transport.calls != 2 { - t.Fatalf("expected one retry, got %d calls", transport.calls) - } - if len(records) != 1 || records[0].DID != "a" { - t.Fatalf("unexpected records: %+v", records) - } -} - -func TestPreflightImportReportsAllMissingCreateScopes(t *testing.T) { - reader := &fakeDumpReader{batches: [][]MigrationRecord{ - { - {ID: "allowed", Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-1"}}, Authz: []string{"/programs/cbds/projects/allowed"}}, - {ID: "denied-1", Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-2"}}, Authz: []string{"/programs/cbds/projects/missing-one"}}, - }, - { - {ID: "denied-2", Checksums: []drs.Checksum{{Type: "sha256", Checksum: "sha-3"}}, ControlledAccess: []string{"/programs/aced/projects/missing-two"}}, - }, - }} - privileges := fakePrivilegeLister{privileges: map[string]map[string]bool{ - "/programs/cbds/projects/allowed": {"create": true}, - }} - - _, err := PreflightImport(context.Background(), reader, privileges, 500) - if err == nil { - t.Fatal("expected preflight to fail") - } - var preflightErr *ImportPreflightError - if !errors.As(err, &preflightErr) { - t.Fatalf("expected ImportPreflightError, got %T %v", err, err) - } - report := preflightErr.Report - if report.Records != 3 || report.MissingRecords != 2 { - t.Fatalf("unexpected report counts: %+v", report) - } - if report.FirstDeniedRecord != "denied-1" { - t.Fatalf("unexpected first denied record: %q", report.FirstDeniedRecord) - } - body := err.Error() - if !strings.Contains(body, "cbds/missing-one") || !strings.Contains(body, "aced/missing-two") { - t.Fatalf("expected formatted missing scopes in error, got %q", body) - } -} - -func TestHTTPClientUserPrivileges(t *testing.T) { - var gotAuth string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/user/user" { - t.Fatalf("unexpected path: %s", r.URL.Path) - } - gotAuth = r.Header.Get("Authorization") - resp := map[string]any{ - "authz": map[string]any{ - "/programs/cbds/projects/p1": []any{ - map[string]any{"service": "indexd", "method": "read"}, - map[string]any{"service": "drs", "method": "create"}, - }, - "/programs/cbds/projects/p2": []any{ - map[string]any{"service": "fence", "method": "create"}, - }, - }, - } - _ = json.NewEncoder(w).Encode(resp) - })) - defer server.Close() - - client, err := NewHTTPClient(server.URL+"/index", AuthConfig{BearerToken: "tok"}, server.Client()) - if err != nil { - t.Fatalf("NewHTTPClient: %v", err) - } - privileges, err := client.UserPrivileges(context.Background()) - if err != nil { - t.Fatalf("UserPrivileges returned error: %v", err) - } - if gotAuth != "Bearer tok" { - t.Fatalf("expected bearer auth header, got %q", gotAuth) - } - if !privileges["/organization/cbds/project/p1"]["create"] { - t.Fatalf("expected create privilege for p1, got %+v", privileges) - } - if privileges["/organization/cbds/project/p2"]["create"] { - t.Fatalf("did not expect non-indexd/drs create privilege to be retained: %+v", privileges) - } -} - -type fakeSource struct { - pages [][]IndexdRecord - starts []string -} - -func (s *fakeSource) ListPage(ctx context.Context, limit int, start string) ([]IndexdRecord, error) { - s.starts = append(s.starts, start) - if len(s.starts) > len(s.pages) { - return nil, nil - } - return s.pages[len(s.starts)-1], nil -} - -type fakeLoader struct { - records []MigrationRecord -} - -func (l *fakeLoader) LoadBatch(ctx context.Context, records []MigrationRecord) error { - l.records = append(l.records, records...) - return nil -} - -type fakeDumpReader struct { - batches [][]MigrationRecord -} - -func (r *fakeDumpReader) ReadBatches(ctx context.Context, batchSize int, fn func([]MigrationRecord) error) error { - for _, batch := range r.batches { - if err := fn(batch); err != nil { - return err - } - } - return nil -} - -type fakePrivilegeLister struct { - privileges map[string]map[string]bool -} - -func (l fakePrivilegeLister) UserPrivileges(ctx context.Context) (map[string]map[string]bool, error) { - return l.privileges, nil -} - -type retryTransport struct { - calls int -} - -func (t *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) { - t.calls++ - if t.calls == 1 { - return nil, timeoutErr{} - } - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(strings.NewReader(`{"records":[{"did":"a","size":1,"hashes":{"sha256":"sha-a"}}]}`)), - Header: make(http.Header), - Request: req, - }, nil -} - -type timeoutErr struct{} - -func (timeoutErr) Error() string { return "timeout" } -func (timeoutErr) Timeout() bool { return true } -func (timeoutErr) Temporary() bool { return true } diff --git a/migrate/transform.go b/migrate/transform.go deleted file mode 100644 index 5dc48c1b..00000000 --- a/migrate/transform.go +++ /dev/null @@ -1,258 +0,0 @@ -package migrate - -import ( - "fmt" - "net/url" - "sort" - "strings" - "time" - - "github.com/calypr/syfon/apigen/server/drs" - syfoncommon "github.com/calypr/syfon/common" - "github.com/calypr/syfon/internal/models" -) - -var indexdDateFormats = []string{ - time.RFC3339Nano, - time.RFC3339, - "2006-01-02T15:04:05.999999", - "2006-01-02T15:04:05.999999Z", - "2006-01-02T15:04:05", - "2006-01-02 15:04:05.999999", - "2006-01-02 15:04:05", - "2006-01-02", -} - -type TransformError struct { - DID string - Err error -} - -func (e TransformError) Error() string { - return fmt.Sprintf("transform %s: %v", e.DID, e.Err) -} - -func Transform(rec IndexdRecord, defaultAuthz []string, now time.Time) (MigrationRecord, error) { - did := strings.TrimSpace(rec.DID) - if did == "" { - return MigrationRecord{}, fmt.Errorf("record has empty did") - } - if now.IsZero() { - now = time.Now().UTC() - } - - controlledAccess := syfoncommon.NormalizeAccessResources(rec.Authz) - if len(controlledAccess) == 0 { - controlledAccess = syfoncommon.NormalizeAccessResources(defaultAuthz) - } - - checksums := checksumsFromHashes(rec.Hashes) - created := firstParsedTime(now, rec.CreatedTime, rec.CreatedDate, rec.ContentCreatedDate) - updated := firstParsedTime(created, rec.UpdatedTime, rec.UpdatedDate, rec.ContentUpdatedDate) - accessMethods := accessMethodsFromURLs(rec.URLs) - - return MigrationRecord{ - ID: did, - Name: stringPtrValue(rec.FileName), - Size: int64PtrValue(rec.Size), - Version: stringPtrValue(rec.Version), - Description: stringPtrValue(rec.Description), - CreatedTime: created, - UpdatedTime: &updated, - Checksums: checksums, - AccessMethods: accessMethods, - ControlledAccess: controlledAccess, - Authz: controlledAccess, - }, nil -} - -func TransformBatch(records []IndexdRecord, defaultAuthz []string, now time.Time) ([]MigrationRecord, []TransformError) { - out := make([]MigrationRecord, 0, len(records)) - var errs []TransformError - for _, rec := range records { - mr, err := Transform(rec, defaultAuthz, now) - if err != nil { - errs = append(errs, TransformError{DID: rec.DID, Err: err}) - continue - } - out = append(out, mr) - } - return out, errs -} - -func Validate(record MigrationRecord) error { - if strings.TrimSpace(record.ID) == "" { - return fmt.Errorf("id is empty") - } - if len(record.Checksums) == 0 { - return fmt.Errorf("no checksums") - } - for _, checksum := range record.Checksums { - if strings.TrimSpace(checksum.Type) == "" || strings.TrimSpace(checksum.Checksum) == "" { - return fmt.Errorf("checksum entry has empty type or value") - } - } - return nil -} - -func MigrationRecordToInternalObject(record MigrationRecord) (models.InternalObject, error) { - if err := Validate(record); err != nil { - return models.InternalObject{}, err - } - controlledAccess := record.ControlledAccess - if len(controlledAccess) == 0 { - controlledAccess = record.Authz - } - controlledAccess = syfoncommon.NormalizeAccessResources(controlledAccess) - authzMap := syfoncommon.ControlledAccessToAuthzMap(controlledAccess) - updated := record.UpdatedTime - if updated == nil { - t := record.CreatedTime - updated = &t - } - obj := drs.DrsObject{ - Id: record.ID, - SelfUri: "drs://" + record.ID, - Size: record.Size, - Name: record.Name, - Version: record.Version, - Description: record.Description, - CreatedTime: record.CreatedTime, - UpdatedTime: updated, - Checksums: append([]drs.Checksum(nil), record.Checksums...), - AccessMethods: nil, - } - if authzMap != nil { - obj.ControlledAccess = &controlledAccess - } - if len(record.AccessMethods) > 0 { - methods := append([]drs.AccessMethod(nil), record.AccessMethods...) - obj.AccessMethods = &methods - } - return models.InternalObject{ - DrsObject: obj, - Authorizations: authzMap, - }, nil -} - -func firstParsedTime(fallback time.Time, values ...*string) time.Time { - for _, value := range values { - if value == nil { - continue - } - if parsed := parseIndexdDate(*value); !parsed.IsZero() { - return parsed - } - } - return fallback.UTC() -} - -func parseIndexdDate(raw string) time.Time { - raw = strings.TrimSpace(raw) - if raw == "" { - return time.Time{} - } - for _, layout := range indexdDateFormats { - if t, err := time.Parse(layout, raw); err == nil { - return t.UTC() - } - } - return time.Time{} -} - -func checksumsFromHashes(hashes map[string]string) []drs.Checksum { - if len(hashes) == 0 { - return nil - } - keys := make([]string, 0, len(hashes)) - for key := range hashes { - keys = append(keys, key) - } - sort.Strings(keys) - out := make([]drs.Checksum, 0, len(keys)) - for _, typ := range keys { - checksum := strings.TrimSpace(hashes[typ]) - typ = strings.TrimSpace(typ) - if typ == "" || checksum == "" { - continue - } - out = append(out, drs.Checksum{Type: typ, Checksum: checksum}) - } - return out -} - -func accessMethodsFromURLs(urls []string) []drs.AccessMethod { - if len(urls) == 0 { - return nil - } - seen := map[string]struct{}{} - out := make([]drs.AccessMethod, 0, len(urls)) - for _, rawURL := range urls { - rawURL = strings.TrimSpace(rawURL) - if rawURL == "" { - continue - } - if _, ok := seen[rawURL]; ok { - continue - } - seen[rawURL] = struct{}{} - methodType := accessMethodType(rawURL) - accessID := string(methodType) - method := drs.AccessMethod{ - Type: methodType, - AccessId: &accessID, - AccessUrl: &struct { - Headers *[]string `json:"headers,omitempty"` - Url string `json:"url"` - }{Url: rawURL}, - } - out = append(out, method) - } - return out -} - -func accessMethodType(rawURL string) drs.AccessMethodType { - parsed, err := url.Parse(strings.TrimSpace(rawURL)) - if err != nil || parsed.Scheme == "" { - return drs.AccessMethodTypeHttps - } - return drs.AccessMethodType(strings.ToLower(parsed.Scheme)) -} - -func uniqueStrings(values []string) []string { - if len(values) == 0 { - return nil - } - seen := map[string]struct{}{} - out := make([]string, 0, len(values)) - for _, value := range values { - value = strings.TrimSpace(value) - if value == "" { - continue - } - if _, ok := seen[value]; ok { - continue - } - seen[value] = struct{}{} - out = append(out, value) - } - return out -} - -func stringPtrValue(value *string) *string { - if value == nil { - return nil - } - trimmed := strings.TrimSpace(*value) - if trimmed == "" { - return nil - } - return &trimmed -} - -func int64PtrValue(value *int64) int64 { - if value == nil { - return 0 - } - return *value -} diff --git a/migrate/transform_test.go b/migrate/transform_test.go deleted file mode 100644 index c56a193e..00000000 --- a/migrate/transform_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package migrate - -import ( - "testing" - "time" -) - -func TestTransformIndexdRecord(t *testing.T) { - size := int64(42) - name := "file.bam" - version := "v1" - desc := "source record" - created := "2024-03-04T05:06:07Z" - rec := IndexdRecord{ - DID: "dg.test/abc", - Size: &size, - FileName: &name, - Version: &version, - Description: &desc, - CreatedDate: &created, - URLs: []string{"s3://bucket/key", "s3://bucket/key", "gs://bucket/key"}, - Hashes: map[string]string{"sha256": "abc123", "md5": "def456"}, - Authz: []string{"/programs/ohsu/projects/brca", "/programs/ohsu/projects/brca"}, - } - - got, err := Transform(rec, nil, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) - if err != nil { - t.Fatalf("Transform returned error: %v", err) - } - if got.ID != rec.DID || got.Size != size { - t.Fatalf("identity fields not preserved: %+v", got) - } - if got.Name == nil || *got.Name != name || got.Version == nil || *got.Version != version || got.Description == nil || *got.Description != desc { - t.Fatalf("metadata not preserved: %+v", got) - } - if len(got.Checksums) != 2 { - t.Fatalf("expected 2 checksums, got %+v", got.Checksums) - } - if len(got.AccessMethods) != 2 { - t.Fatalf("expected de-duplicated access methods, got %+v", got.AccessMethods) - } - if got.CreatedTime.Format(time.RFC3339) != created { - t.Fatalf("created time not parsed: %s", got.CreatedTime.Format(time.RFC3339)) - } - internal, err := MigrationRecordToInternalObject(got) - if err != nil { - t.Fatalf("MigrationRecordToInternalObject returned error: %v", err) - } - if internal.ControlledAccess == nil || len(*internal.ControlledAccess) != 1 || (*internal.ControlledAccess)[0] != "/organization/ohsu/project/brca" { - t.Fatalf("expected controlled access on object, got %+v", internal.ControlledAccess) - } -} - -func TestTransformAppliesDefaultAuthz(t *testing.T) { - size := int64(1) - rec := IndexdRecord{ - DID: "dg.test/default-authz", - Size: &size, - URLs: []string{"s3://bucket/key"}, - Hashes: map[string]string{"sha256": "abc123"}, - } - - got, err := Transform(rec, []string{"/programs/open"}, time.Now()) - if err != nil { - t.Fatalf("Transform returned error: %v", err) - } - if len(got.ControlledAccess) != 1 || got.ControlledAccess[0] != "/organization/open" { - t.Fatalf("expected default controlled access, got %+v", got.ControlledAccess) - } - internal, err := MigrationRecordToInternalObject(got) - if err != nil { - t.Fatalf("MigrationRecordToInternalObject returned error: %v", err) - } - if internal.Authorizations == nil || len(internal.Authorizations["open"]) != 0 { - t.Fatalf("unexpected internal authorizations: %+v", internal.Authorizations) - } -} diff --git a/migrate/types.go b/migrate/types.go deleted file mode 100644 index 93e225e2..00000000 --- a/migrate/types.go +++ /dev/null @@ -1,96 +0,0 @@ -package migrate - -import ( - "context" - "time" - - "github.com/calypr/syfon/apigen/server/drs" -) - -// IndexdRecord is the source record shape returned by Gen3 Indexd APIs. -// Deprecated Indexd fields are accepted so source payloads decode cleanly, -// but they are intentionally not loaded into Syfon. -type IndexdRecord struct { - DID string `json:"did"` - Size *int64 `json:"size,omitempty"` - FileName *string `json:"file_name,omitempty"` - Version *string `json:"version,omitempty"` - Description *string `json:"description,omitempty"` - URLs []string `json:"urls,omitempty"` - Hashes map[string]string `json:"hashes,omitempty"` - Authz []string `json:"authz,omitempty"` - CreatedDate *string `json:"created_date,omitempty"` - UpdatedDate *string `json:"updated_date,omitempty"` - CreatedTime *string `json:"created_time,omitempty"` - UpdatedTime *string `json:"updated_time,omitempty"` - ContentCreatedDate *string `json:"content_created_date,omitempty"` - ContentUpdatedDate *string `json:"content_updated_date,omitempty"` - Aliases []string `json:"aliases,omitempty"` - Form *string `json:"form,omitempty"` - - Baseid *string `json:"baseid,omitempty"` - Rev *string `json:"rev,omitempty"` - Uploader *string `json:"uploader,omitempty"` - ACL []string `json:"acl,omitempty"` - Metadata map[string]interface{} `json:"metadata,omitempty"` - URLsMetadata map[string]interface{} `json:"urls_metadata,omitempty"` -} - -type IndexdPage struct { - Records []IndexdRecord `json:"records"` - IDs []string `json:"ids"` - Start string `json:"start"` - Page *int `json:"page,omitempty"` - Limit *int `json:"limit,omitempty"` -} - -type SourceLister interface { - ListPage(ctx context.Context, limit int, start string) ([]IndexdRecord, error) -} - -type Loader interface { - LoadBatch(ctx context.Context, records []MigrationRecord) error -} - -type PrivilegeLister interface { - UserPrivileges(ctx context.Context) (map[string]map[string]bool, error) -} - -type DumpReader interface { - ReadBatches(ctx context.Context, batchSize int, fn func([]MigrationRecord) error) error -} - -type MigrationRecord struct { - ID string `json:"id"` - Name *string `json:"name,omitempty"` - Size int64 `json:"size"` - Version *string `json:"version,omitempty"` - Description *string `json:"description,omitempty"` - CreatedTime time.Time `json:"created_time"` - UpdatedTime *time.Time `json:"updated_time,omitempty"` - Checksums []drs.Checksum `json:"checksums"` - AccessMethods []drs.AccessMethod `json:"access_methods,omitempty"` - ControlledAccess []string `json:"controlled_access,omitempty"` - Authz []string `json:"authz,omitempty"` -} - -type MigrationRequest struct { - Records []MigrationRecord `json:"records"` -} - -type Stats struct { - Fetched int - Transformed int - Loaded int - Skipped int - Errors int - CountOfUniqueIDs int -} - -type Config struct { - BatchSize int - Limit int - DryRun bool - DefaultAuthz []string - Sweeps int -} diff --git a/mkdocs.yml b/mkdocs.yml index 245328a6..32724e9c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -46,7 +46,6 @@ nav: - Configuration: configuration.md - Encryption: encryption.md - Deployment: deployment.md - - Indexd to Syfon Migration: operator-guide-migration-indexd.md - Troubleshooting: troubleshooting.md - Code Generation: operator-guide-code-generation.md - Advanced: diff --git a/scripts/run_coverage.sh b/scripts/run_coverage.sh index fc761df0..ee684448 100755 --- a/scripts/run_coverage.sh +++ b/scripts/run_coverage.sh @@ -42,7 +42,18 @@ if [[ -z "${PKGS}" ]]; then fi cd "${WORK_DIR}" -CGO_ENABLED=1 go test -count=1 -covermode=atomic -coverprofile "${OUT_FILE}" ${PKGS} +echo "coverage scope: ${SCOPE}" +echo "coverage workdir: ${WORK_DIR}" +echo "coverage output: ${OUT_FILE}" +echo "packages:" +printf ' %s\n' ${PKGS} + +GO_TEST_FLAGS=(-count=1 -covermode=atomic -coverprofile "${OUT_FILE}") +if [[ "${SCOPE}" == "client" ]]; then + GO_TEST_FLAGS=(-v "${GO_TEST_FLAGS[@]}") +fi + +CGO_ENABLED=1 go test "${GO_TEST_FLAGS[@]}" ${PKGS} go tool cover -func="${OUT_FILE}" | tee "${OUT_DIR}/coverage.txt" go tool cover -html="${OUT_FILE}" -o "${HTML_FILE}"