From 8f2b64eb611a00ff6b79cfae843c1c5eaa35f958 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 9 Jun 2026 14:17:07 -0700 Subject: [PATCH 01/36] fix gecko project creation so that gecko project server config is not required to create a new project --- internal/git/reconcile.go | 13 ++- internal/git/reconcile_test.go | 97 ++++++++++++++++ internal/git/service.go | 10 ++ internal/git/types.go | 12 +- internal/server/http/git/installation_test.go | 56 +++++++++ internal/server/http/git/project.go | 8 ++ internal/server/http/git/register.go | 22 ++-- internal/server/middleware/auth.go | 99 ++++++++++++++++ internal/server/middleware/git_test.go | 108 ++++++++++++++++++ 9 files changed, 405 insertions(+), 20 deletions(-) create mode 100644 internal/git/reconcile_test.go diff --git a/internal/git/reconcile.go b/internal/git/reconcile.go index b98497a..70712ee 100644 --- a/internal/git/reconcile.go +++ b/internal/git/reconcile.go @@ -240,6 +240,12 @@ func (service *ReconcileService) BuildOrganizationStatus(ctx context.Context, or } } installation := buildInstallationStatus(responsePayload, state, identity.Owner) + workflowStage := "" + if strings.TrimSpace(cfg.SrcRepo) == "" { + workflowStage = GitWorkflowStageAwaitingGitHubConnect + } else if installation.Installed { + workflowStage = GitWorkflowStageGitHubConnected + } integrations := ProjectIntegrationStatus{ GitHub: ProjectIntegrationCheck{ Pass: installation.Installed, @@ -247,11 +253,11 @@ func (service *ReconcileService) BuildOrganizationStatus(ctx context.Context, or Storage: deriveStorageIntegrationCheck(buckets, bucketsErr, parts[0], parts[1]), } if strings.TrimSpace(cfg.SrcRepo) == "" { - integrations.GitHub.Reason = "missing_repository_link" + integrations.GitHub.Reason = GitWorkflowStageAwaitingGitHubConnect if responsePayload.AppInstalled { - integrations.GitHub.Details = "GitHub App is installed for this organization, but this project is not linked to a repository yet" + integrations.GitHub.Details = "Project creation is complete. Finish the GitHub connect step to link this project to a repository." } else { - integrations.GitHub.Details = "No GitHub repository is linked to this project" + integrations.GitHub.Details = "Project creation is complete. Connect GitHub for this organization, then finish the GitHub connect step for this project." } } else if !installation.Installed { integrations.GitHub.Reason = "missing_github_connection" @@ -264,6 +270,7 @@ func (service *ReconcileService) BuildOrganizationStatus(ctx context.Context, or Project: parts[1], ResourcePath: ProgramProjectResourcePath(parts[0], parts[1]), Repository: identity, + WorkflowStage: workflowStage, Configured: configured, Integrations: integrations, Accessible: readable, diff --git a/internal/git/reconcile_test.go b/internal/git/reconcile_test.go new file mode 100644 index 0000000..6d104e7 --- /dev/null +++ b/internal/git/reconcile_test.go @@ -0,0 +1,97 @@ +package git + +import ( + "context" + "database/sql" + "encoding/json" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + appconfig "github.com/calypr/gecko/config" + geckodb "github.com/calypr/gecko/internal/db" + "github.com/jmoiron/sqlx" +) + +func TestBuildOrganizationStatusTreatsSetupOnlyProjectAsAwaitingGitHubConnect(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + + projectCfg := appconfig.ProjectConfig{ + Title: "proj-a", + OrgTitle: "TEST", + ProjectTitle: "proj-a", + Description: "setup only", + ContactEmail: "test@example.org", + SrcRepo: "", + } + projectContent, err := json.Marshal(projectCfg) + if err != nil { + t.Fatalf("marshal project config: %v", err) + } + mock.ExpectQuery(`SELECT name, content FROM config_schema\.projects WHERE name=\$1`). + WithArgs("TEST/proj-a"). + WillReturnRows(sqlmock.NewRows([]string{"name", "content"}).AddRow("TEST/proj-a", projectContent)) + + service := &ReconcileService{db: sqlx.NewDb(db, "sqlmock")} + organizationStates := map[string]geckodb.GitOrganizationState{ + "TEST": { + Organization: "TEST", + Installed: true, + RepositorySelection: nullableString("selected"), + UpdatedAt: time.Now(), + }, + } + buckets := map[string]StorageBucket{ + "bucket-a": { + Bucket: "bucket-a", + Resources: []string{"/programs/TEST/projects/proj-a"}, + }, + } + + status, err := service.BuildOrganizationStatus( + context.Background(), + "TEST", + []string{"TEST/proj-a"}, + map[string]geckodb.GitProjectState{}, + organizationStates, + []string{"/programs/TEST/projects/proj-a"}, + buckets, + nil, + ) + if err != nil { + t.Fatalf("build organization status: %v", err) + } + if len(status.Projects) != 1 { + t.Fatalf("expected one project, got %d", len(status.Projects)) + } + project := status.Projects[0] + if project.WorkflowStage != GitWorkflowStageAwaitingGitHubConnect { + t.Fatalf("expected workflow stage %q, got %q", GitWorkflowStageAwaitingGitHubConnect, project.WorkflowStage) + } + if project.Integrations.GitHub.Reason != GitWorkflowStageAwaitingGitHubConnect { + t.Fatalf("expected github reason %q, got %q", GitWorkflowStageAwaitingGitHubConnect, project.Integrations.GitHub.Reason) + } + if project.Integrations.GitHub.Pass { + t.Fatal("expected github integration to be incomplete") + } + if !project.Integrations.Storage.Pass { + t.Fatal("expected storage integration to pass") + } + if project.Configured { + t.Fatal("expected setup-only project to remain unconfigured") + } + if project.Installation.Installed { + t.Fatal("expected project installation to remain unbound before connect") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func nullableString(value string) sql.NullString { + return sql.NullString{String: value, Valid: value != ""} +} diff --git a/internal/git/service.go b/internal/git/service.go index 9ec8f54..da08bee 100644 --- a/internal/git/service.go +++ b/internal/git/service.go @@ -75,6 +75,10 @@ func (service *GitService) RefreshProject(ctx context.Context, projectID string, } func (service *GitService) StatusFromState(projectID string, organization string, project string, cfg appconfig.ProjectConfig, identity GitRepositoryIdentity, state *geckodb.GitProjectState, orgState *geckodb.GitOrganizationState) GitProjectStatusResponse { + workflowStage := "" + if strings.TrimSpace(cfg.SrcRepo) == "" { + workflowStage = GitWorkflowStageAwaitingGitHubConnect + } response := GitProjectStatusResponse{ ProjectID: projectID, Organization: organization, @@ -83,6 +87,7 @@ func (service *GitService) StatusFromState(projectID string, organization string RequestAccessResourcePath: ProgramProjectResourcePath(organization, project), Config: cfg, Repository: identity, + WorkflowStage: workflowStage, InstallationState: GitInstallationNotConnected, SyncState: GitSyncNeverSynced, } @@ -96,10 +101,15 @@ func (service *GitService) StatusFromState(projectID string, organization string } } if state == nil { + if response.WorkflowStage == "" && response.OrganizationAppInstalled && response.OrganizationRepositorySelection == "all" { + response.WorkflowStage = GitWorkflowStageGitHubConnected + response.InstallationState = GitInstallationConnected + } return response } if state.InstallationID.Valid || state.InstallationTarget.Valid { response.InstallationState = GitInstallationConnected + response.WorkflowStage = GitWorkflowStageGitHubConnected } if state.InstallationID.Valid { installationID := state.InstallationID.Int64 diff --git a/internal/git/types.go b/internal/git/types.go index 9f8b4d4..bab6558 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -22,6 +22,9 @@ const ( GitInstallationNotConnected = "not_connected" GitInstallationConnected = "connected" + + GitWorkflowStageAwaitingGitHubConnect = "awaiting_github_connect" + GitWorkflowStageGitHubConnected = "github_connected" ) type GitServiceConfig struct { @@ -53,6 +56,7 @@ type GitProjectStatusResponse struct { RequestAccessResourcePath string `json:"request_access_resource_path,omitempty"` Config appconfig.ProjectConfig `json:"config"` Repository GitRepositoryIdentity `json:"repository"` + WorkflowStage string `json:"workflow_stage,omitempty"` InstallationState string `json:"installation_state"` InstallationID *int64 `json:"installation_id,omitempty"` InstallationTarget string `json:"installation_target,omitempty"` @@ -77,7 +81,6 @@ type GitOrganizationConnectResponse struct { // GitRepositoryInstallationStatus is an alias for domain.GitRepositoryInstallationStatus. type GitRepositoryInstallationStatus = domain.GitRepositoryInstallationStatus - type ProjectIntegrationCheck struct { Pass bool `json:"pass"` Reason string `json:"reason,omitempty"` @@ -94,6 +97,7 @@ type GitOrganizationProjectStatus struct { Project string `json:"project"` ResourcePath string `json:"resource_path"` Repository GitRepositoryIdentity `json:"repository"` + WorkflowStage string `json:"workflow_stage,omitempty"` Configured bool `json:"configured"` Readiness *CalyprProjectReadiness `json:"readiness,omitempty"` Integrations ProjectIntegrationStatus `json:"integrations"` @@ -163,8 +167,6 @@ type CalyprProjectSetupResponse struct { // GitHubInstallationRepository is an alias for domain.GitHubInstallationRepository. type GitHubInstallationRepository = domain.GitHubInstallationRepository - - type GitOrganizationStatusResponse struct { Organization string `json:"organization"` Connected bool `json:"connected"` @@ -310,11 +312,9 @@ type GitUploadSessionResponse struct { // GitHubRepositoryMetadata is an alias for domain.GitHubRepositoryMetadata. type GitHubRepositoryMetadata = domain.GitHubRepositoryMetadata - // HTTPStatusError is an alias for domain.HTTPStatusError. type HTTPStatusError = domain.HTTPStatusError - func NewGitService(config GitServiceConfig) *GitService { if config.GitHubAPIBase == "" { config.GitHubAPIBase = "https://api.github.com" @@ -372,8 +372,6 @@ type StorageBucket = domain.StorageBucket // StorageConfig is an alias for domain.StorageConfig. type StorageConfig = domain.StorageConfig - func ProgramProjectResourcePath(organization, project string) string { return fmt.Sprintf("/programs/%s/projects/%s", organization, project) } - diff --git a/internal/server/http/git/installation_test.go b/internal/server/http/git/installation_test.go index fef4da9..6d68bfc 100644 --- a/internal/server/http/git/installation_test.go +++ b/internal/server/http/git/installation_test.go @@ -441,3 +441,59 @@ func TestGitProjectEditConnectRequiresConnectedOrganization(t *testing.T) { t.Fatalf("unmet sql expectations: %v", err) } } + +func TestGitProjectUpdateRejectsSetupOnlyProjectUntilGitHubConnectCompletes(t *testing.T) { + fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusInternalServerError) + })) + defer fenceServer.Close() + + handler, mock, cleanup := newGitHandlerTestServer(t, fenceServer, nil) + defer cleanup() + + projectCfg := appconfig.ProjectConfig{ + Title: "proj-a", + OrgTitle: "TEST", + ProjectTitle: "proj-a", + Description: "setup only", + ContactEmail: "test@example.org", + SrcRepo: "", + } + projectContent, err := json.Marshal(projectCfg) + if err != nil { + t.Fatalf("marshal project config: %v", err) + } + mock.ExpectQuery(`SELECT name, content FROM config_schema\.projects WHERE name=\$1`). + WithArgs("TEST/proj-a"). + WillReturnRows(sqlmock.NewRows([]string{"name", "content"}).AddRow("TEST/proj-a", projectContent)) + + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/update", handler.handleGitProjectUpdatePOST) + req := httptest.NewRequest(http.MethodPost, "/git/projects/TEST/proj-a/update", nil) + req.Header.Set("Authorization", "Bearer test") + + resp := runGitRequest(t, app, req) + defer resp.Body.Close() + if resp.StatusCode != http.StatusConflict { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 409, got %d: %s", resp.StatusCode, payload) + } + var payload struct { + Error struct { + Message string `json:"message"` + Details map[string]any `json:"details"` + } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if !strings.Contains(payload.Error.Message, "GitHub connection has not been completed") { + t.Fatalf("unexpected error message: %q", payload.Error.Message) + } + if got := payload.Error.Details["workflow_stage"]; got != gitservice.GitWorkflowStageAwaitingGitHubConnect { + t.Fatalf("expected workflow stage %q, got %#v", gitservice.GitWorkflowStageAwaitingGitHubConnect, got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} diff --git a/internal/server/http/git/project.go b/internal/server/http/git/project.go index 2ce4be7..ee93c3e 100644 --- a/internal/server/http/git/project.go +++ b/internal/server/http/git/project.go @@ -66,6 +66,14 @@ func (handler *Handler) resolveGitProject(ctx fiber.Ctx) (string, string, string response.WriteLog(handler.logger) return "", "", "", appconfig.ProjectConfig{}, git.GitRepositoryIdentity{}, response } + if strings.TrimSpace(cfg.SrcRepo) == "" { + response := httputil.NewError("conflict", fmt.Sprintf("GitHub connection has not been completed for %s", projectID), http.StatusConflict, map[string]any{ + "project_id": projectID, + "workflow_stage": git.GitWorkflowStageAwaitingGitHubConnect, + }, nil) + response.WriteLog(handler.logger) + return "", "", "", appconfig.ProjectConfig{}, git.GitRepositoryIdentity{}, response + } identity, err := git.ParseRepositoryIdentity(cfg.SrcRepo) if err != nil { response := httputil.NewError("validation_failed", fmt.Sprintf("invalid src_repo for %s: %s", projectID, err), http.StatusBadRequest, map[string]any{"project_id": projectID, "src_repo": cfg.SrcRepo}, nil) diff --git a/internal/server/http/git/register.go b/internal/server/http/git/register.go index 4571184..e1cbc48 100644 --- a/internal/server/http/git/register.go +++ b/internal/server/http/git/register.go @@ -21,6 +21,8 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler gitGroup.Post("/organizations/:orgTitle/reconcile", servermw.GitOrganizationAuth(handler.Logger, authzHandler), handler.handleGitOrganizationReconcilePOST) projectReadAuth := servermw.GitProjectAuth(handler.Logger, authzHandler) + projectSetupAuth := servermw.GitProjectSetupAuth(handler.Logger, authzHandler) + projectWriteAuth := servermw.GitProjectMutationAuth(handler.Logger, authzHandler, "update") gitGroup.Get("/projects/:orgTitle/:projectTitle", projectReadAuth, handler.handleGitProjectGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/refs", projectReadAuth, handler.handleGitProjectRefsGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/tree", projectReadAuth, handler.handleGitProjectTreeGET) @@ -30,14 +32,14 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler gitGroup.Get("/projects/:orgTitle/:projectTitle/thumbnail", handler.handleGitProjectThumbnailGET) projectGitWrite := gitGroup.Group("/projects/:orgTitle/:projectTitle", servermw.RequireAuthorization(handler.Logger)) - projectGitWrite.Put("/setup", handler.handleCalyprProjectSetupPUT) - projectGitWrite.Put("/storage", handler.handleCalyprProjectStoragePUT) - projectGitWrite.Put("/thumbnail", handler.handleGitProjectThumbnailPUT) - projectGitWrite.Delete("/thumbnail", handler.handleGitProjectThumbnailDELETE) - projectGitWrite.Post("/edit-connect", handler.handleGitProjectEditConnectPOST) - projectGitWrite.Post("/update", handler.handleGitProjectUpdatePOST) - projectGitWrite.Post("/uploads/session", handler.handleGitProjectUploadSessionPOST) - projectGitWrite.Get("/uploads/session/:sessionID", handler.handleGitProjectUploadSessionGET) - projectGitWrite.Post("/uploads/session/:sessionID/files", handler.handleGitProjectUploadSessionFilesPOST) - projectGitWrite.Post("/uploads/session/:sessionID/finalize", handler.handleGitProjectUploadSessionFinalizePOST) + projectGitWrite.Put("/setup", projectSetupAuth, handler.handleCalyprProjectSetupPUT) + projectGitWrite.Put("/storage", projectSetupAuth, handler.handleCalyprProjectStoragePUT) + projectGitWrite.Put("/thumbnail", projectWriteAuth, handler.handleGitProjectThumbnailPUT) + projectGitWrite.Delete("/thumbnail", projectWriteAuth, handler.handleGitProjectThumbnailDELETE) + projectGitWrite.Post("/edit-connect", projectWriteAuth, handler.handleGitProjectEditConnectPOST) + projectGitWrite.Post("/update", projectWriteAuth, handler.handleGitProjectUpdatePOST) + projectGitWrite.Post("/uploads/session", projectWriteAuth, handler.handleGitProjectUploadSessionPOST) + projectGitWrite.Get("/uploads/session/:sessionID", projectWriteAuth, handler.handleGitProjectUploadSessionGET) + projectGitWrite.Post("/uploads/session/:sessionID/files", projectWriteAuth, handler.handleGitProjectUploadSessionFilesPOST) + projectGitWrite.Post("/uploads/session/:sessionID/finalize", projectWriteAuth, handler.handleGitProjectUploadSessionFinalizePOST) } diff --git a/internal/server/middleware/auth.go b/internal/server/middleware/auth.go index c64abb1..0fe23d9 100644 --- a/internal/server/middleware/auth.go +++ b/internal/server/middleware/auth.go @@ -245,6 +245,105 @@ func GitProjectAuth(logger arborist.Logger, jwtHandler ResourceAccessHandler) fi } } +func GitProjectMutationAuth(logger arborist.Logger, authzHandler ResourceAccessHandler, method string) fiber.Handler { + return func(ctx fiber.Ctx) error { + authorizationHeader := ctx.Get("Authorization") + if authorizationHeader == "" { + return writeError(ctx, logger, httputil.NewError(apierror.TypeMissingAuthorization, "Authorization token not provided", http.StatusUnauthorized, nil, nil)) + } + organization := strings.TrimSpace(ctx.Params("orgTitle")) + project := strings.TrimSpace(ctx.Params("projectTitle")) + if organization == "" || project == "" { + return writeError(ctx, logger, httputil.NewError("invalid_request", "organization and project are required", http.StatusBadRequest, nil, nil)) + } + resourcePath := ProgramProjectResourcePath(organization, project) + allowed, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, method, "*", resourcePath) + if err != nil { + if serverErr, ok := err.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) + } + if !allowed { + anyList, listErr := authzHandler.GetAllowedResources(authorizationHeader, method, "*") + if listErr != nil { + if serverErr, ok := listErr.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, listErr.Error(), http.StatusForbidden, nil, nil)) + } + resources, conversionErr := convertAnyToStringSlice(anyList) + if conversionErr != nil { + return writeError(ctx, logger, conversionErr) + } + allowed = resourceListAllowsProjectAdminAction(resources, organization, project) + } + if !allowed { + return writeError(ctx, logger, httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User does not have required %s permission on resource %s", method, resourcePath), http.StatusForbidden, map[string]any{ + "resource": resourcePath, + "method": method, + "organization": organization, + "project": project, + }, nil)) + } + return ctx.Next() + } +} + +func GitProjectSetupAuth(logger arborist.Logger, authzHandler ResourceAccessHandler) fiber.Handler { + return func(ctx fiber.Ctx) error { + authorizationHeader := ctx.Get("Authorization") + if authorizationHeader == "" { + return writeError(ctx, logger, httputil.NewError(apierror.TypeMissingAuthorization, "Authorization token not provided", http.StatusUnauthorized, nil, nil)) + } + organization := strings.TrimSpace(ctx.Params("orgTitle")) + project := strings.TrimSpace(ctx.Params("projectTitle")) + if organization == "" || project == "" { + return writeError(ctx, logger, httputil.NewError("invalid_request", "organization and project are required", http.StatusBadRequest, nil, nil)) + } + projectResource := ProgramProjectResourcePath(organization, project) + projectReadable, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, "read", "*", projectResource) + if err != nil { + if serverErr, ok := err.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) + } + if projectReadable { + return ctx.Next() + } + orgProjectsResource := fmt.Sprintf("/programs/%s/projects", organization) + orgCreateAllowed, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, "create-descendant", "arborist", orgProjectsResource) + if err != nil { + if serverErr, ok := err.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) + } + if orgCreateAllowed { + return ctx.Next() + } + orgResource := fmt.Sprintf("/programs/%s", organization) + orgManageAllowed, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, "manage-owners", "arborist", orgResource) + if err != nil { + if serverErr, ok := err.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) + } + if orgManageAllowed { + return ctx.Next() + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User does not have required setup permission on resource %s", projectResource), http.StatusForbidden, map[string]any{ + "resource": projectResource, + "organization": organization, + "project": project, + "org_resource": orgResource, + "projects_path": orgProjectsResource, + }, nil)) + } +} + func GitOrganizationAuth(logger arborist.Logger, jwtHandler ResourceAccessHandler) fiber.Handler { return func(ctx fiber.Ctx) error { if jwtHandler == nil { diff --git a/internal/server/middleware/git_test.go b/internal/server/middleware/git_test.go index bfcae95..c6cf4f4 100644 --- a/internal/server/middleware/git_test.go +++ b/internal/server/middleware/git_test.go @@ -158,6 +158,114 @@ func TestRequireAuthorizationAllowsBearerHeader(t *testing.T) { } } +func TestGitProjectMutationAuthAllowsExactProjectResource(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/edit-connect", GitProjectMutationAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/org-a/projects/proj-a"}}, "update"), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("POST", "/git/projects/org-a/proj-a/edit-connect", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestGitProjectMutationAuthRejectsDifferentProjectResource(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/edit-connect", GitProjectMutationAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/org-a/projects/other"}}, "update"), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("POST", "/git/projects/org-a/proj-a/edit-connect", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusForbidden { + t.Fatalf("expected 403, got %d", resp.StatusCode) + } +} + +func TestGitProjectMutationAuthAllowsAdminWildcardResource(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/edit-connect", GitProjectMutationAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"*"}}, "update"), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("POST", "/git/projects/org-a/proj-a/edit-connect", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestGitProjectSetupAuthAllowsOrgProjectCreate(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Put("/git/projects/:orgTitle/:projectTitle/setup", GitProjectSetupAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/org-a/projects"}}), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("PUT", "/git/projects/org-a/proj-a/setup", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestGitProjectSetupAuthAllowsOrgManageOwners(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Put("/git/projects/:orgTitle/:projectTitle/setup", GitProjectSetupAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/org-a"}}), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("PUT", "/git/projects/org-a/proj-a/setup", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestGitProjectSetupAuthRejectsMissingProjectAndOrgAccess(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Put("/git/projects/:orgTitle/:projectTitle/setup", GitProjectSetupAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/other"}}), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("PUT", "/git/projects/org-a/proj-a/setup", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusForbidden { + t.Fatalf("expected 403, got %d", resp.StatusCode) + } +} + func TestParseResourceAccessSnapshotPrefersAuthzBlock(t *testing.T) { snapshot, err := parseResourceAccessSnapshot(map[string]any{ "authz": map[string]any{ From f178e3dd69b3a932753dbe24d9c93c8a89e561ae Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 9 Jun 2026 14:37:55 -0700 Subject: [PATCH 02/36] fix gecko github disconnect logic --- internal/server/http/git/installation.go | 36 +++++++++-- internal/server/http/git/installation_test.go | 62 +++++++++++++++++++ 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/internal/server/http/git/installation.go b/internal/server/http/git/installation.go index 2f25cc4..e7d8053 100644 --- a/internal/server/http/git/installation.go +++ b/internal/server/http/git/installation.go @@ -179,11 +179,6 @@ func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { return errResponse.Write(ctx) } } - if strings.TrimSpace(requestBody.RepositoryFullName) == "" { - response := httputil.NewError(apierror.Type("invalid_request"), "repository_full_name is required", http.StatusBadRequest, map[string]any{"organization": organization, "project": project}, nil) - response.WriteLog(handler.logger) - return response.Write(ctx) - } projectID := organization + "/" + project authorizationHeader, tokenErr := servermw.ValidateAuthorizationHeader(ctx.Get("Authorization")) if tokenErr != nil { @@ -198,6 +193,14 @@ func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { errResponse.WriteLog(handler.logger) return errResponse.Write(ctx) } + if strings.TrimSpace(requestBody.RepositoryFullName) == "" { + if err := handler.unbindProjectRepository(connectCtx, projectID, projectCfg); err != nil { + response := httputil.NewError(apierror.TypeDatabaseError, fmt.Sprintf("failed to clear project repository binding: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + return httputil.JSON(git.GitOrganizationConnectResponse{Mode: "disconnected"}, http.StatusOK).Write(ctx) + } orgState, errResponse := handler.loadConnectedOrganizationState(connectCtx, organization, project) if errResponse != nil { errResponse.WriteLog(handler.logger) @@ -358,6 +361,29 @@ func (handler *Handler) bindProjectRepository(ctx context.Context, projectID str return nil } +func (handler *Handler) unbindProjectRepository(ctx context.Context, projectID string, projectCfg appconfig.ProjectConfig) error { + projectCfg.SrcRepo = "" + + tx, err := handler.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("begin project repository unbind transaction: %w", err) + } + defer func() { + _ = tx.Rollback() + }() + + if err := geckodb.ConfigPUTGenericTxContext(ctx, tx, projectID, string(appconfig.TypeProjects), &projectCfg); err != nil { + return fmt.Errorf("update project config: %w", err) + } + if _, err := tx.ExecContext(ctx, `DELETE FROM config_schema.git_project_state WHERE project_id = $1`, projectID); err != nil { + return fmt.Errorf("delete project git state: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit project repository unbind transaction: %w", err) + } + return nil +} + func (handler *Handler) handleGitProjectUpdatePOST(ctx fiber.Ctx) error { organization, project, projectID, cfg, identity, errResponse := handler.resolveGitProject(ctx) if errResponse != nil { diff --git a/internal/server/http/git/installation_test.go b/internal/server/http/git/installation_test.go index 6d68bfc..ce6804b 100644 --- a/internal/server/http/git/installation_test.go +++ b/internal/server/http/git/installation_test.go @@ -442,6 +442,68 @@ func TestGitProjectEditConnectRequiresConnectedOrganization(t *testing.T) { } } +func TestGitProjectEditConnectClearsRepositoryBinding(t *testing.T) { + fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusInternalServerError) + })) + defer fenceServer.Close() + + handler, mock, cleanup := newGitHandlerTestServer(t, fenceServer, nil) + defer cleanup() + + projectCfg := appconfig.ProjectConfig{ + Title: "proj-a", + OrgTitle: "TEST", + ProjectTitle: "proj-a", + SrcRepo: "https://github.com/EllrottLab/git_drs_test", + } + projectContent, err := json.Marshal(projectCfg) + if err != nil { + t.Fatalf("marshal project config: %v", err) + } + mock.ExpectQuery(`SELECT name, content FROM config_schema\.projects WHERE name=\$1`). + WithArgs("TEST/proj-a"). + WillReturnRows(sqlmock.NewRows([]string{"name", "content"}).AddRow("TEST/proj-a", projectContent)) + updatedCfg := projectCfg + updatedCfg.SrcRepo = "" + updatedContent, err := json.Marshal(&updatedCfg) + if err != nil { + t.Fatalf("marshal updated project config: %v", err) + } + mock.ExpectBegin() + mock.ExpectExec(`INSERT INTO config_schema\.projects`). + WithArgs("TEST/proj-a", updatedContent). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`DELETE FROM config_schema\.git_project_state WHERE project_id = \$1`). + WithArgs("TEST/proj-a"). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/edit-connect", handler.handleGitProjectEditConnectPOST) + body := bytes.NewBufferString(`{"repository_full_name":""}`) + req := httptest.NewRequest(http.MethodPost, "/git/projects/TEST/proj-a/edit-connect", body) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + + resp := runGitRequest(t, app, req) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 200, got %d: %s", resp.StatusCode, payload) + } + var payload gitservice.GitOrganizationConnectResponse + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if payload.Mode != "disconnected" { + t.Fatalf("expected disconnected mode, got %+v", payload) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + func TestGitProjectUpdateRejectsSetupOnlyProjectUntilGitHubConnectCompletes(t *testing.T) { fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { writer.WriteHeader(http.StatusInternalServerError) From d661ca4f67998eca99e77cf9daca35553c9be61d Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 10 Jun 2026 08:16:08 -0700 Subject: [PATCH 03/36] beef up security for fence endpoint --- internal/git/service.go | 4 +- internal/integrations/fence/client.go | 4 +- internal/integrations/fence/client_test.go | 8 +- internal/server/http/git/installation.go | 19 ++++- internal/server/http/git/installation_test.go | 75 ++++++++++++++++++- 5 files changed, 99 insertions(+), 11 deletions(-) diff --git a/internal/git/service.go b/internal/git/service.go index da08bee..54af114 100644 --- a/internal/git/service.go +++ b/internal/git/service.go @@ -203,11 +203,11 @@ func (service *GitService) RequestOrganizationInstallationStatus(ctx context.Con return service.fenceAPI.RequestOrganizationInstallationStatus(ctx, authorizationHeader, organization, owner) } -func (service *GitService) ListInstallationRepositories(ctx context.Context, authorizationHeader string, installationID int64) ([]GitHubInstallationRepository, error) { +func (service *GitService) ListInstallationRepositories(ctx context.Context, authorizationHeader string, organization string, owner string, installationID int64) ([]GitHubInstallationRepository, error) { if service.fenceAPI == nil { return nil, fmt.Errorf("fence client is not initialized") } - return service.fenceAPI.ListInstallationRepositories(ctx, authorizationHeader, installationID) + return service.fenceAPI.ListInstallationRepositories(ctx, authorizationHeader, organization, owner, installationID) } func (service *GitService) RequestInstallationStatus(ctx context.Context, authorizationHeader string, organization string, identity GitRepositoryIdentity) (GitRepositoryInstallationStatus, error) { diff --git a/internal/integrations/fence/client.go b/internal/integrations/fence/client.go index 2de6dba..4e9b699 100644 --- a/internal/integrations/fence/client.go +++ b/internal/integrations/fence/client.go @@ -166,10 +166,12 @@ func (c *Client) RequestOrganizationInstallationStatus(ctx context.Context, auth }, nil } -func (c *Client) ListInstallationRepositories(ctx context.Context, authorizationHeader string, installationID int64) ([]domain.GitHubInstallationRepository, error) { +func (c *Client) ListInstallationRepositories(ctx context.Context, authorizationHeader string, organization string, owner string, installationID int64) ([]domain.GitHubInstallationRepository, error) { var payload fenceGitHubInstallationRepositoriesResponse if err := c.requestFenceGitHubBroker(ctx, authorizationHeader, map[string]any{ "action": "installation_repositories", + "organization": organization, + "owner": owner, "installation_id": installationID, }, &payload); err != nil { return nil, err diff --git a/internal/integrations/fence/client_test.go b/internal/integrations/fence/client_test.go index 0387314..c5a35b4 100644 --- a/internal/integrations/fence/client_test.go +++ b/internal/integrations/fence/client_test.go @@ -139,7 +139,7 @@ func TestListInstallationRepositoriesForwardsAuthorizationAndParsesRepositories( defer server.Close() client := NewClient(server.Client(), Config{BaseURL: server.URL}) - repositories, err := client.ListInstallationRepositories(context.Background(), "Bearer user-token", 42) + repositories, err := client.ListInstallationRepositories(context.Background(), "Bearer user-token", "Ellrott_Lab", "EllrottLab", 42) if err != nil { t.Fatalf("list installation repositories: %v", err) } @@ -152,6 +152,12 @@ func TestListInstallationRepositoriesForwardsAuthorizationAndParsesRepositories( if receivedBody["installation_id"] != float64(42) { t.Fatalf("expected installation id in request body, got %#v", receivedBody) } + if receivedBody["organization"] != "Ellrott_Lab" { + t.Fatalf("expected organization in request body, got %#v", receivedBody) + } + if receivedBody["owner"] != "EllrottLab" { + t.Fatalf("expected owner in request body, got %#v", receivedBody) + } if len(repositories) != 1 { t.Fatalf("expected one repository, got %+v", repositories) } diff --git a/internal/server/http/git/installation.go b/internal/server/http/git/installation.go index e7d8053..41f4f5f 100644 --- a/internal/server/http/git/installation.go +++ b/internal/server/http/git/installation.go @@ -124,6 +124,7 @@ func (handler *Handler) handleGitOrganizationConnectPOST(ctx fiber.Ctx) error { } type connectRequest struct { InstallationID *int64 `json:"installation_id"` + GitHubOwner string `json:"github_owner"` } requestBody := connectRequest{} if len(ctx.Body()) > 0 { @@ -137,11 +138,19 @@ func (handler *Handler) handleGitOrganizationConnectPOST(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } + githubOwner := strings.TrimSpace(requestBody.GitHubOwner) + if githubOwner == "" { + response := httputil.NewError(apierror.Type("invalid_request"), "github_owner is required", http.StatusBadRequest, map[string]any{"organization": organization}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } connectCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() repositories, err := handler.gitService.ListInstallationRepositories( connectCtx, authorizationHeader, + organization, + githubOwner, *requestBody.InstallationID, ) if err != nil { @@ -206,7 +215,11 @@ func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { errResponse.WriteLog(handler.logger) return errResponse.Write(ctx) } - repositories, errResponse := handler.listConnectedInstallationRepositories(connectCtx, authorizationHeader, organization, project, orgState.InstallationID.Int64) + owner := organization + if orgState.InstallationTarget.Valid { + owner = strings.TrimSpace(orgState.InstallationTarget.String) + } + repositories, errResponse := handler.listConnectedInstallationRepositories(connectCtx, authorizationHeader, organization, project, owner, orgState.InstallationID.Int64) if errResponse != nil { errResponse.WriteLog(handler.logger) return errResponse.Write(ctx) @@ -307,8 +320,8 @@ func (handler *Handler) loadConnectedOrganizationState(ctx context.Context, orga return orgState, nil } -func (handler *Handler) listConnectedInstallationRepositories(ctx context.Context, authorizationHeader string, organization string, project string, installationID int64) ([]git.GitHubInstallationRepository, *httputil.ErrorResponse) { - repositories, err := handler.gitService.ListInstallationRepositories(ctx, authorizationHeader, installationID) +func (handler *Handler) listConnectedInstallationRepositories(ctx context.Context, authorizationHeader string, organization string, project string, owner string, installationID int64) ([]git.GitHubInstallationRepository, *httputil.ErrorResponse) { + repositories, err := handler.gitService.ListInstallationRepositories(ctx, authorizationHeader, organization, owner, installationID) if err != nil { if statusErr, ok := err.(*git.HTTPStatusError); ok { response := httputil.NewError(apierror.Type(statusErr.Code), statusErr.Message, statusErr.StatusCode, map[string]any{"organization": organization, "project": project, "installation_id": installationID}, nil) diff --git a/internal/server/http/git/installation_test.go b/internal/server/http/git/installation_test.go index ce6804b..4e22cc6 100644 --- a/internal/server/http/git/installation_test.go +++ b/internal/server/http/git/installation_test.go @@ -337,10 +337,16 @@ func TestGitProjectEditConnectUpdatesProjectConfigAndState(t *testing.T) { payload, _ := io.ReadAll(resp.Body) t.Fatalf("expected 200, got %d: %s", resp.StatusCode, payload) } - if receivedBody["action"] != "installation_repositories" { - t.Fatalf("expected installation_repositories action, got %#v", receivedBody) - } - var payload gitservice.GitOrganizationConnectResponse + if receivedBody["action"] != "installation_repositories" { + t.Fatalf("expected installation_repositories action, got %#v", receivedBody) + } + if receivedBody["organization"] != "TEST" { + t.Fatalf("expected organization in fence request body, got %#v", receivedBody) + } + if receivedBody["owner"] != "EllrottLab" { + t.Fatalf("expected owner in fence request body, got %#v", receivedBody) + } + var payload gitservice.GitOrganizationConnectResponse if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { t.Fatalf("decode response: %v", err) } @@ -352,6 +358,67 @@ func TestGitProjectEditConnectUpdatesProjectConfigAndState(t *testing.T) { } } +func TestGitOrganizationConnectRequiresGitHubOwner(t *testing.T) { + handler, _, cleanup := newGitHandlerTestServer(t, nil, nil) + defer cleanup() + + app := fiber.New() + app.Post("/git/organizations/:orgTitle/connect", handler.handleGitOrganizationConnectPOST) + req := httptest.NewRequest(http.MethodPost, "/git/organizations/TEST/connect", bytes.NewBufferString(`{"installation_id":42}`)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + + resp := runGitRequest(t, app, req) + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 400, got %d: %s", resp.StatusCode, payload) + } +} + +func TestGitOrganizationConnectForwardsGitHubOwner(t *testing.T) { + var receivedBody map[string]any + fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if err := json.NewDecoder(request.Body).Decode(&receivedBody); err != nil { + t.Fatalf("decode fence request body: %v", err) + } + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]any{ + "installation_id": 42, + "repositories": []map[string]any{{ + "id": 101, + "name": "git_drs_test", + "full_name": "EllrottLab/git_drs_test", + "html_url": "https://github.com/EllrottLab/git_drs_test", + "clone_url": "https://github.com/EllrottLab/git_drs_test.git", + }}, + }) + })) + defer fenceServer.Close() + + handler, _, cleanup := newGitHandlerTestServer(t, fenceServer, nil) + defer cleanup() + + app := fiber.New() + app.Post("/git/organizations/:orgTitle/connect", handler.handleGitOrganizationConnectPOST) + req := httptest.NewRequest(http.MethodPost, "/git/organizations/TEST/connect", bytes.NewBufferString(`{"installation_id":42,"github_owner":"EllrottLab"}`)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + + resp := runGitRequest(t, app, req) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 200, got %d: %s", resp.StatusCode, payload) + } + if receivedBody["organization"] != "TEST" { + t.Fatalf("expected organization in fence request body, got %#v", receivedBody) + } + if receivedBody["owner"] != "EllrottLab" { + t.Fatalf("expected github owner in fence request body, got %#v", receivedBody) + } +} + func TestGitProjectEditConnectRejectsRepositoryOutsideInstallation(t *testing.T) { fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { writer.Header().Set("Content-Type", "application/json") From e53fe469c246ce2d32db2043e7c16495b8f81f1a Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 10 Jun 2026 09:14:47 -0700 Subject: [PATCH 04/36] patch auth --- internal/server/http/git/register.go | 4 +++- internal/server/middleware/auth.go | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/server/http/git/register.go b/internal/server/http/git/register.go index e1cbc48..80d6786 100644 --- a/internal/server/http/git/register.go +++ b/internal/server/http/git/register.go @@ -32,7 +32,9 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler gitGroup.Get("/projects/:orgTitle/:projectTitle/thumbnail", handler.handleGitProjectThumbnailGET) projectGitWrite := gitGroup.Group("/projects/:orgTitle/:projectTitle", servermw.RequireAuthorization(handler.Logger)) - projectGitWrite.Put("/setup", projectSetupAuth, handler.handleCalyprProjectSetupPUT) + // Setup must stay auth-only so a brand-new organization can be bootstrapped + // before any org/project Arborist resources exist for the caller. + projectGitWrite.Put("/setup", handler.handleCalyprProjectSetupPUT) projectGitWrite.Put("/storage", projectSetupAuth, handler.handleCalyprProjectStoragePUT) projectGitWrite.Put("/thumbnail", projectWriteAuth, handler.handleGitProjectThumbnailPUT) projectGitWrite.Delete("/thumbnail", projectWriteAuth, handler.handleGitProjectThumbnailDELETE) diff --git a/internal/server/middleware/auth.go b/internal/server/middleware/auth.go index 0fe23d9..199afd9 100644 --- a/internal/server/middleware/auth.go +++ b/internal/server/middleware/auth.go @@ -290,6 +290,11 @@ func GitProjectMutationAuth(logger arborist.Logger, authzHandler ResourceAccessH } } +// GitProjectSetupAuth is for setup-adjacent routes that operate on an existing +// project or organization context, such as storage configuration. Do not place +// this in front of the bootstrap /git/projects/:orgTitle/:projectTitle/setup +// route because Arborist must decide whether the caller can create or attach to +// missing org/project resources. func GitProjectSetupAuth(logger arborist.Logger, authzHandler ResourceAccessHandler) fiber.Handler { return func(ctx fiber.Ctx) error { authorizationHeader := ctx.Get("Authorization") From 00fa23f37db8512b5fcc67ec489406d1cc9ee9d6 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 10 Jun 2026 09:35:08 -0700 Subject: [PATCH 05/36] fix an issue where git org/repo were not persisted in gecko project setup when defined in the frontend --- internal/server/http/git/installation.go | 62 ++++++++++++++++++ internal/server/http/git/installation_test.go | 63 +++++++++++++++---- 2 files changed, 113 insertions(+), 12 deletions(-) diff --git a/internal/server/http/git/installation.go b/internal/server/http/git/installation.go index 41f4f5f..f044806 100644 --- a/internal/server/http/git/installation.go +++ b/internal/server/http/git/installation.go @@ -146,6 +146,32 @@ func (handler *Handler) handleGitOrganizationConnectPOST(ctx fiber.Ctx) error { } connectCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() + installation, err := handler.gitService.RequestOrganizationInstallationStatus( + connectCtx, + authorizationHeader, + organization, + githubOwner, + ) + if err != nil { + if statusErr, ok := err.(*git.HTTPStatusError); ok { + response := httputil.NewError(apierror.Type(statusErr.Code), statusErr.Message, statusErr.StatusCode, map[string]any{"organization": organization, "github_owner": githubOwner}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + response := httputil.NewError(apierror.Type("integration_error"), fmt.Sprintf("failed to load GitHub organization installation status: %s", err), http.StatusBadGateway, map[string]any{"organization": organization, "github_owner": githubOwner}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + if !installation.Installed || installation.InstallationID == nil || *installation.InstallationID <= 0 { + response := httputil.NewError(apierror.Type("conflict"), "organization is not connected to the GitHub App", http.StatusConflict, map[string]any{"organization": organization, "github_owner": githubOwner}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + if err := handler.persistConnectedOrganizationState(organization, githubOwner, installation); err != nil { + response := httputil.NewError(apierror.TypeDatabaseError, fmt.Sprintf("failed to persist organization git state: %s", err), http.StatusInternalServerError, map[string]any{"organization": organization, "github_owner": githubOwner}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } repositories, err := handler.gitService.ListInstallationRepositories( connectCtx, authorizationHeader, @@ -170,6 +196,42 @@ func (handler *Handler) handleGitOrganizationConnectPOST(ctx fiber.Ctx) error { }, http.StatusOK).Write(ctx) } +func (handler *Handler) persistConnectedOrganizationState(organization string, githubOwner string, installation git.GitRepositoryInstallationStatus) error { + now := time.Now().UTC() + state := geckodb.GitOrganizationState{ + Organization: organization, + Installed: installation.Installed, + UpdatedAt: now, + LastSeenAt: sql.NullTime{Time: now, Valid: true}, + ConfiguredAt: sql.NullTime{Time: now, Valid: installation.Installed}, + LastError: sql.NullString{}, + } + if installation.InstallationID != nil && *installation.InstallationID > 0 { + state.InstallationID = sql.NullInt64{Int64: *installation.InstallationID, Valid: true} + } + target := strings.TrimSpace(installation.Target) + if target == "" { + target = strings.TrimSpace(githubOwner) + } + if target != "" { + state.InstallationTarget = sql.NullString{String: target, Valid: true} + } + targetType := strings.TrimSpace(installation.TargetType) + if targetType == "" && target != "" { + targetType = "Organization" + } + if targetType != "" { + state.InstallationTargetType = sql.NullString{String: targetType, Valid: true} + } + if strings.TrimSpace(installation.HTMLURL) != "" { + state.HTMLURL = sql.NullString{String: strings.TrimSpace(installation.HTMLURL), Valid: true} + } + if strings.TrimSpace(installation.RepositorySelection) != "" { + state.RepositorySelection = sql.NullString{String: strings.TrimSpace(installation.RepositorySelection), Valid: true} + } + return geckodb.UpsertGitOrganizationState(handler.db, state) +} + func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { organization := strings.TrimSpace(ctx.Params("orgTitle")) project := strings.TrimSpace(ctx.Params("projectTitle")) diff --git a/internal/server/http/git/installation_test.go b/internal/server/http/git/installation_test.go index 4e22cc6..1812f16 100644 --- a/internal/server/http/git/installation_test.go +++ b/internal/server/http/git/installation_test.go @@ -40,11 +40,17 @@ func newGitHandlerTestServer(t *testing.T, fenceServer *httptest.Server, githubT if githubTransport != nil { githubClient.Transport = githubTransport } + fenceClient := http.DefaultClient + fenceBaseURL := "" + if fenceServer != nil { + fenceClient = fenceServer.Client() + fenceBaseURL = fenceServer.URL + } gitSvc := gitservice.NewGitService(gitservice.GitServiceConfig{ DataDir: t.TempDir(), GitHubAPIBase: "https://api.github.com", HTTPClient: githubClient, - FenceClient: intfence.NewClient(fenceServer.Client(), intfence.Config{BaseURL: fenceServer.URL}), + FenceClient: intfence.NewClient(fenceClient, intfence.Config{BaseURL: fenceBaseURL}), }) handler := &Handler{ Handler: &shared.Handler{}, @@ -383,22 +389,52 @@ func TestGitOrganizationConnectForwardsGitHubOwner(t *testing.T) { t.Fatalf("decode fence request body: %v", err) } writer.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(writer).Encode(map[string]any{ - "installation_id": 42, - "repositories": []map[string]any{{ - "id": 101, - "name": "git_drs_test", - "full_name": "EllrottLab/git_drs_test", - "html_url": "https://github.com/EllrottLab/git_drs_test", - "clone_url": "https://github.com/EllrottLab/git_drs_test.git", - }}, - }) + switch receivedBody["action"] { + case "organization_installation": + _ = json.NewEncoder(writer).Encode(map[string]any{ + "installed": true, + "installation_id": 42, + "target": "EllrottLab", + "target_type": "Organization", + "html_url": "https://github.com/organizations/EllrottLab/settings/installations/42", + "repository_selection": "selected", + }) + case "installation_repositories": + _ = json.NewEncoder(writer).Encode(map[string]any{ + "installation_id": 42, + "repositories": []map[string]any{{ + "id": 101, + "name": "git_drs_test", + "full_name": "EllrottLab/git_drs_test", + "html_url": "https://github.com/EllrottLab/git_drs_test", + "clone_url": "https://github.com/EllrottLab/git_drs_test.git", + }}, + }) + default: + t.Fatalf("unexpected action: %#v", receivedBody["action"]) + } })) defer fenceServer.Close() - handler, _, cleanup := newGitHandlerTestServer(t, fenceServer, nil) + handler, mock, cleanup := newGitHandlerTestServer(t, fenceServer, nil) defer cleanup() + mock.ExpectExec(`INSERT INTO config_schema\.git_organization_state`). + WithArgs( + "TEST", + true, + int64(42), + "Organization", + "EllrottLab", + "https://github.com/organizations/EllrottLab/settings/installations/42", + "selected", + sqlmock.AnyArg(), + sqlmock.AnyArg(), + sqlmock.AnyArg(), + sql.NullString{}, + ). + WillReturnResult(sqlmock.NewResult(1, 1)) + app := fiber.New() app.Post("/git/organizations/:orgTitle/connect", handler.handleGitOrganizationConnectPOST) req := httptest.NewRequest(http.MethodPost, "/git/organizations/TEST/connect", bytes.NewBufferString(`{"installation_id":42,"github_owner":"EllrottLab"}`)) @@ -417,6 +453,9 @@ func TestGitOrganizationConnectForwardsGitHubOwner(t *testing.T) { if receivedBody["owner"] != "EllrottLab" { t.Fatalf("expected github owner in fence request body, got %#v", receivedBody) } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } } func TestGitProjectEditConnectRejectsRepositoryOutsideInstallation(t *testing.T) { From 3d7be0695fc27e6a6c310365ce84c4509e6f8399 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 10 Jun 2026 13:05:56 -0700 Subject: [PATCH 06/36] add support for edit, add, delete connnections in github connection workflow --- internal/server/http/git/installation.go | 36 ++++++++++++++++++- internal/server/http/git/installation_test.go | 12 ++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/internal/server/http/git/installation.go b/internal/server/http/git/installation.go index f044806..868246e 100644 --- a/internal/server/http/git/installation.go +++ b/internal/server/http/git/installation.go @@ -272,7 +272,7 @@ func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { } return httputil.JSON(git.GitOrganizationConnectResponse{Mode: "disconnected"}, http.StatusOK).Write(ctx) } - orgState, errResponse := handler.loadConnectedOrganizationState(connectCtx, organization, project) + orgState, errResponse := handler.loadOrBootstrapConnectedOrganizationState(connectCtx, authorizationHeader, organization, project, requestBody.RepositoryFullName) if errResponse != nil { errResponse.WriteLog(handler.logger) return errResponse.Write(ctx) @@ -306,6 +306,40 @@ func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { return httputil.JSON(git.GitOrganizationConnectResponse{Mode: "connected"}, http.StatusOK).Write(ctx) } +func (handler *Handler) loadOrBootstrapConnectedOrganizationState(ctx context.Context, authorizationHeader string, organization string, project string, repositoryFullName string) (*geckodb.GitOrganizationState, *httputil.ErrorResponse) { + orgState, errResponse := handler.loadConnectedOrganizationState(ctx, organization, project) + if errResponse == nil { + return orgState, nil + } + if errResponse.Error.Code != http.StatusConflict { + return nil, errResponse + } + + identity, err := parseRequestedRepositoryIdentity(repositoryFullName) + if err != nil { + response := httputil.NewError(apierror.Type("invalid_request"), fmt.Sprintf("invalid repository_full_name %q: %s", repositoryFullName, err), http.StatusBadRequest, map[string]any{"organization": organization, "project": project}, nil) + return nil, response + } + installation, installErr := handler.gitService.RequestOrganizationInstallationStatus(ctx, authorizationHeader, organization, identity.Owner) + if installErr != nil { + if statusErr, ok := installErr.(*git.HTTPStatusError); ok { + response := httputil.NewError(apierror.Type(statusErr.Code), statusErr.Message, statusErr.StatusCode, map[string]any{"organization": organization, "project": project, "github_owner": identity.Owner}, nil) + return nil, response + } + response := httputil.NewError(apierror.Type("integration_error"), fmt.Sprintf("failed to load GitHub organization installation status: %s", installErr), http.StatusBadGateway, map[string]any{"organization": organization, "project": project, "github_owner": identity.Owner}, nil) + return nil, response + } + if !installation.Installed || installation.InstallationID == nil || *installation.InstallationID <= 0 { + response := httputil.NewError(apierror.Type("conflict"), "organization is not connected to the GitHub App", http.StatusConflict, map[string]any{"organization": organization, "project": project, "github_owner": identity.Owner}, nil) + return nil, response + } + if err := handler.persistConnectedOrganizationState(organization, identity.Owner, installation); err != nil { + response := httputil.NewError(apierror.TypeDatabaseError, fmt.Sprintf("failed to persist organization git state: %s", err), http.StatusInternalServerError, map[string]any{"organization": organization, "project": project, "github_owner": identity.Owner}, nil) + return nil, response + } + return handler.loadConnectedOrganizationState(ctx, organization, project) +} + func normalizeInstallationRepository(repositoryFullName string, repositories []git.GitHubInstallationRepository, organization string, project string) (git.GitRepositoryIdentity, bool, *httputil.ErrorResponse) { requested := strings.TrimSpace(repositoryFullName) for _, repository := range repositories { diff --git a/internal/server/http/git/installation_test.go b/internal/server/http/git/installation_test.go index 1812f16..1f9828d 100644 --- a/internal/server/http/git/installation_test.go +++ b/internal/server/http/git/installation_test.go @@ -510,7 +510,17 @@ func TestGitProjectEditConnectRejectsRepositoryOutsideInstallation(t *testing.T) func TestGitProjectEditConnectRequiresConnectedOrganization(t *testing.T) { fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - writer.WriteHeader(http.StatusInternalServerError) + var receivedBody map[string]any + if err := json.NewDecoder(request.Body).Decode(&receivedBody); err != nil { + t.Fatalf("decode fence request body: %v", err) + } + if receivedBody["action"] != "organization_installation" { + t.Fatalf("expected organization_installation action, got %#v", receivedBody["action"]) + } + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]any{ + "installed": false, + }) })) defer fenceServer.Close() From bc718281553e7dba7fb0a12d821476658918f234 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Sat, 13 Jun 2026 18:18:24 -0700 Subject: [PATCH 07/36] add presentation route --- config/presentationConfig.go | 307 ++++++++++ config/presentationConfig_test.go | 38 ++ docs/docs.go | 199 ++++++- docs/pr-feature-project-config.md | 545 ++++++++++++++++++ docs/swagger.json | 201 ++++++- docs/swagger.yaml | 145 ++++- internal/presentation/interface.go | 8 + internal/presentation/store.go | 70 +++ internal/presentation/store_test.go | 44 ++ internal/presentation/types.go | 12 + internal/server/http/directory/directory.go | 128 ---- .../server/http/directory/directory_test.go | 55 -- internal/server/http/directory/handler.go | 23 - internal/server/http/directory/register.go | 18 - internal/server/http/git/handler.go | 45 +- .../server/http/git/presentation_config.go | 64 ++ .../http/git/presentation_config_test.go | 239 ++++++++ internal/server/http/git/register.go | 5 + internal/server/http/register.go | 2 - internal/server/http/shared/handler.go | 61 +- internal/server/server.go | 42 +- main.go | 2 + 22 files changed, 1907 insertions(+), 346 deletions(-) create mode 100644 config/presentationConfig.go create mode 100644 config/presentationConfig_test.go create mode 100644 docs/pr-feature-project-config.md create mode 100644 internal/presentation/interface.go create mode 100644 internal/presentation/store.go create mode 100644 internal/presentation/store_test.go create mode 100644 internal/presentation/types.go delete mode 100644 internal/server/http/directory/directory.go delete mode 100644 internal/server/http/directory/directory_test.go delete mode 100644 internal/server/http/directory/handler.go delete mode 100644 internal/server/http/directory/register.go create mode 100644 internal/server/http/git/presentation_config.go create mode 100644 internal/server/http/git/presentation_config_test.go diff --git a/config/presentationConfig.go b/config/presentationConfig.go new file mode 100644 index 0000000..1aa5d51 --- /dev/null +++ b/config/presentationConfig.go @@ -0,0 +1,307 @@ +package config + +import ( + "bytes" + "fmt" + "net/url" + "strings" + + "golang.org/x/net/html" +) + +type PresentationConfig struct { + PresentationConfig string `json:"presentationConfig"` +} + +func (p PresentationConfig) IsZero() bool { + return strings.TrimSpace(p.PresentationConfig) == "" +} + +func (p *PresentationConfig) Validate() error { + if p == nil { + return fmt.Errorf("presentation config is required") + } + sanitized, err := sanitizePresentationHTML(p.PresentationConfig) + if err != nil { + return err + } + p.PresentationConfig = sanitized + return nil +} + +func sanitizePresentationHTML(raw string) (string, error) { + if strings.TrimSpace(raw) == "" { + return "", nil + } + if err := validatePresentationHTML(raw); err != nil { + return "", err + } + doc, err := html.Parse(strings.NewReader(raw)) + if err != nil { + return "", fmt.Errorf("presentationConfig must contain valid HTML: %w", err) + } + + container := &html.Node{Type: html.ElementNode, Data: "div"} + source := findHTMLElement(doc, "body") + if source == nil { + source = doc + } + for child := source.FirstChild; child != nil; child = child.NextSibling { + sanitizePresentationNode(container, child) + } + + var buf bytes.Buffer + for child := container.FirstChild; child != nil; child = child.NextSibling { + if err := html.Render(&buf, child); err != nil { + return "", fmt.Errorf("render sanitized presentation HTML: %w", err) + } + } + return strings.TrimSpace(buf.String()), nil +} + +func validatePresentationHTML(raw string) error { + tokenizer := html.NewTokenizer(strings.NewReader(raw)) + var stack []string + + for { + switch tokenizer.Next() { + case html.ErrorToken: + if err := tokenizer.Err(); err != nil { + if err.Error() == "EOF" { + if len(stack) != 0 { + return fmt.Errorf("presentationConfig must contain balanced HTML tags") + } + return nil + } + return fmt.Errorf("presentationConfig must contain valid HTML: %w", err) + } + if len(stack) != 0 { + return fmt.Errorf("presentationConfig must contain balanced HTML tags") + } + return nil + case html.StartTagToken: + tagName, hasAttr := tokenizer.TagName() + tag := strings.ToLower(string(tagName)) + if !presentationVoidElements[tag] { + stack = append(stack, tag) + } + if hasAttr { + for { + _, _, more := tokenizer.TagAttr() + if !more { + break + } + } + } + case html.SelfClosingTagToken: + if _, hasAttr := tokenizer.TagName(); hasAttr { + for { + _, _, more := tokenizer.TagAttr() + if !more { + break + } + } + } + case html.EndTagToken: + tagName, _ := tokenizer.TagName() + tag := strings.ToLower(string(tagName)) + if presentationVoidElements[tag] { + return fmt.Errorf("presentationConfig must not close void element <%s>", tag) + } + if len(stack) == 0 || stack[len(stack)-1] != tag { + return fmt.Errorf("presentationConfig must contain balanced HTML tags") + } + stack = stack[:len(stack)-1] + } + } +} + +func sanitizePresentationNode(parent *html.Node, node *html.Node) { + switch node.Type { + case html.TextNode: + parent.AppendChild(&html.Node{Type: html.TextNode, Data: node.Data}) + case html.ElementNode: + tag := strings.ToLower(node.Data) + if presentationDropContentTags[tag] { + return + } + if !presentationAllowedTags[tag] { + for child := node.FirstChild; child != nil; child = child.NextSibling { + sanitizePresentationNode(parent, child) + } + return + } + + sanitized := &html.Node{Type: html.ElementNode, Data: tag} + for _, attr := range sanitizePresentationAttrs(tag, node.Attr) { + sanitized.Attr = append(sanitized.Attr, attr) + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + sanitizePresentationNode(sanitized, child) + } + parent.AppendChild(sanitized) + } +} + +func sanitizePresentationAttrs(tag string, attrs []html.Attribute) []html.Attribute { + sanitized := make([]html.Attribute, 0, len(attrs)) + hasRel := false + hasHref := false + targetBlank := false + + for _, attr := range attrs { + key := strings.ToLower(strings.TrimSpace(attr.Key)) + if !presentationAttrAllowed(tag, key) { + continue + } + value := strings.TrimSpace(attr.Val) + switch key { + case "href": + if !isSafePresentationURL(value, false) { + continue + } + hasHref = true + case "src": + if !isSafePresentationURL(value, true) { + continue + } + case "target": + if value == "_blank" { + targetBlank = true + } + case "rel": + hasRel = true + } + sanitized = append(sanitized, html.Attribute{Key: key, Val: value}) + } + + if tag == "a" && !hasHref { + filtered := sanitized[:0] + for _, attr := range sanitized { + if attr.Key == "target" || attr.Key == "rel" { + continue + } + filtered = append(filtered, attr) + } + sanitized = filtered + } else if tag == "a" && targetBlank && !hasRel { + sanitized = append(sanitized, html.Attribute{Key: "rel", Val: "noopener noreferrer"}) + } + return sanitized +} + +func presentationAttrAllowed(tag string, key string) bool { + if presentationGlobalAttrs[key] { + return true + } + if strings.HasPrefix(key, "aria-") || strings.HasPrefix(key, "data-") { + return true + } + if allowed := presentationTagAttrs[tag]; allowed != nil { + return allowed[key] + } + return false +} + +func isSafePresentationURL(raw string, allowDataImage bool) bool { + if raw == "" { + return true + } + lower := strings.ToLower(raw) + if strings.HasPrefix(lower, "#") || strings.HasPrefix(lower, "/") || strings.HasPrefix(lower, "./") || strings.HasPrefix(lower, "../") { + return true + } + if allowDataImage && strings.HasPrefix(lower, "data:image/") { + return true + } + parsed, err := url.Parse(raw) + if err != nil { + return false + } + if parsed.Scheme == "" { + return true + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https", "mailto", "tel": + return true + default: + return false + } +} + +func findHTMLElement(node *html.Node, tag string) *html.Node { + if node == nil { + return nil + } + if node.Type == html.ElementNode && strings.EqualFold(node.Data, tag) { + return node + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + if found := findHTMLElement(child, tag); found != nil { + return found + } + } + return nil +} + +var presentationAllowedTags = map[string]bool{ + "a": true, "article": true, "aside": true, "b": true, "blockquote": true, + "br": true, "caption": true, "code": true, "dd": true, "div": true, + "dl": true, "dt": true, "em": true, "figcaption": true, "figure": true, + "footer": true, "h1": true, "h2": true, "h3": true, "h4": true, + "h5": true, "h6": true, "header": true, "hr": true, "i": true, + "img": true, "li": true, "main": true, "mark": true, "ol": true, + "p": true, "pre": true, "section": true, "small": true, "span": true, + "strong": true, "sub": true, "sup": true, "table": true, "tbody": true, + "td": true, "tfoot": true, "th": true, "thead": true, "tr": true, + "u": true, "ul": true, +} + +var presentationDropContentTags = map[string]bool{ + "embed": true, + "iframe": true, + "link": true, + "meta": true, + "noscript": true, + "object": true, + "script": true, + "style": true, +} + +var presentationVoidElements = map[string]bool{ + "area": true, "base": true, "br": true, "col": true, "embed": true, + "hr": true, "img": true, "input": true, "link": true, "meta": true, + "param": true, "source": true, "track": true, "wbr": true, +} + +var presentationGlobalAttrs = map[string]bool{ + "class": true, + "dir": true, + "id": true, + "lang": true, + "role": true, + "title": true, +} + +var presentationTagAttrs = map[string]map[string]bool{ + "a": { + "href": true, + "rel": true, + "target": true, + }, + "img": { + "alt": true, + "height": true, + "src": true, + "width": true, + }, + "td": { + "colspan": true, + "rowspan": true, + }, + "th": { + "colspan": true, + "rowspan": true, + "scope": true, + }, +} diff --git a/config/presentationConfig_test.go b/config/presentationConfig_test.go new file mode 100644 index 0000000..9adbe9f --- /dev/null +++ b/config/presentationConfig_test.go @@ -0,0 +1,38 @@ +package config + +import "testing" + +func TestPresentationConfigValidateSanitizesUnsafeHTML(t *testing.T) { + cfg := &PresentationConfig{ + PresentationConfig: `
link

Hello

`, + } + + if err := cfg.Validate(); err != nil { + t.Fatalf("validate failed: %v", err) + } + + if cfg.PresentationConfig != `
link

Hello

` { + t.Fatalf("unexpected sanitized HTML: %q", cfg.PresentationConfig) + } +} + +func TestPresentationConfigValidateRejectsMalformedHTML(t *testing.T) { + cfg := &PresentationConfig{ + PresentationConfig: `

broken

`, + } + + if err := cfg.Validate(); err == nil { + t.Fatal("expected malformed HTML validation error") + } +} + +func TestPresentationConfigValidateAllowsEmptyHTML(t *testing.T) { + cfg := &PresentationConfig{} + + if err := cfg.Validate(); err != nil { + t.Fatalf("validate failed: %v", err) + } + if cfg.PresentationConfig != "" { + t.Fatalf("expected empty presentationConfig, got %q", cfg.PresentationConfig) + } +} diff --git a/docs/docs.go b/docs/docs.go index f26d636..bb80cbb 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -215,56 +215,213 @@ const docTemplate = `{ } } }, - "/dir/{projectId}": { + "/git/projects/{orgTitle}/{projectTitle}/presentationConfig": { "get": { - "description": "Retrieve directory details for the given project ID and Directory path", + "description": "Retrieve the sanitized presentation HTML configured for a project. Returns empty content when no presentation has been configured.", "produces": [ "application/json" ], "tags": [ - "Directory" + "Git" ], - "summary": "Retrieve directory information for a project", + "summary": "Get project presentation HTML", "parameters": [ { "type": "string", - "description": "Project ID (format: program-project)", - "name": "projectId", + "description": "Organization title", + "name": "orgTitle", "in": "path", "required": true }, { "type": "string", - "description": "Directory Path (e.g., /data/my-dir)", - "name": "directory_path", - "in": "query", + "description": "Project title", + "name": "projectTitle", + "in": "path", "required": true } ], "responses": { "200": { - "description": "Directory information", + "description": "Presentation configuration", "schema": { - "type": "object", - "additionalProperties": true + "$ref": "#/definitions/config.PresentationConfig" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "500": { + "description": "Server error", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + } + } + }, + "post": { + "description": "Validate, sanitize, and persist presentation HTML for a project.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Git" + ], + "summary": "Create project presentation HTML", + "parameters": [ + { + "type": "string", + "description": "Organization title", + "name": "orgTitle", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Project title", + "name": "projectTitle", + "in": "path", + "required": true + }, + { + "description": "Presentation HTML payload", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + } + ], + "responses": { + "200": { + "description": "Persisted sanitized presentation configuration", + "schema": { + "$ref": "#/definitions/config.PresentationConfig" } }, "400": { - "description": "Invalid request body or Directory path", + "description": "Invalid request body or malformed HTML", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } }, "403": { - "description": "User is not allowed on any resource path", + "description": "Forbidden", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } }, "500": { "description": "Server error", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + } + } + }, + "put": { + "description": "Validate, sanitize, and persist presentation HTML for a project.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Git" + ], + "summary": "Update project presentation HTML", + "parameters": [ + { + "type": "string", + "description": "Organization title", + "name": "orgTitle", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Project title", + "name": "projectTitle", + "in": "path", + "required": true + }, + { + "description": "Presentation HTML payload", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + } + ], + "responses": { + "200": { + "description": "Persisted sanitized presentation configuration", + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + }, + "400": { + "description": "Invalid request body or malformed HTML", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "500": { + "description": "Server error", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } } } @@ -1043,6 +1200,14 @@ const docTemplate = `{ } } }, + "config.PresentationConfig": { + "type": "object", + "properties": { + "presentationConfig": { + "type": "string" + } + } + }, "config.ConfigItem": { "type": "object", "properties": { diff --git a/docs/pr-feature-project-config.md b/docs/pr-feature-project-config.md new file mode 100644 index 0000000..79a87ab --- /dev/null +++ b/docs/pr-feature-project-config.md @@ -0,0 +1,545 @@ +# PR Notes: `feature/project-config` + +## What This Branch Actually Does + +This branch is not a small project-config patch. It changes Gecko from a relatively flat config and vector service into a multi-surface backend that now owns: + +- typed project configuration records +- project-oriented config CRUD routes +- Git-backed project repository state and read APIs +- GitHub/Fence integration points for installation and token brokering +- organization-level Git connect/reconcile flows +- upload session persistence and PR-oriented file submission flows +- thumbnail storage for projects +- a full internal package reorganization +- CI and runtime changes required by the new Git subsystem + +The branch is large because it combines feature delivery with a substantial internal re-layout. The biggest review question is not any one handler. It is whether the new package boundaries, auth model, data model, and route surface still line up cleanly with the CALYPR frontend and deployment stack. + +## Scope by Area + +`git diff --dirstat` against `main` shows the branch is concentrated in these areas: + +- `internal/git/` ~10% +- `internal/server/http/git/` ~9% +- `internal/server/middleware/` ~8% +- `gecko/` legacy removal/replacement ~8% +- `internal/server/http/config/` ~7% +- `internal/db/` ~6% +- `config/` ~5% +- `internal/thumbnail/` ~5% +- `tests/integration/` ~4% + +That matches the real shape of the work: this is mostly a Git/project backend branch, plus the refactor needed to support it. + +## Architecture Reorganization + +### Before + +The old service logic lived mostly in the top-level `gecko/` package with mixed concerns: + +- HTTP handlers +- middleware +- DB logic +- vector logic +- config logic +- response helpers + +### After + +The branch moves Gecko toward a clearer package layout: + +- `config/` + - typed config models and validation +- `internal/db/` + - config and Git persistence +- `internal/git/` + - repository domain logic, reconciliation, setup, sync, upload workflows +- `internal/integrations/fence/` + - Fence broker client for GitHub App operations +- `internal/integrations/github/` + - GitHub API client wrapper built on `go-github` +- `internal/server/http/` + - route families split by surface: + - `config` + - `directory` + - `git` + - `health` + - `vector` + - `shared` +- `internal/server/middleware/` + - request auth, access checks, logging, resource helpers +- `internal/thumbnail/` + - thumbnail storage and validation +- `internal/httputil/` + - shared JSON/error response utilities +- `internal/logging/` + - service logging wrapper/helpers +- `internal/vectoradapter/` + - Qdrant request/response translation + +This is the right direction structurally. The tradeoff is that the branch mixes architectural reorganization with product behavior changes, so code review needs to separate “moved” from “changed.” + +## Runtime / Bootstrap Changes + +`main.go` now builds Gecko as a composable server with optional integrations: + +- PostgreSQL +- Git service +- thumbnail store +- Qdrant +- Grip +- JWKS-backed JWT validation + +The important runtime additions are: + +- `--git-data-dir` / `GIT_DATA_DIR` +- `--fence-base-url` / `FENCE_BASE_URL` +- `--github-api-base-url` / `GITHUB_API_BASE_URL` + +Once DB connectivity is available, Gecko now also constructs: + +- `git.NewGitService(...)` +- `thumbnail.NewFilesystemStore(gitDataDir)` + +That makes `gitDataDir` a hard runtime dependency for the Git-enabled server path. + +### CI impact + +Because Gecko now requires a Git data directory, CI had to be updated. The branch modifies: + +- `.github/workflows/tests.yaml` + +to launch Gecko with: + +```bash +-git-data-dir /tmp/gecko-git +``` + +Without that, the service exits before health checks come up. + +## Config Model Changes + +### Project config becomes first-class + +This branch adds a typed `ProjectConfig` model in: + +- `config/projectConfig.go` + +Supported fields: + +- `title` +- `contact_email` +- `src_repo` +- `org_title` +- `description` +- `project_title` + +Validation includes: + +- required-field checks +- email validation +- repository URL normalization into a GitHub-style `host/owner/repo` form + +This is a real shift from using generic config blobs everywhere. Project metadata now has a stronger contract and normalization behavior. + +### Explorer config compatibility + +This branch also carries forward explorer config compatibility work, including the richer `fileActions` shape: + +```json +{ + "extensions": {"ext": ["action"]}, + "actions": {"action": "/path"} +} +``` + +That matters because the frontend expects this richer form, and Gecko has to unmarshal it correctly for CALYPR configs to load. + +## Database / Persistence Changes + +The database layer is no longer only about simple `config_schema.` JSON tables. + +### Existing typed config tables + +The init path now ensures config tables for: + +- `explorer` +- `nav` +- `file_summary` +- `project` +- `projects` + +### New Git-related state tables + +`internal/db/EnsureGitProjectStateTable` expands the DB footprint significantly. This branch adds persistence for: + +1. `config_schema.git_project_state` + - repository identity + - installation metadata + - mirror path + - sync/default branch/error state + +2. `config_schema.git_organization_state` + - organization installation/configuration status + - target metadata + - timestamps and last error + +3. `config_schema.git_upload_session` + - upload/PR submission session metadata + +4. `config_schema.git_upload_session_file` + - per-file status inside an upload session + +5. `config_schema.git_pending_repository` + - repositories discovered but not yet reconciled into project config + - supports both webhook-originated and user-scoped pending records + +6. `config_schema.git_setup_session` + - setup snapshot used to compare repository sets before/after install/connect flows + +This is one of the biggest branch changes. Gecko is no longer stateless around Git operations; it is now persisting lifecycle and reconciliation state explicitly. + +## HTTP Surface Changes + +The route surface is much broader than before. + +### Top-level registration + +The new entrypoint is: + +- `internal/server/http/register.go` + +Registered route families: + +- `/health` +- `/Dir...` +- `/config...` +- `/git...` +- vector routes +- swagger JSON route + +### Config routes + +`internal/server/http/config/register.go` now exposes: + +#### Generic config routes + +- `GET /config/types` +- `GET /config/list` + +Typed config groups: + +- `/config/explorer` +- `/config/nav` +- `/config/file_summary` +- `/config/project` + +Per-type operations include combinations of: + +- `GET /list` +- `GET /:configId` +- `PUT /:configId` +- `DELETE /:configId` + +#### Project config routes + +This branch adds a dedicated project config surface: + +- `GET /config/projects` +- `GET /config/projects/list` +- `GET /config/projects/summary` +- `GET /config/projects/:orgTitle/:projectTitle` +- `PUT /config/projects/:orgTitle/:projectTitle` +- `DELETE /config/projects/:orgTitle/:projectTitle` +- `DELETE /config/projects/:orgTitle` + +This is materially different from the old “config row by arbitrary key” model. The route shape now matches organization/project semantics directly. + +### Git routes + +`internal/server/http/git/register.go` is one of the largest new route families in the service. + +#### Organization-level Git routes + +- `GET /git/projects` +- `GET /git/organizations/status` +- `POST /git/organizations/reconcile` +- `POST /git/organizations/:orgTitle/init-connect` +- `POST /git/organizations/:orgTitle/connect` +- `GET /git/organizations/:orgTitle/status` +- `POST /git/organizations/:orgTitle/reconcile` + +These routes cover installation status, connect flows, and organization-wide repository reconciliation. + +#### Project-level Git read routes + +- `GET /git/projects/:orgTitle/:projectTitle` +- `GET /git/projects/:orgTitle/:projectTitle/refs` +- `GET /git/projects/:orgTitle/:projectTitle/tree` +- `GET /git/projects/:orgTitle/:projectTitle/tree/*` +- `GET /git/projects/:orgTitle/:projectTitle/file/*` +- `GET /git/projects/:orgTitle/:projectTitle/download/*` +- `GET /git/projects/:orgTitle/:projectTitle/thumbnail` + +These expose Gecko as a repository-backed project read service, not just a config API. + +#### Project-level Git write/workflow routes + +- `PUT /git/projects/:orgTitle/:projectTitle/setup` +- `PUT /git/projects/:orgTitle/:projectTitle/storage` +- `PUT /git/projects/:orgTitle/:projectTitle/thumbnail` +- `DELETE /git/projects/:orgTitle/:projectTitle/thumbnail` +- `POST /git/projects/:orgTitle/:projectTitle/edit-connect` +- `POST /git/projects/:orgTitle/:projectTitle/update` +- `POST /git/projects/:orgTitle/:projectTitle/uploads/session` +- `GET /git/projects/:orgTitle/:projectTitle/uploads/session/:sessionID` +- `POST /git/projects/:orgTitle/:projectTitle/uploads/session/:sessionID/files` +- `POST /git/projects/:orgTitle/:projectTitle/uploads/session/:sessionID/finalize` + +This is a major expansion of product surface. Gecko is now responsible for project repository setup, update, artifact flow staging, and PR-style finalization. + +## Auth and Access Model + +The middleware layer has been significantly expanded. + +### Resource path model + +The branch standardizes project authorization around Arborist-style paths: + +```text +/programs/{organization}/projects/{project} +``` + +Helper functions in `internal/server/middleware/access.go` normalize and check these resource paths. + +### Config auth model + +`ConfigAuth` now treats explorer configs differently from base/global config routes: + +- explorer config access is project-scoped +- non-explorer GET routes are broadly readable +- non-explorer write/delete routes require route-specific authorization + +### Project config auth model + +`ProjectConfigAuth` checks direct access on: + +```text +/programs/{org}/projects/{project} +``` + +and also allows certain broader admin-like resource paths such as: + +- `*` +- `/programs` +- `/programs/{org}` +- `/programs/{org}/projects` + +### Git auth model + +Git reads and organization reads are protected separately: + +- `GitProjectAuth` +- `GitOrganizationAuth` + +The important boundary in this branch is: + +- caller authorization still comes from the request `Authorization` token and Arborist/Fence checks +- Git remote access is not the same thing as caller auth and is handled separately through the Git integration flow + +Project write routes are intentionally split: + +- `PUT /git/projects/:orgTitle/:projectTitle/setup` should only require an `Authorization` header at the route layer +- setup then calls Arborist directly to determine whether the caller can read an existing project, create descendants under `/programs/{org}/projects`, manage owners on `/programs/{org}`, or bootstrap missing ownership resources +- Gecko should not pre-judge whether the org or project already exists before making that Arborist call +- follow-on mutation routes such as storage, thumbnail updates, connect edits, uploads, and update actions can use stricter project-scoped middleware because they operate after bootstrap + +That split is correct because Gecko does not have enough local state to distinguish "missing org/project" from "existing but unauthorized"; Arborist is the authority for that decision. + +## Fence / GitHub Integration Model + +This branch adds explicit integration clients instead of spreading ad hoc HTTP logic through handlers. + +### Fence integration + +`internal/integrations/fence/client.go` turns Fence into a GitHub App broker that Gecko calls for: + +- install URL requests +- organization installation status +- repository installation status +- installation repository listing +- installation token minting + +The request target is: + +- `POST {FENCE_BASE_URL}/credentials/github` + +with action-based payloads. + +This is architecturally important. Gecko is moving away from owning GitHub App secrets directly and toward asking Fence for short-lived GitHub access on demand. + +### GitHub integration + +`internal/integrations/github/client.go` uses: + +- `github.com/google/go-github/v87/github` + +for GitHub API metadata reads. + +At minimum, it currently centralizes repository metadata lookup: + +- default branch +- HTML URL + +This is the right direction and avoids more hand-written GitHub REST client code. + +## Git Service and Repository Semantics + +The branch adds a large `internal/git/` package that now owns: + +- repository identity/domain types +- setup and reconcile flows +- repository state persistence coordination +- update/sync operations +- upload workflows +- response shaping +- error mapping + +The key product shift is that Gecko is no longer just proxying config or metadata. It now maintains local repository state under a configured data directory and serves project Git views from there. + +Reviewers should pay special attention to: + +- when local repository state is created +- how `update` behaves when the local repo is missing vs already present +- how default branch information is sourced and persisted +- where Fence tokens are requested and how long they are retained in memory + +## Upload and Thumbnail Workflows + +Two entirely new concerns land in this branch. + +### Upload sessions + +Upload state is now explicit and persistent: + +- session creation +- file list replacement/storage +- session lookup +- finalize flow +- PR metadata persistence + +This means Gecko now participates in a staged contribution workflow rather than just reading repository state. + +### Thumbnails + +`internal/thumbnail/` adds filesystem-backed thumbnail storage plus validation. + +Route support includes: + +- `GET /git/projects/:orgTitle/:projectTitle/thumbnail` +- `PUT /git/projects/:orgTitle/:projectTitle/thumbnail` +- `DELETE /git/projects/:orgTitle/:projectTitle/thumbnail` + +That is a durable product-surface change and should be reviewed as such, not as a minor helper addition. + +## Legacy Code Removal + +A large part of the diff is deletion of the old flat handlers from `gecko/`, including legacy files such as: + +- `handleConfig.go` +- `handleDir.go` +- `handleVector.go` +- `middleware.go` +- `response.go` +- `server.go` + +This is not dead-code cleanup alone. These deletions are paired with replacements under `internal/server/http/...`, `internal/server/middleware/...`, and supporting packages. + +## Build / Tooling Changes + +This branch also touches: + +- `Dockerfile` +- `Makefile` +- `.dockerignore` +- `go.mod` +- `go.sum` +- swagger/docs artifacts + +Those are not side noise. They are part of the fallout from: + +- root build compatibility +- package reorganization +- new GitHub client dependency +- swagger generation drift + +Review should include a sanity pass on container build assumptions and root `go build .` behavior. + +## Testing Changes + +The branch adds or updates tests across multiple layers: + +- config tests +- project config tests +- middleware tests +- Git service tests +- upload tests +- Fence integration tests +- thumbnail tests +- integration tests + +Notable files include: + +- `config/explorerConfig_test.go` +- `config/projectConfig_test.go` +- `internal/git/service_test.go` +- `internal/git/upload_test.go` +- `internal/integrations/fence/client_test.go` +- `internal/server/middleware/git_test.go` +- `internal/thumbnail/store_test.go` +- `tests/integration/*` + +Given the branch size, the main review question is coverage shape rather than raw test count: + +- do route tests reflect current auth semantics? +- do config tests lock frontend/backend schema compatibility? +- do Git tests cover first-time setup, update, and persistence transitions? + +## Highest-Risk Areas + +If reviewing this branch for merge readiness, focus here first: + +1. **Auth correctness** + - project vs organization vs generic config auth + - read vs write route consistency + +2. **Route compatibility** + - does the route surface still match the frontend and revproxy expectations? + +3. **DB/state lifecycle** + - are new tables initialized everywhere Gecko runs? + - are deletes/updates cleaning up related Git state correctly? + +4. **Git/Fence boundary** + - does Gecko request the right thing from Fence? + - is Gecko still assuming too much GitHub App behavior locally? + +5. **Config schema compatibility** + - especially explorer config and project config alignment with the frontend + +6. **Startup behavior** + - new hard dependency on `git-data-dir` + - optional integrations degrading cleanly when unset + +## Bottom Line + +This branch should be read as a backend expansion and service re-platforming branch, not as a narrow project-config feature. + +The durable outcomes are: + +- Gecko now has a first-class project config model. +- Gecko now has a real Git-backed project API surface. +- Gecko now persists Git lifecycle state instead of treating repo operations as transient. +- Gecko now relies on Fence as the GitHub App broker boundary. +- Gecko’s internal organization is materially better, but the branch is broad enough that compatibility review has to be disciplined. diff --git a/docs/swagger.json b/docs/swagger.json index be01477..400fac7 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -209,56 +209,213 @@ } } }, - "/dir/{projectId}": { + "/git/projects/{orgTitle}/{projectTitle}/presentationConfig": { "get": { - "description": "Retrieve directory details for the given project ID and Directory path", + "description": "Retrieve the sanitized presentation HTML configured for a project. Returns empty content when no presentation has been configured.", "produces": [ "application/json" ], "tags": [ - "Directory" + "Git" ], - "summary": "Retrieve directory information for a project", + "summary": "Get project presentation HTML", "parameters": [ { "type": "string", - "description": "Project ID (format: program-project)", - "name": "projectId", + "description": "Organization title", + "name": "orgTitle", "in": "path", "required": true }, { "type": "string", - "description": "Directory Path (e.g., /data/my-dir)", - "name": "directory_path", - "in": "query", + "description": "Project title", + "name": "projectTitle", + "in": "path", "required": true } ], "responses": { "200": { - "description": "Directory information", + "description": "Presentation configuration", "schema": { - "type": "object", - "additionalProperties": true + "$ref": "#/definitions/config.PresentationConfig" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "500": { + "description": "Server error", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + } + } + }, + "post": { + "description": "Validate, sanitize, and persist presentation HTML for a project.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Git" + ], + "summary": "Create project presentation HTML", + "parameters": [ + { + "type": "string", + "description": "Organization title", + "name": "orgTitle", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Project title", + "name": "projectTitle", + "in": "path", + "required": true + }, + { + "description": "Presentation HTML payload", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + } + ], + "responses": { + "200": { + "description": "Persisted sanitized presentation configuration", + "schema": { + "$ref": "#/definitions/config.PresentationConfig" } }, "400": { - "description": "Invalid request body or Directory path", + "description": "Invalid request body or malformed HTML", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } }, "403": { - "description": "User is not allowed on any resource path", + "description": "Forbidden", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } }, "500": { "description": "Server error", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + } + } + }, + "put": { + "description": "Validate, sanitize, and persist presentation HTML for a project.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Git" + ], + "summary": "Update project presentation HTML", + "parameters": [ + { + "type": "string", + "description": "Organization title", + "name": "orgTitle", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Project title", + "name": "projectTitle", + "in": "path", + "required": true + }, + { + "description": "Presentation HTML payload", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + } + ], + "responses": { + "200": { + "description": "Persisted sanitized presentation configuration", + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + }, + "400": { + "description": "Invalid request body or malformed HTML", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "500": { + "description": "Server error", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } } } @@ -1037,6 +1194,14 @@ } } }, + "config.PresentationConfig": { + "type": "object", + "properties": { + "presentationConfig": { + "type": "string" + } + } + }, "config.ConfigItem": { "type": "object", "properties": { @@ -1707,4 +1872,4 @@ "in": "header" } } -} \ No newline at end of file +} diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 337884f..ce04647 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -193,6 +193,11 @@ definitions: sharedFilters: $ref: '#/definitions/config.SharedFiltersConfig' type: object + config.PresentationConfig: + properties: + presentationConfig: + type: string + type: object config.ConfigItem: properties: buttons: @@ -828,44 +833,148 @@ paths: summary: List all configuration IDs for a specific type tags: - Config - /dir/{projectId}: + /git/projects/{orgTitle}/{projectTitle}/presentationConfig: get: - description: Retrieve directory details for the given project ID and Directory - path + description: Retrieve the sanitized presentation HTML configured for a project. Returns empty content when no presentation has been configured. parameters: - - description: 'Project ID (format: program-project)' + - description: Organization title in: path - name: projectId + name: orgTitle required: true type: string - - description: Directory Path (e.g., /data/my-dir) - in: query - name: directory_path + - description: Project title + in: path + name: projectTitle required: true type: string produces: - application/json responses: "200": - description: Directory information + description: Presentation configuration schema: - additionalProperties: true - type: object + $ref: '#/definitions/config.PresentationConfig' + "401": + description: Missing authorization + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "404": + description: Project not found + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "500": + description: Server error + schema: + $ref: '#/definitions/httputil.ErrorResponse' + summary: Get project presentation HTML + tags: + - Git + post: + consumes: + - application/json + description: Validate, sanitize, and persist presentation HTML for a project. + parameters: + - description: Organization title + in: path + name: orgTitle + required: true + type: string + - description: Project title + in: path + name: projectTitle + required: true + type: string + - description: Presentation HTML payload + in: body + name: body + required: true + schema: + $ref: '#/definitions/config.PresentationConfig' + produces: + - application/json + responses: + "200": + description: Persisted sanitized presentation configuration + schema: + $ref: '#/definitions/config.PresentationConfig' "400": - description: Invalid request body or Directory path + description: Invalid request body or malformed HTML schema: - $ref: '#/definitions/gecko.ErrorResponse' + $ref: '#/definitions/httputil.ErrorResponse' + "401": + description: Missing authorization + schema: + $ref: '#/definitions/httputil.ErrorResponse' "403": - description: User is not allowed on any resource path + description: Forbidden schema: - $ref: '#/definitions/gecko.ErrorResponse' + $ref: '#/definitions/httputil.ErrorResponse' + "404": + description: Project not found + schema: + $ref: '#/definitions/httputil.ErrorResponse' "500": description: Server error schema: - $ref: '#/definitions/gecko.ErrorResponse' - summary: Retrieve directory information for a project + $ref: '#/definitions/httputil.ErrorResponse' + summary: Create project presentation HTML + tags: + - Git + put: + consumes: + - application/json + description: Validate, sanitize, and persist presentation HTML for a project. + parameters: + - description: Organization title + in: path + name: orgTitle + required: true + type: string + - description: Project title + in: path + name: projectTitle + required: true + type: string + - description: Presentation HTML payload + in: body + name: body + required: true + schema: + $ref: '#/definitions/config.PresentationConfig' + produces: + - application/json + responses: + "200": + description: Persisted sanitized presentation configuration + schema: + $ref: '#/definitions/config.PresentationConfig' + "400": + description: Invalid request body or malformed HTML + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "401": + description: Missing authorization + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "404": + description: Project not found + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "500": + description: Server error + schema: + $ref: '#/definitions/httputil.ErrorResponse' + summary: Update project presentation HTML tags: - - Directory + - Git /health: get: description: Checks the database connection and returns the server status diff --git a/internal/presentation/interface.go b/internal/presentation/interface.go new file mode 100644 index 0000000..1e76faa --- /dev/null +++ b/internal/presentation/interface.go @@ -0,0 +1,8 @@ +package presentation + +// Manager defines the interface for project presentation HTML storage and retrieval. +type Manager interface { + Get(organization string, project string) (string, error) + Save(organization string, project string, content string) error + ProjectPresentationPath(organization string, project string) string +} diff --git a/internal/presentation/store.go b/internal/presentation/store.go new file mode 100644 index 0000000..8b9821f --- /dev/null +++ b/internal/presentation/store.go @@ -0,0 +1,70 @@ +package presentation + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +type FilesystemStore struct { + dataDir string +} + +func NewFilesystemStore(dataDir string) *FilesystemStore { + return &FilesystemStore{dataDir: strings.TrimSpace(dataDir)} +} + +func (s *FilesystemStore) projectPresentationDir(organization string) string { + return filepath.Join( + s.dataDir, + ProjectPresentationDirectory, + sanitizePathPart(strings.TrimSpace(organization)), + ) +} + +func (s *FilesystemStore) ProjectPresentationPath(organization string, project string) string { + return filepath.Join( + s.projectPresentationDir(organization), + sanitizePathPart(strings.TrimSpace(project))+"_presentation.html", + ) +} + +func (s *FilesystemStore) Get(organization string, project string) (string, error) { + if s.dataDir == "" { + return "", ErrDataDirRequired + } + data, err := os.ReadFile(s.ProjectPresentationPath(organization, project)) + if err != nil { + if os.IsNotExist(err) { + return "", ErrNoPresentation + } + return "", fmt.Errorf("read presentation HTML: %w", err) + } + return string(data), nil +} + +func (s *FilesystemStore) Save(organization string, project string, content string) error { + if s.dataDir == "" { + return ErrDataDirRequired + } + dir := s.projectPresentationDir(organization) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create presentation directory: %w", err) + } + path := s.ProjectPresentationPath(organization, project) + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, []byte(content), 0o644); err != nil { + return fmt.Errorf("write presentation HTML: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("persist presentation HTML: %w", err) + } + return nil +} + +func sanitizePathPart(value string) string { + replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_") + return replacer.Replace(value) +} diff --git a/internal/presentation/store_test.go b/internal/presentation/store_test.go new file mode 100644 index 0000000..d29a8ba --- /dev/null +++ b/internal/presentation/store_test.go @@ -0,0 +1,44 @@ +package presentation + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFilesystemStoreUsesOrgProjectFilenameLayout(t *testing.T) { + tempDir := t.TempDir() + store := NewFilesystemStore(tempDir) + + expectedPath := filepath.Join(tempDir, ProjectPresentationDirectory, "Org_A", "Project_1_presentation.html") + if got := store.ProjectPresentationPath("Org A", "Project/1"); got != expectedPath { + t.Fatalf("unexpected presentation path: got %q want %q", got, expectedPath) + } +} + +func TestFilesystemStoreSaveAndGet(t *testing.T) { + tempDir := t.TempDir() + store := NewFilesystemStore(tempDir) + + if err := store.Save("org-a", "proj-a", "

Hello

"); err != nil { + t.Fatalf("save failed: %v", err) + } + got, err := store.Get("org-a", "proj-a") + if err != nil { + t.Fatalf("get failed: %v", err) + } + if got != "

Hello

" { + t.Fatalf("unexpected content: %q", got) + } + if _, err := os.Stat(store.ProjectPresentationPath("org-a", "proj-a")); err != nil { + t.Fatalf("expected stored file: %v", err) + } +} + +func TestFilesystemStoreGetMissing(t *testing.T) { + store := NewFilesystemStore(t.TempDir()) + _, err := store.Get("org-a", "proj-a") + if err != ErrNoPresentation { + t.Fatalf("expected ErrNoPresentation, got %v", err) + } +} diff --git a/internal/presentation/types.go b/internal/presentation/types.go new file mode 100644 index 0000000..9a24a2f --- /dev/null +++ b/internal/presentation/types.go @@ -0,0 +1,12 @@ +package presentation + +import "errors" + +const ( + ProjectPresentationDirectory = "_project_presentations" +) + +var ( + ErrNoPresentation = errors.New("project has no presentation HTML") + ErrDataDirRequired = errors.New("presentation storage directory is required") +) diff --git a/internal/server/http/directory/directory.go b/internal/server/http/directory/directory.go deleted file mode 100644 index 6ec7174..0000000 --- a/internal/server/http/directory/directory.go +++ /dev/null @@ -1,128 +0,0 @@ -package directory - -import ( - "fmt" - "net/http" - "path" - "strings" - - "github.com/bmeg/grip/gripql" - "github.com/calypr/gecko/apierror" - "github.com/calypr/gecko/internal/httputil" - servermw "github.com/calypr/gecko/internal/server/middleware" - "github.com/gofiber/fiber/v3" -) - -func (handler *Handler) registerDirectoryHandlers(app fiber.Router, authMiddleware fiber.Handler) { - app.Get("/dir", handler.handleListProjects) - app.Get("/dir/:projectId", authMiddleware, handler.handleDirGet) -} - -// handleListProjects godoc -// @Summary List authorized projects -// @Description Retrieve the set of projects visible to the current user. -// @Tags Directory -// @Produce json -// @Success 200 {array} string "Project resource paths" -// @Failure 401 {object} ErrorResponse "Unauthorized" -// @Failure 500 {object} ErrorResponse "Server error" -// @Router /dir [get] -func (handler *Handler) handleListProjects(ctx fiber.Ctx) error { - projs, errResponse := servermw.GetProjectsFromToken(ctx, servermw.NewFenceUserAccessHandler(nil), "read", "*") - if errResponse != nil { - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - q := buildListProjectsQuery(projs) - res, err := handler.gripqlClient.Traversal(ctx, &gripql.GraphQuery{Graph: handler.gripGraphName, Query: q.Statements}) - if err != nil { - errResponse = httputil.NewError(apierror.TypeGraphQueryFailed, "graph query failed", http.StatusInternalServerError, nil, &err) - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - out := []string{} - for r := range res { - renda, ok := r.GetRender().GetStructValue().AsMap()["project"].(string) - if ok { - out = append(out, renda) - } - } - return httputil.JSON(out, http.StatusOK).Write(ctx) -} - -// handleDirGet godoc -// @Summary Retrieve directory information for a project -// @Description Retrieve directory details for the given project ID and directory path. -// @Tags Directory -// @Produce json -// @Param projectId path string true "Project ID (format: program-project)" -// @Param directory query string true "Directory path (e.g. /data/my-dir)" -// @Success 200 {array} map[string]interface{} "Directory information" -// @Failure 400 {object} ErrorResponse "Invalid request body or directory path" -// @Failure 403 {object} ErrorResponse "User is not allowed on the resource path" -// @Failure 500 {object} ErrorResponse "Server error" -// @Router /dir/{projectId} [get] -func (handler *Handler) handleDirGet(ctx fiber.Ctx) error { - projectID := ctx.Params("projectId") - dirPath := ctx.Query("directory") - if dirPath == "" || !isValidPosixPath(&dirPath) { - errResponse := httputil.NewError(apierror.TypeInvalidDirectory, fmt.Sprintf("Invalid or missing Directory path: %s", dirPath), http.StatusBadRequest, map[string]any{"directory": dirPath}, nil) - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - - projectSplit := strings.Split(projectID, "-") - if len(projectSplit) != 2 { - errResponse := httputil.NewError(apierror.TypeInvalidProjectID, fmt.Sprintf("Failed to parse request body: %v", fmt.Sprintf("incorrect path %s", ctx.Path())), http.StatusNotFound, map[string]any{"project_id": projectID}, nil) - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - projectID = "/programs/" + projectSplit[0] + "/projects/" + projectSplit[1] - - q := buildDirGetQuery(projectID, dirPath) - res, err := handler.gripqlClient.Traversal(ctx, &gripql.GraphQuery{Graph: handler.gripGraphName, Query: q.Statements}) - if err != nil { - errResponse := httputil.NewError(apierror.TypeGraphQueryFailed, "graph query failed", http.StatusInternalServerError, nil, &err) - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - out := []any{} - for r := range res { - out = append(out, r.GetVertex()) - } - return httputil.JSON(out, http.StatusOK).Write(ctx) -} - -func isValidPosixPath(p *string) bool { - if strings.ContainsRune(*p, 000) || !path.IsAbs(*p) || strings.Contains(*p, "\\") { - return false - } - cleaned := path.Clean(*p) - if *p == "" || cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "/..") { - return false - } - return true -} - -func buildListProjectsQuery(projs []any) *gripql.Query { - return gripql.V(). - HasLabel("ResearchStudy"). - Has(gripql.Within("auth_resource_path", projs...)). - As("project"). - OutE("rootDir_Directory"). - Select("project"). - Distinct("auth_resource_path"). - Render(map[string]any{"project": "$project.auth_resource_path"}) -} - -func buildDirGetQuery(projectID, dirPath string) *gripql.Query { - q := gripql.V().HasLabel("ResearchStudy").Has(gripql.Eq("auth_resource_path", projectID)).OutE("rootDir_Directory").OutNull().OutNull() - if dirPath != "/" { - for splStr := range strings.SplitSeq(strings.Trim(dirPath, "/"), "/") { - q = q.Has(gripql.Eq("name", splStr)).Has(gripql.Eq("auth_resource_path", projectID)).OutNull() - } - } else { - q = q.Has(gripql.Eq("auth_resource_path", projectID)) - } - return q -} diff --git a/internal/server/http/directory/directory_test.go b/internal/server/http/directory/directory_test.go deleted file mode 100644 index 0237e40..0000000 --- a/internal/server/http/directory/directory_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package directory - -import ( - "encoding/json" - "testing" - - "github.com/bmeg/grip/gripql" - "github.com/stretchr/testify/assert" -) - -// Helper to convert query to string for assertion -func queryString(q *gripql.Query) string { - b, _ := json.Marshal(q.Statements) - return string(b) -} - -func TestBuildListProjectsQuery(t *testing.T) { - projs := []any{"PROG-PROJ1", "PROG-PROJ2"} - q := buildListProjectsQuery(projs) - - // Verify key components of the query - jsonStr := queryString(q) - assert.Contains(t, jsonStr, "ResearchStudy") - assert.Contains(t, jsonStr, "auth_resource_path") - assert.Contains(t, jsonStr, "rootDir_Directory") // Must have OutE - assert.Contains(t, jsonStr, "Distinct") // Must have Distinct -} - -func TestBuildDirGetQuery_Root(t *testing.T) { - projectId := "/programs/PROG/projects/PROJ" - dirPath := "/" - q := buildDirGetQuery(projectId, dirPath) - - jsonStr := queryString(q) - assert.Contains(t, jsonStr, "ResearchStudy") - // The query logic for root just filters by auth_resource_path - assert.Contains(t, jsonStr, "auth_resource_path") - assert.Contains(t, jsonStr, "rootDir_Directory") - // Verify it does NOT contain loop logic - assert.NotContains(t, jsonStr, "name") -} - -func TestBuildDirGetQuery_SubDir(t *testing.T) { - projectId := "/programs/PROG/projects/PROJ" - dirPath := "/data/foo" - q := buildDirGetQuery(projectId, dirPath) - - jsonStr := queryString(q) - // Verify traversal - assert.Contains(t, jsonStr, "data") - assert.Contains(t, jsonStr, "foo") - // Verify security check is present in loop (auth_resource_path appears multiple times) - // We can't easily count occurrences in JSON string without parsing, but Contains is a good smoke test - assert.Contains(t, jsonStr, "auth_resource_path") -} diff --git a/internal/server/http/directory/handler.go b/internal/server/http/directory/handler.go deleted file mode 100644 index c2351a8..0000000 --- a/internal/server/http/directory/handler.go +++ /dev/null @@ -1,23 +0,0 @@ -package directory - -import ( - "github.com/bmeg/grip/gripql" - "github.com/calypr/gecko/internal/server/http/shared" - "github.com/uc-cdis/arborist/arborist" -) - -type Handler struct { - *shared.Handler - logger arborist.Logger - gripqlClient *gripql.Client - gripGraphName string -} - -func NewHandler(sharedHandler *shared.Handler) *Handler { - return &Handler{ - Handler: sharedHandler, - logger: sharedHandler.Logger, - gripqlClient: sharedHandler.GripqlClient, - gripGraphName: sharedHandler.GripGraphName, - } -} diff --git a/internal/server/http/directory/register.go b/internal/server/http/directory/register.go deleted file mode 100644 index d361e5a..0000000 --- a/internal/server/http/directory/register.go +++ /dev/null @@ -1,18 +0,0 @@ -package directory - -import ( - "github.com/calypr/gecko/internal/server/http/shared" - servermw "github.com/calypr/gecko/internal/server/middleware" - "github.com/gofiber/fiber/v3" -) - -func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler servermw.ResourceAccessHandler) { - handler := NewHandler(sharedHandler) - if handler.GripqlClient == nil { - handler.Logger.Warning("Skipping gripql Directory endpoints — no database configured") - return - } - authMiddleware := servermw.GeneralAuth(handler.Logger, authzHandler, "read", "*") - app.Get("/dir", handler.handleListProjects) - app.Get("/dir/:projectId", authMiddleware, handler.handleDirGet) -} diff --git a/internal/server/http/git/handler.go b/internal/server/http/git/handler.go index 6ee4aa0..947fda8 100644 --- a/internal/server/http/git/handler.go +++ b/internal/server/http/git/handler.go @@ -3,6 +3,7 @@ package git import ( "github.com/bmeg/grip/gripql" "github.com/calypr/gecko/internal/git" + "github.com/calypr/gecko/internal/presentation" "github.com/calypr/gecko/internal/server/http/shared" "github.com/calypr/gecko/internal/thumbnail" "github.com/jmoiron/sqlx" @@ -12,30 +13,32 @@ import ( type Handler struct { *shared.Handler - db *sqlx.DB - logger arborist.Logger - jwtApp arborist.JWTDecoder - qdrantClient *qdrant.Client - gripqlClient *gripql.Client - gripGraphName string - gitService *git.GitService - projectSetup *git.SetupService - projectSync *git.ReconcileService - thumbnailStore thumbnail.Manager + db *sqlx.DB + logger arborist.Logger + jwtApp arborist.JWTDecoder + qdrantClient *qdrant.Client + gripqlClient *gripql.Client + gripGraphName string + gitService *git.GitService + projectSetup *git.SetupService + projectSync *git.ReconcileService + thumbnailStore thumbnail.Manager + presentationStore presentation.Manager } func NewHandler(sharedHandler *shared.Handler) *Handler { return &Handler{ - Handler: sharedHandler, - db: sharedHandler.DB, - logger: sharedHandler.Logger, - jwtApp: sharedHandler.JWTApp, - qdrantClient: sharedHandler.QdrantClient, - gripqlClient: sharedHandler.GripqlClient, - gripGraphName: sharedHandler.GripGraphName, - gitService: sharedHandler.GitService, - projectSetup: sharedHandler.ProjectSetup, - projectSync: sharedHandler.ProjectSync, - thumbnailStore: sharedHandler.ThumbnailStore, + Handler: sharedHandler, + db: sharedHandler.DB, + logger: sharedHandler.Logger, + jwtApp: sharedHandler.JWTApp, + qdrantClient: sharedHandler.QdrantClient, + gripqlClient: sharedHandler.GripqlClient, + gripGraphName: sharedHandler.GripGraphName, + gitService: sharedHandler.GitService, + projectSetup: sharedHandler.ProjectSetup, + projectSync: sharedHandler.ProjectSync, + thumbnailStore: sharedHandler.ThumbnailStore, + presentationStore: sharedHandler.PresentationStore, } } diff --git a/internal/server/http/git/presentation_config.go b/internal/server/http/git/presentation_config.go new file mode 100644 index 0000000..1317948 --- /dev/null +++ b/internal/server/http/git/presentation_config.go @@ -0,0 +1,64 @@ +package git + +import ( + "errors" + "fmt" + "net/http" + + appconfig "github.com/calypr/gecko/config" + "github.com/calypr/gecko/internal/httputil" + "github.com/calypr/gecko/internal/presentation" + "github.com/gofiber/fiber/v3" +) + +func (handler *Handler) handleGitProjectPresentationConfigGET(ctx fiber.Ctx) error { + organization, project, projectID, errResponse := handler.resolveExistingProject(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + if handler.presentationStore == nil { + errResponse = httputil.NewError("internal_error", "presentation storage is not configured", http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + + content, err := handler.presentationStore.Get(organization, project) + if err != nil { + if errors.Is(err, presentation.ErrNoPresentation) { + return httputil.JSON(appconfig.PresentationConfig{}, http.StatusOK).Write(ctx) + } + errResponse = httputil.NewError("internal_error", fmt.Sprintf("failed to load project presentation config: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + return httputil.JSON(appconfig.PresentationConfig{PresentationConfig: content}, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectPresentationConfigPUT(ctx fiber.Ctx) error { + organization, project, projectID, errResponse := handler.resolveExistingProject(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + if handler.presentationStore == nil { + errResponse = httputil.NewError("internal_error", "presentation storage is not configured", http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + + cfg := &appconfig.PresentationConfig{} + if errResponse = httputil.ParseJSONBody(ctx.Body(), cfg, map[string]any{"project_id": projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + if err := cfg.Validate(); err != nil { + errResponse = httputil.NewError("validation_failed", fmt.Sprintf("body data validation failed: %s", err), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + if err := handler.presentationStore.Save(organization, project, cfg.PresentationConfig); err != nil { + errResponse = httputil.NewError("internal_error", fmt.Sprintf("failed to persist project presentation config: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + return httputil.JSON(cfg, http.StatusOK).Write(ctx) +} diff --git a/internal/server/http/git/presentation_config_test.go b/internal/server/http/git/presentation_config_test.go new file mode 100644 index 0000000..819ce51 --- /dev/null +++ b/internal/server/http/git/presentation_config_test.go @@ -0,0 +1,239 @@ +package git + +import ( + "bytes" + "encoding/json" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + appconfig "github.com/calypr/gecko/config" + geckologging "github.com/calypr/gecko/internal/logging" + "github.com/calypr/gecko/internal/presentation" + servermw "github.com/calypr/gecko/internal/server/middleware" + "github.com/gofiber/fiber/v3" + "github.com/jmoiron/sqlx" +) + +type fakePresentationAccessHandler struct { + resources []any +} + +func (handler fakePresentationAccessHandler) GetAllowedResources(_ string, _ string, _ string) ([]any, error) { + return handler.resources, nil +} + +func (handler fakePresentationAccessHandler) CheckResourceServiceAccess(_ string, _ string, _ string, resourcePath string) (bool, error) { + for _, resource := range handler.resources { + if value, ok := resource.(string); ok && value == resourcePath { + return true, nil + } + } + return false, nil +} + +func newPresentationConfigTestServer(t *testing.T) (*Handler, sqlmock.Sqlmock, func()) { + t.Helper() + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + handler := &Handler{ + db: sqlx.NewDb(db, "sqlmock"), + logger: &geckologging.Handler{Logger: log.New(os.Stdout, "", 0)}, + presentationStore: presentation.NewFilesystemStore(t.TempDir()), + } + return handler, mock, func() { _ = db.Close() } +} + +func newPresentationConfigApp(handler *Handler, authz servermw.ResourceAccessHandler) *fiber.App { + app := fiber.New() + app.Get("/git/projects/:orgTitle/:projectTitle/presentationConfig", servermw.ProjectConfigAuth(handler.logger, authz, "read"), handler.handleGitProjectPresentationConfigGET) + app.Put("/git/projects/:orgTitle/:projectTitle/presentationConfig", servermw.ProjectConfigAuth(handler.logger, authz, "update"), handler.handleGitProjectPresentationConfigPUT) + app.Post("/git/projects/:orgTitle/:projectTitle/presentationConfig", servermw.ProjectConfigAuth(handler.logger, authz, "update"), handler.handleGitProjectPresentationConfigPUT) + return app +} + +func runPresentationConfigRequest(t *testing.T, app *fiber.App, req *http.Request) *http.Response { + t.Helper() + resp, err := app.Test(req, fiber.TestConfig{Timeout: 0, FailOnTimeout: false}) + if err != nil { + t.Fatalf("fiber test request failed: %v", err) + } + return resp +} + +func expectProjectLookup(mock sqlmock.Sqlmock, organization string, project string) { + projectCfg := appconfig.ProjectConfig{ + Title: project, + ContactEmail: "owner@example.org", + SrcRepo: "github.com/example/" + project, + OrgTitle: organization, + Description: "project", + ProjectTitle: project, + } + content, _ := json.Marshal(projectCfg) + mock.ExpectQuery(`SELECT name, content FROM config_schema\.projects WHERE name=\$1`). + WithArgs(organization + "/" + project). + WillReturnRows(sqlmock.NewRows([]string{"name", "content"}).AddRow(organization+"/"+project, content)) +} + +func decodePresentationResponse(t *testing.T, body io.Reader) appconfig.PresentationConfig { + t.Helper() + var payload appconfig.PresentationConfig + if err := json.NewDecoder(body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + return payload +} + +func TestGitProjectPresentationConfigGETNoConfigFallsBackToEmpty(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + expectProjectLookup(mock, "org-a", "proj-a") + + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) + req := httptest.NewRequest(http.MethodGet, "/git/projects/org-a/proj-a/presentationConfig", nil) + req.Header.Set("Authorization", "Bearer test") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + payload := decodePresentationResponse(t, resp.Body) + if payload.PresentationConfig != "" { + t.Fatalf("expected empty presentationConfig, got %q", payload.PresentationConfig) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectPresentationConfigPUTSanitizesAndPersistsHTML(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + expectProjectLookup(mock, "org-a", "proj-a") + + requestPayload := appconfig.PresentationConfig{ + PresentationConfig: `

Hello

`, + } + requestBody, _ := json.Marshal(requestPayload) + expectedStored := `

Hello

` + + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) + req := httptest.NewRequest(http.MethodPut, "/git/projects/org-a/proj-a/presentationConfig", bytes.NewReader(requestBody)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + payload := decodePresentationResponse(t, resp.Body) + if payload.PresentationConfig != expectedStored { + t.Fatalf("unexpected sanitized response: %q", payload.PresentationConfig) + } + data, err := os.ReadFile(handler.presentationStore.ProjectPresentationPath("org-a", "proj-a")) + if err != nil { + t.Fatalf("read stored presentation: %v", err) + } + if string(data) != expectedStored { + t.Fatalf("unexpected stored HTML: %q", string(data)) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectPresentationConfigPOSTUsesSameWritePath(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + expectProjectLookup(mock, "org-a", "proj-a") + + requestPayload := appconfig.PresentationConfig{PresentationConfig: `

Hello

`} + requestBody, _ := json.Marshal(requestPayload) + + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) + req := httptest.NewRequest(http.MethodPost, "/git/projects/org-a/proj-a/presentationConfig", bytes.NewReader(requestBody)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + if filepath.Base(handler.presentationStore.ProjectPresentationPath("org-a", "proj-a")) != "proj-a_presentation.html" { + t.Fatalf("unexpected presentation filename: %q", filepath.Base(handler.presentationStore.ProjectPresentationPath("org-a", "proj-a"))) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectPresentationConfigPUTRejectsMalformedHTML(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + expectProjectLookup(mock, "org-a", "proj-a") + + requestBody := []byte(`{"presentationConfig":"

broken

"}`) + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) + req := httptest.NewRequest(http.MethodPut, "/git/projects/org-a/proj-a/presentationConfig", bytes.NewReader(requestBody)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected status 400, got %d", resp.StatusCode) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectPresentationConfigPUTRejectsUnknownFields(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + expectProjectLookup(mock, "org-a", "proj-a") + + requestBody := []byte(`{"presentationConfig":"

Hello

","extra":"nope"}`) + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) + req := httptest.NewRequest(http.MethodPut, "/git/projects/org-a/proj-a/presentationConfig", bytes.NewReader(requestBody)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected status 400, got %d", resp.StatusCode) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectPresentationConfigRouteRequiresProjectAccess(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/other"}}) + req := httptest.NewRequest(http.MethodGet, "/git/projects/org-a/proj-a/presentationConfig", nil) + req.Header.Set("Authorization", "Bearer test") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("expected status 403, got %d", resp.StatusCode) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} diff --git a/internal/server/http/git/register.go b/internal/server/http/git/register.go index 80d6786..47c8a4f 100644 --- a/internal/server/http/git/register.go +++ b/internal/server/http/git/register.go @@ -23,6 +23,8 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler projectReadAuth := servermw.GitProjectAuth(handler.Logger, authzHandler) projectSetupAuth := servermw.GitProjectSetupAuth(handler.Logger, authzHandler) projectWriteAuth := servermw.GitProjectMutationAuth(handler.Logger, authzHandler, "update") + projectConfigReadAuth := servermw.ProjectConfigAuth(handler.Logger, authzHandler, "read") + projectConfigWriteAuth := servermw.ProjectConfigAuth(handler.Logger, authzHandler, "update") gitGroup.Get("/projects/:orgTitle/:projectTitle", projectReadAuth, handler.handleGitProjectGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/refs", projectReadAuth, handler.handleGitProjectRefsGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/tree", projectReadAuth, handler.handleGitProjectTreeGET) @@ -30,6 +32,7 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler gitGroup.Get("/projects/:orgTitle/:projectTitle/file/*", projectReadAuth, handler.handleGitProjectFileGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/download/*", projectReadAuth, handler.handleGitProjectDownloadGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/thumbnail", handler.handleGitProjectThumbnailGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/presentationConfig", projectConfigReadAuth, handler.handleGitProjectPresentationConfigGET) projectGitWrite := gitGroup.Group("/projects/:orgTitle/:projectTitle", servermw.RequireAuthorization(handler.Logger)) // Setup must stay auth-only so a brand-new organization can be bootstrapped @@ -38,6 +41,8 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler projectGitWrite.Put("/storage", projectSetupAuth, handler.handleCalyprProjectStoragePUT) projectGitWrite.Put("/thumbnail", projectWriteAuth, handler.handleGitProjectThumbnailPUT) projectGitWrite.Delete("/thumbnail", projectWriteAuth, handler.handleGitProjectThumbnailDELETE) + projectGitWrite.Put("/presentationConfig", projectConfigWriteAuth, handler.handleGitProjectPresentationConfigPUT) + projectGitWrite.Post("/presentationConfig", projectConfigWriteAuth, handler.handleGitProjectPresentationConfigPUT) projectGitWrite.Post("/edit-connect", projectWriteAuth, handler.handleGitProjectEditConnectPOST) projectGitWrite.Post("/update", projectWriteAuth, handler.handleGitProjectUpdatePOST) projectGitWrite.Post("/uploads/session", projectWriteAuth, handler.handleGitProjectUploadSessionPOST) diff --git a/internal/server/http/register.go b/internal/server/http/register.go index d9417ae..b0b1c1b 100644 --- a/internal/server/http/register.go +++ b/internal/server/http/register.go @@ -6,7 +6,6 @@ import ( "github.com/calypr/gecko/internal/httputil" "github.com/calypr/gecko/internal/server/http/config" - "github.com/calypr/gecko/internal/server/http/directory" "github.com/calypr/gecko/internal/server/http/git" "github.com/calypr/gecko/internal/server/http/health" "github.com/calypr/gecko/internal/server/http/shared" @@ -26,7 +25,6 @@ func Register(app *fiber.App, deps Dependencies) { }) health.RegisterRoutes(app, handler) - directory.RegisterRoutes(app, handler, authzHandler) config.RegisterRoutes(app, handler, authzHandler) git.RegisterRoutes(app, handler, authzHandler) vector.RegisterRoutes(app, handler) diff --git a/internal/server/http/shared/handler.go b/internal/server/http/shared/handler.go index fdceeab..9ffe04a 100644 --- a/internal/server/http/shared/handler.go +++ b/internal/server/http/shared/handler.go @@ -8,6 +8,7 @@ import ( "github.com/bmeg/grip/gripql" "github.com/calypr/gecko/internal/git" gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + "github.com/calypr/gecko/internal/presentation" servermw "github.com/calypr/gecko/internal/server/middleware" "github.com/calypr/gecko/internal/thumbnail" "github.com/jmoiron/sqlx" @@ -16,27 +17,29 @@ import ( ) type Dependencies struct { - DB *sqlx.DB - Logger arborist.Logger - JWTApp arborist.JWTDecoder - QdrantClient *qdrant.Client - GripqlClient *gripql.Client - GripGraphName string - GitService *git.GitService - ThumbnailStore thumbnail.Manager + DB *sqlx.DB + Logger arborist.Logger + JWTApp arborist.JWTDecoder + QdrantClient *qdrant.Client + GripqlClient *gripql.Client + GripGraphName string + GitService *git.GitService + ThumbnailStore thumbnail.Manager + PresentationStore presentation.Manager } type Handler struct { - DB *sqlx.DB - Logger arborist.Logger - JWTApp arborist.JWTDecoder - QdrantClient *qdrant.Client - GripqlClient *gripql.Client - GripGraphName string - GitService *git.GitService - ProjectSetup *git.SetupService - ProjectSync *git.ReconcileService - ThumbnailStore thumbnail.Manager + DB *sqlx.DB + Logger arborist.Logger + JWTApp arborist.JWTDecoder + QdrantClient *qdrant.Client + GripqlClient *gripql.Client + GripGraphName string + GitService *git.GitService + ProjectSetup *git.SetupService + ProjectSync *git.ReconcileService + ThumbnailStore thumbnail.Manager + PresentationStore presentation.Manager } func NewHandler(deps Dependencies) *Handler { @@ -52,16 +55,16 @@ func NewHandler(deps Dependencies) *Handler { ) } return &Handler{ - DB: deps.DB, - Logger: deps.Logger, - JWTApp: deps.JWTApp, - QdrantClient: deps.QdrantClient, - GripqlClient: deps.GripqlClient, - GripGraphName: deps.GripGraphName, - GitService: deps.GitService, - ProjectSetup: projectSetup, - ProjectSync: projectSync, - ThumbnailStore: deps.ThumbnailStore, + DB: deps.DB, + Logger: deps.Logger, + JWTApp: deps.JWTApp, + QdrantClient: deps.QdrantClient, + GripqlClient: deps.GripqlClient, + GripGraphName: deps.GripGraphName, + GitService: deps.GitService, + ProjectSetup: projectSetup, + ProjectSync: projectSync, + ThumbnailStore: deps.ThumbnailStore, + PresentationStore: deps.PresentationStore, } } - diff --git a/internal/server/server.go b/internal/server/server.go index df8a45e..a7c847b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -9,6 +9,7 @@ import ( "github.com/bmeg/grip/gripql" "github.com/calypr/gecko/internal/git" geckologging "github.com/calypr/gecko/internal/logging" + "github.com/calypr/gecko/internal/presentation" httpapi "github.com/calypr/gecko/internal/server/http" servermw "github.com/calypr/gecko/internal/server/middleware" "github.com/calypr/gecko/internal/thumbnail" @@ -19,15 +20,16 @@ import ( ) type Server struct { - db *sqlx.DB - jwtApp arborist.JWTDecoder - Logger *geckologging.Handler - stmts *arborist.CachedStmts - qdrantClient *qdrant.Client - gripqlClient *gripql.Client - gripGraphName string - gitService *git.GitService - thumbnailStore thumbnail.Manager + db *sqlx.DB + jwtApp arborist.JWTDecoder + Logger *geckologging.Handler + stmts *arborist.CachedStmts + qdrantClient *qdrant.Client + gripqlClient *gripql.Client + gripGraphName string + gitService *git.GitService + thumbnailStore thumbnail.Manager + presentationStore presentation.Manager } func NewServer() *Server { return &Server{} } @@ -69,6 +71,11 @@ func (server *Server) WithThumbnailStore(store thumbnail.Manager) *Server { return server } +func (server *Server) WithPresentationStore(store presentation.Manager) *Server { + server.presentationStore = store + return server +} + func (server *Server) Init() (*Server, error) { if server.jwtApp == nil { return nil, errors.New("gecko server initialized without JWT app") @@ -116,14 +123,15 @@ func (server *Server) MakeRouter() *fiber.App { app.Use(servermw.RequestLogger(server.Logger)) httpapi.Register(app, httpapi.Dependencies{ - DB: server.db, - Logger: server.Logger, - JWTApp: server.jwtApp, - QdrantClient: server.qdrantClient, - GripqlClient: server.gripqlClient, - GripGraphName: server.gripGraphName, - GitService: server.gitService, - ThumbnailStore: server.thumbnailStore, + DB: server.db, + Logger: server.Logger, + JWTApp: server.jwtApp, + QdrantClient: server.qdrantClient, + GripqlClient: server.gripqlClient, + GripGraphName: server.gripGraphName, + GitService: server.gitService, + ThumbnailStore: server.thumbnailStore, + PresentationStore: server.presentationStore, }) return app } diff --git a/main.go b/main.go index 1237b71..bdf9c14 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( "github.com/calypr/gecko/internal/git" integrationfence "github.com/calypr/gecko/internal/integrations/fence" integrationgithub "github.com/calypr/gecko/internal/integrations/github" + "github.com/calypr/gecko/internal/presentation" server "github.com/calypr/gecko/internal/server" "github.com/calypr/gecko/internal/thumbnail" ) @@ -86,6 +87,7 @@ func main() { }) serverBuilder = serverBuilder.WithGitService(gitService) serverBuilder = serverBuilder.WithThumbnailStore(thumbnail.NewFilesystemStore(gitDataDir)) + serverBuilder = serverBuilder.WithPresentationStore(presentation.NewFilesystemStore(gitDataDir)) } if qdrantHost != "" && qdrantPort != 0 { From 71c27ac7c3aff9dd6f8a542c1081c6597c71ca3f Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Sat, 13 Jun 2026 19:22:04 -0700 Subject: [PATCH 08/36] update presentation route to not sanitize html --- config/presentationConfig.go | 293 +----------------- config/presentationConfig_test.go | 16 +- docs/docs.go | 14 +- docs/swagger.json | 14 +- docs/swagger.yaml | 14 +- .../http/git/presentation_config_test.go | 25 +- 6 files changed, 28 insertions(+), 348 deletions(-) diff --git a/config/presentationConfig.go b/config/presentationConfig.go index 1aa5d51..116398d 100644 --- a/config/presentationConfig.go +++ b/config/presentationConfig.go @@ -1,307 +1,18 @@ package config -import ( - "bytes" - "fmt" - "net/url" - "strings" - - "golang.org/x/net/html" -) +import "fmt" type PresentationConfig struct { PresentationConfig string `json:"presentationConfig"` } func (p PresentationConfig) IsZero() bool { - return strings.TrimSpace(p.PresentationConfig) == "" + return p.PresentationConfig == "" } func (p *PresentationConfig) Validate() error { if p == nil { return fmt.Errorf("presentation config is required") } - sanitized, err := sanitizePresentationHTML(p.PresentationConfig) - if err != nil { - return err - } - p.PresentationConfig = sanitized return nil } - -func sanitizePresentationHTML(raw string) (string, error) { - if strings.TrimSpace(raw) == "" { - return "", nil - } - if err := validatePresentationHTML(raw); err != nil { - return "", err - } - doc, err := html.Parse(strings.NewReader(raw)) - if err != nil { - return "", fmt.Errorf("presentationConfig must contain valid HTML: %w", err) - } - - container := &html.Node{Type: html.ElementNode, Data: "div"} - source := findHTMLElement(doc, "body") - if source == nil { - source = doc - } - for child := source.FirstChild; child != nil; child = child.NextSibling { - sanitizePresentationNode(container, child) - } - - var buf bytes.Buffer - for child := container.FirstChild; child != nil; child = child.NextSibling { - if err := html.Render(&buf, child); err != nil { - return "", fmt.Errorf("render sanitized presentation HTML: %w", err) - } - } - return strings.TrimSpace(buf.String()), nil -} - -func validatePresentationHTML(raw string) error { - tokenizer := html.NewTokenizer(strings.NewReader(raw)) - var stack []string - - for { - switch tokenizer.Next() { - case html.ErrorToken: - if err := tokenizer.Err(); err != nil { - if err.Error() == "EOF" { - if len(stack) != 0 { - return fmt.Errorf("presentationConfig must contain balanced HTML tags") - } - return nil - } - return fmt.Errorf("presentationConfig must contain valid HTML: %w", err) - } - if len(stack) != 0 { - return fmt.Errorf("presentationConfig must contain balanced HTML tags") - } - return nil - case html.StartTagToken: - tagName, hasAttr := tokenizer.TagName() - tag := strings.ToLower(string(tagName)) - if !presentationVoidElements[tag] { - stack = append(stack, tag) - } - if hasAttr { - for { - _, _, more := tokenizer.TagAttr() - if !more { - break - } - } - } - case html.SelfClosingTagToken: - if _, hasAttr := tokenizer.TagName(); hasAttr { - for { - _, _, more := tokenizer.TagAttr() - if !more { - break - } - } - } - case html.EndTagToken: - tagName, _ := tokenizer.TagName() - tag := strings.ToLower(string(tagName)) - if presentationVoidElements[tag] { - return fmt.Errorf("presentationConfig must not close void element <%s>", tag) - } - if len(stack) == 0 || stack[len(stack)-1] != tag { - return fmt.Errorf("presentationConfig must contain balanced HTML tags") - } - stack = stack[:len(stack)-1] - } - } -} - -func sanitizePresentationNode(parent *html.Node, node *html.Node) { - switch node.Type { - case html.TextNode: - parent.AppendChild(&html.Node{Type: html.TextNode, Data: node.Data}) - case html.ElementNode: - tag := strings.ToLower(node.Data) - if presentationDropContentTags[tag] { - return - } - if !presentationAllowedTags[tag] { - for child := node.FirstChild; child != nil; child = child.NextSibling { - sanitizePresentationNode(parent, child) - } - return - } - - sanitized := &html.Node{Type: html.ElementNode, Data: tag} - for _, attr := range sanitizePresentationAttrs(tag, node.Attr) { - sanitized.Attr = append(sanitized.Attr, attr) - } - for child := node.FirstChild; child != nil; child = child.NextSibling { - sanitizePresentationNode(sanitized, child) - } - parent.AppendChild(sanitized) - } -} - -func sanitizePresentationAttrs(tag string, attrs []html.Attribute) []html.Attribute { - sanitized := make([]html.Attribute, 0, len(attrs)) - hasRel := false - hasHref := false - targetBlank := false - - for _, attr := range attrs { - key := strings.ToLower(strings.TrimSpace(attr.Key)) - if !presentationAttrAllowed(tag, key) { - continue - } - value := strings.TrimSpace(attr.Val) - switch key { - case "href": - if !isSafePresentationURL(value, false) { - continue - } - hasHref = true - case "src": - if !isSafePresentationURL(value, true) { - continue - } - case "target": - if value == "_blank" { - targetBlank = true - } - case "rel": - hasRel = true - } - sanitized = append(sanitized, html.Attribute{Key: key, Val: value}) - } - - if tag == "a" && !hasHref { - filtered := sanitized[:0] - for _, attr := range sanitized { - if attr.Key == "target" || attr.Key == "rel" { - continue - } - filtered = append(filtered, attr) - } - sanitized = filtered - } else if tag == "a" && targetBlank && !hasRel { - sanitized = append(sanitized, html.Attribute{Key: "rel", Val: "noopener noreferrer"}) - } - return sanitized -} - -func presentationAttrAllowed(tag string, key string) bool { - if presentationGlobalAttrs[key] { - return true - } - if strings.HasPrefix(key, "aria-") || strings.HasPrefix(key, "data-") { - return true - } - if allowed := presentationTagAttrs[tag]; allowed != nil { - return allowed[key] - } - return false -} - -func isSafePresentationURL(raw string, allowDataImage bool) bool { - if raw == "" { - return true - } - lower := strings.ToLower(raw) - if strings.HasPrefix(lower, "#") || strings.HasPrefix(lower, "/") || strings.HasPrefix(lower, "./") || strings.HasPrefix(lower, "../") { - return true - } - if allowDataImage && strings.HasPrefix(lower, "data:image/") { - return true - } - parsed, err := url.Parse(raw) - if err != nil { - return false - } - if parsed.Scheme == "" { - return true - } - switch strings.ToLower(parsed.Scheme) { - case "http", "https", "mailto", "tel": - return true - default: - return false - } -} - -func findHTMLElement(node *html.Node, tag string) *html.Node { - if node == nil { - return nil - } - if node.Type == html.ElementNode && strings.EqualFold(node.Data, tag) { - return node - } - for child := node.FirstChild; child != nil; child = child.NextSibling { - if found := findHTMLElement(child, tag); found != nil { - return found - } - } - return nil -} - -var presentationAllowedTags = map[string]bool{ - "a": true, "article": true, "aside": true, "b": true, "blockquote": true, - "br": true, "caption": true, "code": true, "dd": true, "div": true, - "dl": true, "dt": true, "em": true, "figcaption": true, "figure": true, - "footer": true, "h1": true, "h2": true, "h3": true, "h4": true, - "h5": true, "h6": true, "header": true, "hr": true, "i": true, - "img": true, "li": true, "main": true, "mark": true, "ol": true, - "p": true, "pre": true, "section": true, "small": true, "span": true, - "strong": true, "sub": true, "sup": true, "table": true, "tbody": true, - "td": true, "tfoot": true, "th": true, "thead": true, "tr": true, - "u": true, "ul": true, -} - -var presentationDropContentTags = map[string]bool{ - "embed": true, - "iframe": true, - "link": true, - "meta": true, - "noscript": true, - "object": true, - "script": true, - "style": true, -} - -var presentationVoidElements = map[string]bool{ - "area": true, "base": true, "br": true, "col": true, "embed": true, - "hr": true, "img": true, "input": true, "link": true, "meta": true, - "param": true, "source": true, "track": true, "wbr": true, -} - -var presentationGlobalAttrs = map[string]bool{ - "class": true, - "dir": true, - "id": true, - "lang": true, - "role": true, - "title": true, -} - -var presentationTagAttrs = map[string]map[string]bool{ - "a": { - "href": true, - "rel": true, - "target": true, - }, - "img": { - "alt": true, - "height": true, - "src": true, - "width": true, - }, - "td": { - "colspan": true, - "rowspan": true, - }, - "th": { - "colspan": true, - "rowspan": true, - "scope": true, - }, -} diff --git a/config/presentationConfig_test.go b/config/presentationConfig_test.go index 9adbe9f..2fee096 100644 --- a/config/presentationConfig_test.go +++ b/config/presentationConfig_test.go @@ -2,7 +2,7 @@ package config import "testing" -func TestPresentationConfigValidateSanitizesUnsafeHTML(t *testing.T) { +func TestPresentationConfigValidateAllowsRawHTML(t *testing.T) { cfg := &PresentationConfig{ PresentationConfig: `
link

Hello

`, } @@ -11,18 +11,8 @@ func TestPresentationConfigValidateSanitizesUnsafeHTML(t *testing.T) { t.Fatalf("validate failed: %v", err) } - if cfg.PresentationConfig != `
link

Hello

` { - t.Fatalf("unexpected sanitized HTML: %q", cfg.PresentationConfig) - } -} - -func TestPresentationConfigValidateRejectsMalformedHTML(t *testing.T) { - cfg := &PresentationConfig{ - PresentationConfig: `

broken

`, - } - - if err := cfg.Validate(); err == nil { - t.Fatal("expected malformed HTML validation error") + if cfg.PresentationConfig != `
link

Hello

` { + t.Fatalf("unexpected raw HTML: %q", cfg.PresentationConfig) } } diff --git a/docs/docs.go b/docs/docs.go index bb80cbb..6a93eab 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -217,7 +217,7 @@ const docTemplate = `{ }, "/git/projects/{orgTitle}/{projectTitle}/presentationConfig": { "get": { - "description": "Retrieve the sanitized presentation HTML configured for a project. Returns empty content when no presentation has been configured.", + "description": "Retrieve the raw presentation HTML configured for a project. Returns empty content when no presentation has been configured.", "produces": [ "application/json" ], @@ -275,7 +275,7 @@ const docTemplate = `{ } }, "post": { - "description": "Validate, sanitize, and persist presentation HTML for a project.", + "description": "Persist raw presentation HTML for a project.", "consumes": [ "application/json" ], @@ -313,13 +313,13 @@ const docTemplate = `{ ], "responses": { "200": { - "description": "Persisted sanitized presentation configuration", + "description": "Persisted presentation configuration", "schema": { "$ref": "#/definitions/config.PresentationConfig" } }, "400": { - "description": "Invalid request body or malformed HTML", + "description": "Invalid request body", "schema": { "$ref": "#/definitions/httputil.ErrorResponse" } @@ -351,7 +351,7 @@ const docTemplate = `{ } }, "put": { - "description": "Validate, sanitize, and persist presentation HTML for a project.", + "description": "Persist raw presentation HTML for a project.", "consumes": [ "application/json" ], @@ -389,13 +389,13 @@ const docTemplate = `{ ], "responses": { "200": { - "description": "Persisted sanitized presentation configuration", + "description": "Persisted presentation configuration", "schema": { "$ref": "#/definitions/config.PresentationConfig" } }, "400": { - "description": "Invalid request body or malformed HTML", + "description": "Invalid request body", "schema": { "$ref": "#/definitions/httputil.ErrorResponse" } diff --git a/docs/swagger.json b/docs/swagger.json index 400fac7..e54836a 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -211,7 +211,7 @@ }, "/git/projects/{orgTitle}/{projectTitle}/presentationConfig": { "get": { - "description": "Retrieve the sanitized presentation HTML configured for a project. Returns empty content when no presentation has been configured.", + "description": "Retrieve the raw presentation HTML configured for a project. Returns empty content when no presentation has been configured.", "produces": [ "application/json" ], @@ -269,7 +269,7 @@ } }, "post": { - "description": "Validate, sanitize, and persist presentation HTML for a project.", + "description": "Persist raw presentation HTML for a project.", "consumes": [ "application/json" ], @@ -307,13 +307,13 @@ ], "responses": { "200": { - "description": "Persisted sanitized presentation configuration", + "description": "Persisted presentation configuration", "schema": { "$ref": "#/definitions/config.PresentationConfig" } }, "400": { - "description": "Invalid request body or malformed HTML", + "description": "Invalid request body", "schema": { "$ref": "#/definitions/httputil.ErrorResponse" } @@ -345,7 +345,7 @@ } }, "put": { - "description": "Validate, sanitize, and persist presentation HTML for a project.", + "description": "Persist raw presentation HTML for a project.", "consumes": [ "application/json" ], @@ -383,13 +383,13 @@ ], "responses": { "200": { - "description": "Persisted sanitized presentation configuration", + "description": "Persisted presentation configuration", "schema": { "$ref": "#/definitions/config.PresentationConfig" } }, "400": { - "description": "Invalid request body or malformed HTML", + "description": "Invalid request body", "schema": { "$ref": "#/definitions/httputil.ErrorResponse" } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index ce04647..373462b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -835,7 +835,7 @@ paths: - Config /git/projects/{orgTitle}/{projectTitle}/presentationConfig: get: - description: Retrieve the sanitized presentation HTML configured for a project. Returns empty content when no presentation has been configured. + description: Retrieve the raw presentation HTML configured for a project. Returns empty content when no presentation has been configured. parameters: - description: Organization title in: path @@ -876,7 +876,7 @@ paths: post: consumes: - application/json - description: Validate, sanitize, and persist presentation HTML for a project. + description: Persist raw presentation HTML for a project. parameters: - description: Organization title in: path @@ -898,11 +898,11 @@ paths: - application/json responses: "200": - description: Persisted sanitized presentation configuration + description: Persisted presentation configuration schema: $ref: '#/definitions/config.PresentationConfig' "400": - description: Invalid request body or malformed HTML + description: Invalid request body schema: $ref: '#/definitions/httputil.ErrorResponse' "401": @@ -927,7 +927,7 @@ paths: put: consumes: - application/json - description: Validate, sanitize, and persist presentation HTML for a project. + description: Persist raw presentation HTML for a project. parameters: - description: Organization title in: path @@ -949,11 +949,11 @@ paths: - application/json responses: "200": - description: Persisted sanitized presentation configuration + description: Persisted presentation configuration schema: $ref: '#/definitions/config.PresentationConfig' "400": - description: Invalid request body or malformed HTML + description: Invalid request body schema: $ref: '#/definitions/httputil.ErrorResponse' "401": diff --git a/internal/server/http/git/presentation_config_test.go b/internal/server/http/git/presentation_config_test.go index 819ce51..abbf3c6 100644 --- a/internal/server/http/git/presentation_config_test.go +++ b/internal/server/http/git/presentation_config_test.go @@ -115,7 +115,7 @@ func TestGitProjectPresentationConfigGETNoConfigFallsBackToEmpty(t *testing.T) { } } -func TestGitProjectPresentationConfigPUTSanitizesAndPersistsHTML(t *testing.T) { +func TestGitProjectPresentationConfigPUTPersistsRawHTML(t *testing.T) { handler, mock, cleanup := newPresentationConfigTestServer(t) defer cleanup() expectProjectLookup(mock, "org-a", "proj-a") @@ -124,7 +124,7 @@ func TestGitProjectPresentationConfigPUTSanitizesAndPersistsHTML(t *testing.T) { PresentationConfig: `

Hello

`, } requestBody, _ := json.Marshal(requestPayload) - expectedStored := `

Hello

` + expectedStored := `

Hello

` app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) req := httptest.NewRequest(http.MethodPut, "/git/projects/org-a/proj-a/presentationConfig", bytes.NewReader(requestBody)) @@ -178,27 +178,6 @@ func TestGitProjectPresentationConfigPOSTUsesSameWritePath(t *testing.T) { } } -func TestGitProjectPresentationConfigPUTRejectsMalformedHTML(t *testing.T) { - handler, mock, cleanup := newPresentationConfigTestServer(t) - defer cleanup() - expectProjectLookup(mock, "org-a", "proj-a") - - requestBody := []byte(`{"presentationConfig":"

broken

"}`) - app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) - req := httptest.NewRequest(http.MethodPut, "/git/projects/org-a/proj-a/presentationConfig", bytes.NewReader(requestBody)) - req.Header.Set("Authorization", "Bearer test") - req.Header.Set("Content-Type", "application/json") - resp := runPresentationConfigRequest(t, app, req) - defer resp.Body.Close() - - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("expected status 400, got %d", resp.StatusCode) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet sql expectations: %v", err) - } -} - func TestGitProjectPresentationConfigPUTRejectsUnknownFields(t *testing.T) { handler, mock, cleanup := newPresentationConfigTestServer(t) defer cleanup() From 242411ea9ec60c431a4536f23484b36df15fb8cd Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 16 Jun 2026 14:20:08 -0700 Subject: [PATCH 09/36] patch gecko to have fallback authentication link be repo scoped so as to avoid any permissions issues --- docs/pr-feature-project-config.md | 20 ++++++++++++++++ internal/server/http/git/installation.go | 24 +++++++++++++++++-- internal/server/http/git/installation_test.go | 6 ++--- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/pr-feature-project-config.md b/docs/pr-feature-project-config.md index 79a87ab..a88be6f 100644 --- a/docs/pr-feature-project-config.md +++ b/docs/pr-feature-project-config.md @@ -284,6 +284,26 @@ These routes cover installation status, connect flows, and organization-wide rep These expose Gecko as a repository-backed project read service, not just a config API. +#### Known issue: inconsistent Git LFS classification on file detail responses + +As of June 15, 2026, the frontend has a defensive fix for Git LFS-backed files in the git-style file browser because some `GET /git/projects/:orgTitle/:projectTitle/file/*` responses appear to omit `lfs_pointer` even when the fetched file content is a valid Git LFS pointer. + +Current impact: + +- the Gecko contract is intended to mark LFS-backed files with `lfs_pointer` +- some file detail responses may still look like ordinary GitHub-backed text/blob responses +- when that happens, a naive client can show the raw GitHub LFS pointer content or link users to GitHub, which is not the desired UX + +Current mitigation: + +- the frontend now detects Git LFS pointer text directly in the fetched raw content +- if detected, it suppresses GitHub preview/link actions and routes the user to the Syfon download flow instead + +Follow-up: + +- Gecko should be investigated later to determine why some `/file/*` responses are not consistently populated with `lfs_pointer` +- once Gecko is made consistent, clients can treat `lfs_pointer` as the authoritative classification signal again + #### Project-level Git write/workflow routes - `PUT /git/projects/:orgTitle/:projectTitle/setup` diff --git a/internal/server/http/git/installation.go b/internal/server/http/git/installation.go index 868246e..dc727ad 100644 --- a/internal/server/http/git/installation.go +++ b/internal/server/http/git/installation.go @@ -76,7 +76,7 @@ func (handler *Handler) handleGitOrganizationInitConnectPOST(ctx fiber.Ctx) erro targetID, repoID, resolveErr := handler.gitService.ResolveTargetAndRepositoryIDs(connectCtx, identity) if resolveErr != nil { handler.logger.Warning(fmt.Sprintf("skipping GitHub install redirect optimization for %s/%s: %v", identity.Owner, identity.Repo, resolveErr)) - if settingsURL, ok := handler.organizationInstallationSettingsURL(connectCtx, authorizationHeader, organization, identity.Owner); ok { + if settingsURL, ok := handler.organizationInstallationSettingsURL(connectCtx, authorizationHeader, organization, identity.Owner, redirectURL); ok { redirectURL = settingsURL } } else { @@ -90,7 +90,7 @@ func (handler *Handler) handleGitOrganizationInitConnectPOST(ctx fiber.Ctx) erro }, http.StatusOK).Write(ctx) } -func (handler *Handler) organizationInstallationSettingsURL(ctx context.Context, authorizationHeader string, organization string, owner string) (string, bool) { +func (handler *Handler) organizationInstallationSettingsURL(ctx context.Context, authorizationHeader string, organization string, owner string, redirectURL string) (string, bool) { installation, err := handler.gitService.RequestOrganizationInstallationStatus(ctx, authorizationHeader, organization, owner) if err != nil { handler.logger.Warning(fmt.Sprintf("failed to load GitHub organization installation for %s: %v", owner, err)) @@ -102,6 +102,11 @@ func (handler *Handler) organizationInstallationSettingsURL(ctx context.Context, if installation.InstallationID == nil || *installation.InstallationID <= 0 { return "", false } + + if appInstallationURL := appInstallationRedirectURL(strings.TrimSpace(redirectURL), *installation.InstallationID); appInstallationURL != "" { + return appInstallationURL, true + } + owner = strings.TrimSpace(owner) if owner == "" { return "", false @@ -390,6 +395,21 @@ func decorateInstallationRedirectURL(redirectURL string, targetID int64, repoID return fmt.Sprintf("%s&repository_ids[]=%d", redirectURL, repoID) } +func appInstallationRedirectURL(redirectURL string, installationID int64) string { + if installationID <= 0 || strings.TrimSpace(redirectURL) == "" { + return "" + } + request, err := http.NewRequest(http.MethodGet, redirectURL, nil) + if err != nil || request.URL == nil { + return "" + } + pathParts := strings.Split(strings.Trim(request.URL.Path, "/"), "/") + if len(pathParts) < 2 || pathParts[0] != "apps" || strings.TrimSpace(pathParts[1]) == "" { + return "" + } + return fmt.Sprintf("%s://%s/apps/%s/installations/%d", request.URL.Scheme, request.URL.Host, pathParts[1], installationID) +} + func (handler *Handler) loadProjectConfig(ctx context.Context, projectID string) (appconfig.ProjectConfig, *httputil.ErrorResponse) { var projectCfg appconfig.ProjectConfig if err := geckodb.ConfigGETGenericContext(ctx, handler.db, projectID, string(appconfig.TypeProjects), &projectCfg); err != nil { diff --git a/internal/server/http/git/installation_test.go b/internal/server/http/git/installation_test.go index 1f9828d..0c57e39 100644 --- a/internal/server/http/git/installation_test.go +++ b/internal/server/http/git/installation_test.go @@ -205,7 +205,7 @@ func TestGitOrganizationInitConnectFallsBackToPlainRedirectWhenRepositoryLookupF } } -func TestGitOrganizationInitConnectFallsBackToCleanOrgSettingsURLWhenRepositoryLookupFails(t *testing.T) { +func TestGitOrganizationInitConnectFallsBackToAppInstallationURLWhenRepositoryLookupFails(t *testing.T) { fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { var receivedBody map[string]any if err := json.NewDecoder(request.Body).Decode(&receivedBody); err != nil { @@ -268,8 +268,8 @@ func TestGitOrganizationInitConnectFallsBackToCleanOrgSettingsURLWhenRepositoryL if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { t.Fatalf("decode response: %v", err) } - if payload.RedirectURL != "https://github.com/organizations/EllrottLab/settings/installations/134470697" { - t.Fatalf("expected clean organization settings redirect when repo lookup fails, got %q", payload.RedirectURL) + if payload.RedirectURL != "https://github.com/apps/calypr-github/installations/134470697" { + t.Fatalf("expected app installation redirect when repo lookup fails, got %q", payload.RedirectURL) } if strings.Contains(payload.RedirectURL, "suggested_target_id=") || strings.Contains(payload.RedirectURL, "repository_ids") { t.Fatalf("did not expect partial redirect optimization or empty repository_ids, got %q", payload.RedirectURL) From b2d1038e045086e354aa812489790182784109fe Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 25 Jun 2026 15:48:58 -0700 Subject: [PATCH 10/36] optimize git endpoints --- internal/git/response.go | 198 +++++++++++++++++-- internal/git/service_test.go | 255 ++++++++++++++++++++++++- internal/git/types.go | 33 +++- internal/server/http/git/register.go | 2 + internal/server/http/git/repository.go | 165 +++++++++++++++- 5 files changed, 628 insertions(+), 25 deletions(-) diff --git a/internal/git/response.go b/internal/git/response.go index d3615cd..4e19bee 100644 --- a/internal/git/response.go +++ b/internal/git/response.go @@ -2,6 +2,8 @@ package git import ( "context" + "encoding/base64" + "encoding/json" "fmt" "io" "net/http" @@ -9,14 +11,15 @@ import ( "sort" "strings" + servermw "github.com/calypr/gecko/internal/server/middleware" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/google/go-github/v87/github" - servermw "github.com/calypr/gecko/internal/server/middleware" ) -func BuildGitTreeResponse(projectID string, ref string, path string, repo *gogit.Repository, hash plumbing.Hash) (*GitProjectTreeResponse, error) { +func BuildGitTreeResponse(projectID string, ref string, path string, repo *gogit.Repository, hash plumbing.Hash, options GitTreeResponseOptions) (*GitProjectTreeResponse, error) { commit, err := repo.CommitObject(hash) if err != nil { return nil, fmt.Errorf("load commit for ref %s: %w", ref, err) @@ -43,29 +46,196 @@ func BuildGitTreeResponse(projectID string, ref string, path string, repo *gogit gitEntry.Type = "tree" } else { gitEntry.Type = "blob" + } + entries = append(entries, gitEntry) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].Type != entries[j].Type { + return entries[i].Type == "tree" + } + return strings.ToLower(entries[i].Name) < strings.ToLower(entries[j].Name) + }) + + entryCount := len(entries) + truncated := false + if options.Limit > 0 && len(entries) > options.Limit { + entries = entries[:options.Limit] + truncated = true + } + + for index := range entries { + entry := &entries[index] + if entry.Type != "blob" { + if options.IncludeLastModified { + if lastModifiedAt, err := lookupGitPathLastModified(repo, hash, entry.Path); err == nil && lastModifiedAt != nil { + entry.LastModifiedAt = lastModifiedAt + } + } + continue + } + + needsFileOpen := options.IncludeSize || options.IncludeLFSPointer + if needsFileOpen { if file, err := tree.File(entry.Name); err == nil { - gitEntry.Size = file.Size - if reader, err := file.Reader(); err == nil { - contentBytes, readErr := io.ReadAll(io.LimitReader(reader, 2048)) - _ = reader.Close() - if readErr == nil { - gitEntry.LFSPointer = ParseGitLFSPointer(contentBytes) + if options.IncludeSize { + entry.Size = file.Size + } + if options.IncludeLFSPointer { + if reader, err := file.Reader(); err == nil { + contentBytes, readErr := io.ReadAll(io.LimitReader(reader, 2048)) + _ = reader.Close() + if readErr == nil { + entry.LFSPointer = ParseGitLFSPointer(contentBytes) + } } } } } - if lastModifiedAt, err := lookupGitPathLastModified(repo, hash, entryPath); err == nil && lastModifiedAt != nil { - gitEntry.LastModifiedAt = lastModifiedAt + + if options.IncludeLastModified { + if lastModifiedAt, err := lookupGitPathLastModified(repo, hash, entry.Path); err == nil && lastModifiedAt != nil { + entry.LastModifiedAt = lastModifiedAt + } } - entries = append(entries, gitEntry) + } + + return &GitProjectTreeResponse{ + ProjectID: projectID, + Ref: ref, + Path: normalizedPath, + EntryCount: entryCount, + Truncated: truncated, + Entries: entries, + }, nil +} + +type gitManifestCursor struct { + Ref string `json:"ref"` + Path string `json:"path"` + Offset int `json:"offset"` +} + +func BuildGitManifestResponse(projectID string, ref string, path string, repo *gogit.Repository, hash plumbing.Hash, options GitManifestResponseOptions) (*GitProjectManifestResponse, error) { + commit, err := repo.CommitObject(hash) + if err != nil { + return nil, fmt.Errorf("load commit for ref %s: %w", ref, err) + } + tree, err := commit.Tree() + if err != nil { + return nil, fmt.Errorf("load git tree for ref %s: %w", ref, err) + } + normalizedPath := strings.Trim(strings.TrimSpace(path), "/") + if normalizedPath != "" { + tree, err = tree.Tree(normalizedPath) + if err != nil { + return nil, fmt.Errorf("load git tree path %s: %w", normalizedPath, err) + } + } + + offset, err := parseGitManifestCursor(options.Cursor, ref, normalizedPath) + if err != nil { + return nil, err + } + + entries := make([]GitTreeEntry, 0, len(tree.Entries)) + if err := walkGitManifestTree(normalizedPath, tree, options.FilesOnly, &entries); err != nil { + return nil, err } sort.Slice(entries, func(i, j int) bool { - if entries[i].Type != entries[j].Type { + if !options.FilesOnly && entries[i].Type != entries[j].Type { return entries[i].Type == "tree" } - return strings.ToLower(entries[i].Name) < strings.ToLower(entries[j].Name) + return entries[i].Path < entries[j].Path + }) + if offset > len(entries) { + return nil, fmt.Errorf("invalid manifest cursor") + } + + end := len(entries) + if options.Limit > 0 && offset+options.Limit < end { + end = offset + options.Limit + } + pageEntries := append([]GitTreeEntry(nil), entries[offset:end]...) + hasMore := end < len(entries) + nextCursor := "" + if hasMore { + nextCursor, err = encodeGitManifestCursor(ref, normalizedPath, end) + if err != nil { + return nil, err + } + } + + return &GitProjectManifestResponse{ + ProjectID: projectID, + Ref: ref, + Path: normalizedPath, + EntryCount: len(pageEntries), + HasMore: hasMore, + NextCursor: nextCursor, + Entries: pageEntries, + }, nil +} + +func walkGitManifestTree(prefix string, tree *object.Tree, filesOnly bool, entries *[]GitTreeEntry) error { + for _, entry := range tree.Entries { + entryPath := entry.Name + if prefix != "" { + entryPath = prefix + "/" + entry.Name + } + entryType := "blob" + if entry.Mode == filemode.Dir { + entryType = "tree" + } + if !filesOnly || entryType == "blob" { + *entries = append(*entries, GitTreeEntry{ + Name: entry.Name, + Path: entryPath, + Type: entryType, + Hash: entry.Hash.String(), + }) + } + if entryType == "tree" { + childTree, err := tree.Tree(entry.Name) + if err != nil { + return fmt.Errorf("load nested git tree %s: %w", entryPath, err) + } + if err := walkGitManifestTree(entryPath, childTree, filesOnly, entries); err != nil { + return err + } + } + } + return nil +} + +func parseGitManifestCursor(raw string, ref string, path string) (int, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return 0, nil + } + decoded, err := base64.RawURLEncoding.DecodeString(raw) + if err != nil { + return 0, fmt.Errorf("invalid manifest cursor") + } + var payload gitManifestCursor + if err := json.Unmarshal(decoded, &payload); err != nil { + return 0, fmt.Errorf("invalid manifest cursor") + } + if payload.Ref != ref || payload.Path != path || payload.Offset < 0 { + return 0, fmt.Errorf("invalid manifest cursor") + } + return payload.Offset, nil +} + +func encodeGitManifestCursor(ref string, path string, offset int) (string, error) { + payload, err := json.Marshal(gitManifestCursor{ + Ref: ref, + Path: path, + Offset: offset, }) - return &GitProjectTreeResponse{ProjectID: projectID, Ref: ref, Path: normalizedPath, Entries: entries}, nil + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(payload), nil } func BuildGitRefsResponse(projectID string, defaultBranch string, repo *gogit.Repository) (*GitProjectRefsResponse, error) { diff --git a/internal/git/service_test.go b/internal/git/service_test.go index fc7f5e2..5b0859e 100644 --- a/internal/git/service_test.go +++ b/internal/git/service_test.go @@ -46,7 +46,7 @@ func TestSyncRepositoryMirrorPullsUpdatesAndReadsTree(t *testing.T) { if err != nil { t.Fatalf("resolve HEAD: %v", err) } - treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash) + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{}) if err != nil { t.Fatalf("build tree response: %v", err) } @@ -225,7 +225,10 @@ func TestBuildGitResponsesDetectLFSPointers(t *testing.T) { if err != nil { t.Fatalf("resolve HEAD: %v", err) } - treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "data", mirrorRepo, hash) + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "data", mirrorRepo, hash, GitTreeResponseOptions{ + IncludeLFSPointer: true, + IncludeSize: true, + }) if err != nil { t.Fatalf("build tree response: %v", err) } @@ -254,3 +257,251 @@ func TestBuildGitResponsesDetectLFSPointers(t *testing.T) { t.Fatalf("expected matching lfs oid, got %q and %q", fileResponse.LFSPointer.OID, treePointer.OID) } } + +func TestBuildGitTreeResponseDefaultsToCheapFields(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + if err := os.WriteFile(filepath.Join(sourcePath, "README.md"), []byte("hello gecko"), 0o644); err != nil { + t.Fatalf("write readme: %v", err) + } + if _, err := worktree.Add("README.md"); err != nil { + t.Fatalf("add readme: %v", err) + } + if _, err := worktree.Commit("initial commit", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit readme: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{}) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if treeResponse.EntryCount != 1 { + t.Fatalf("expected entry count 1, got %d", treeResponse.EntryCount) + } + if treeResponse.Truncated { + t.Fatal("expected non-truncated response by default") + } + if len(treeResponse.Entries) != 1 { + t.Fatalf("expected one tree entry, got %+v", treeResponse.Entries) + } + if treeResponse.Entries[0].Size != 0 { + t.Fatalf("expected default tree response to omit size, got %d", treeResponse.Entries[0].Size) + } + if treeResponse.Entries[0].LFSPointer != nil { + t.Fatalf("expected default tree response to omit lfs pointer, got %+v", treeResponse.Entries[0].LFSPointer) + } + if treeResponse.Entries[0].LastModifiedAt != nil { + t.Fatalf("expected default tree response to omit last modified, got %+v", treeResponse.Entries[0].LastModifiedAt) + } +} + +func TestBuildGitTreeResponseHonorsLimitBeforeEnrichment(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + for _, name := range []string{"a.txt", "b.txt", "c.txt"} { + if err := os.WriteFile(filepath.Join(sourcePath, name), []byte(name), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + if _, err := worktree.Add(name); err != nil { + t.Fatalf("add %s: %v", name, err) + } + } + if _, err := worktree.Commit("add files", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit files: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{ + IncludeSize: true, + Limit: 2, + }) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if !treeResponse.Truncated { + t.Fatal("expected truncated response when limit is smaller than entry count") + } + if treeResponse.EntryCount != 3 { + t.Fatalf("expected total entry count 3, got %d", treeResponse.EntryCount) + } + if len(treeResponse.Entries) != 2 { + t.Fatalf("expected two returned entries, got %+v", treeResponse.Entries) + } + for _, entry := range treeResponse.Entries { + if entry.Size == 0 { + t.Fatalf("expected size to be included for limited entry %+v", entry) + } + } +} + +func TestBuildGitManifestResponseRecursesAndPaginates(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + for path, content := range map[string]string{ + "README.md": "hello", + "data/a.txt": "a", + "data/nested/b.txt": "b", + "data/nested/c/c.txt": "c", + } { + fullPath := filepath.Join(sourcePath, filepath.FromSlash(path)) + if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } + if err := os.WriteFile(fullPath, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + if _, err := worktree.Add(path); err != nil { + t.Fatalf("add %s: %v", path, err) + } + } + if _, err := worktree.Commit("add files", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit files: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + firstPage, err := BuildGitManifestResponse("org-a/proj-a", refName, "data", mirrorRepo, hash, GitManifestResponseOptions{ + FilesOnly: true, + Limit: 2, + }) + if err != nil { + t.Fatalf("build first manifest page: %v", err) + } + if firstPage.EntryCount != 2 { + t.Fatalf("expected two entries on first page, got %d", firstPage.EntryCount) + } + if !firstPage.HasMore || strings.TrimSpace(firstPage.NextCursor) == "" { + t.Fatalf("expected pagination cursor, got %+v", firstPage) + } + if got := []string{firstPage.Entries[0].Path, firstPage.Entries[1].Path}; strings.Join(got, ",") != "data/a.txt,data/nested/b.txt" { + t.Fatalf("unexpected first page paths: %v", got) + } + + secondPage, err := BuildGitManifestResponse("org-a/proj-a", refName, "data", mirrorRepo, hash, GitManifestResponseOptions{ + FilesOnly: true, + Limit: 2, + Cursor: firstPage.NextCursor, + }) + if err != nil { + t.Fatalf("build second manifest page: %v", err) + } + if secondPage.HasMore { + t.Fatalf("expected second page to be terminal, got %+v", secondPage) + } + if got := []string{secondPage.Entries[0].Path}; strings.Join(got, ",") != "data/nested/c/c.txt" { + t.Fatalf("unexpected second page paths: %v", got) + } +} + +func TestBuildGitManifestResponseIncludesDirectoriesWhenRequested(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + fullPath := filepath.Join(sourcePath, "data", "nested", "b.txt") + if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil { + t.Fatalf("mkdir nested: %v", err) + } + if err := os.WriteFile(fullPath, []byte("b"), 0o644); err != nil { + t.Fatalf("write file: %v", err) + } + if _, err := worktree.Add("data/nested/b.txt"); err != nil { + t.Fatalf("add file: %v", err) + } + if _, err := worktree.Commit("add file", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit file: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + manifest, err := BuildGitManifestResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitManifestResponseOptions{ + FilesOnly: false, + Limit: 20, + }) + if err != nil { + t.Fatalf("build manifest: %v", err) + } + if len(manifest.Entries) != 3 { + t.Fatalf("expected tree+tree+blob entries, got %+v", manifest.Entries) + } + if manifest.Entries[0].Type != "tree" || manifest.Entries[1].Type != "tree" || manifest.Entries[2].Type != "blob" { + t.Fatalf("unexpected manifest entry ordering: %+v", manifest.Entries) + } +} diff --git a/internal/git/types.go b/internal/git/types.go index bab6558..5c854c6 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -230,11 +230,36 @@ type GitTreeEntry struct { LFSPointer *GitLFSPointerInfo `json:"lfs_pointer,omitempty"` } +type GitTreeResponseOptions struct { + IncludeSize bool + IncludeLastModified bool + IncludeLFSPointer bool + Limit int +} + type GitProjectTreeResponse struct { - ProjectID string `json:"project_id"` - Ref string `json:"ref"` - Path string `json:"path"` - Entries []GitTreeEntry `json:"entries"` + ProjectID string `json:"project_id"` + Ref string `json:"ref"` + Path string `json:"path"` + EntryCount int `json:"entry_count"` + Truncated bool `json:"truncated,omitempty"` + Entries []GitTreeEntry `json:"entries"` +} + +type GitManifestResponseOptions struct { + Limit int + Cursor string + FilesOnly bool +} + +type GitProjectManifestResponse struct { + ProjectID string `json:"project_id"` + Ref string `json:"ref"` + Path string `json:"path"` + EntryCount int `json:"entry_count"` + HasMore bool `json:"has_more"` + NextCursor string `json:"next_cursor,omitempty"` + Entries []GitTreeEntry `json:"entries"` } type GitProjectFileResponse struct { diff --git a/internal/server/http/git/register.go b/internal/server/http/git/register.go index 47c8a4f..53da818 100644 --- a/internal/server/http/git/register.go +++ b/internal/server/http/git/register.go @@ -29,6 +29,8 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler gitGroup.Get("/projects/:orgTitle/:projectTitle/refs", projectReadAuth, handler.handleGitProjectRefsGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/tree", projectReadAuth, handler.handleGitProjectTreeGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/tree/*", projectReadAuth, handler.handleGitProjectTreeGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/manifest", projectReadAuth, handler.handleGitProjectManifestGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/manifest/*", projectReadAuth, handler.handleGitProjectManifestGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/file/*", projectReadAuth, handler.handleGitProjectFileGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/download/*", projectReadAuth, handler.handleGitProjectDownloadGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/thumbnail", handler.handleGitProjectThumbnailGET) diff --git a/internal/server/http/git/repository.go b/internal/server/http/git/repository.go index 7863777..94e1ee4 100644 --- a/internal/server/http/git/repository.go +++ b/internal/server/http/git/repository.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strconv" "strings" "time" @@ -91,10 +92,11 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { refName = state.DefaultBranch.String } return httputil.JSON(&git.GitProjectTreeResponse{ - ProjectID: projectID, - Ref: refName, - Path: strings.Trim(ctx.Params("*"), "/"), - Entries: []git.GitTreeEntry{}, + ProjectID: projectID, + Ref: refName, + Path: strings.Trim(ctx.Params("*"), "/"), + EntryCount: 0, + Entries: []git.GitTreeEntry{}, }, http.StatusOK).Write(ctx) } refName, hash, err := git.ResolveGitReference(repo, strings.TrimSpace(ctx.Query("ref")), state.DefaultBranch.String) @@ -104,7 +106,13 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { return response.Write(ctx) } path := strings.Trim(ctx.Params("*"), "/") - treeResponse, err := git.BuildGitTreeResponse(projectID, refName, path, repo, hash) + treeOptions, treeOptionErr := buildGitTreeResponseOptions(ctx) + if treeOptionErr != nil { + response := httputil.NewError("invalid_request", treeOptionErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + treeResponse, err := git.BuildGitTreeResponse(projectID, refName, path, repo, hash, treeOptions) if err != nil { response := httputil.NewError("not_found", fmt.Sprintf("failed to read git tree: %s", err), http.StatusNotFound, map[string]any{"project_id": projectID, "ref": refName, "path": path}, nil) response.WriteLog(handler.logger) @@ -113,6 +121,153 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { return httputil.JSON(treeResponse, http.StatusOK).Write(ctx) } +func (handler *Handler) handleGitProjectManifestGET(ctx fiber.Ctx) error { + _, _, projectID, _, identity, errResponse := handler.resolveGitProject(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + state, err := handler.loadGitProjectState(projectID, identity) + if err != nil { + response := httputil.NewError("database_error", fmt.Sprintf("failed to read git state: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + if state == nil || state.MirrorPath == "" { + response := httputil.NewError("conflict", fmt.Sprintf("project %s has not been refreshed yet", projectID), http.StatusConflict, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + authorizationHeader := strings.TrimSpace(ctx.Get("Authorization")) + if authorizationHeader != "" { + refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + if err != nil { + handler.logger.Warning("failed to warm git mirror for %s manifest: %v", projectID, err) + } + } + repo, err := git.OpenRepository(state.MirrorPath) + if err != nil { + response := httputil.NewError("integration_error", fmt.Sprintf("failed to open git mirror: %s", err), http.StatusBadGateway, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + if git.RepositoryIsEmpty(repo) { + refName := strings.TrimSpace(ctx.Query("ref")) + if refName == "" { + refName = state.DefaultBranch.String + } + return httputil.JSON(&git.GitProjectManifestResponse{ + ProjectID: projectID, + Ref: refName, + Path: strings.Trim(ctx.Params("*"), "/"), + EntryCount: 0, + HasMore: false, + Entries: []git.GitTreeEntry{}, + }, http.StatusOK).Write(ctx) + } + refName, hash, err := git.ResolveGitReference(repo, strings.TrimSpace(ctx.Query("ref")), state.DefaultBranch.String) + if err != nil { + response := httputil.NewError("not_found", fmt.Sprintf("failed to resolve git ref: %s", err), http.StatusNotFound, map[string]any{"project_id": projectID, "ref": ctx.Query("ref")}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + path := strings.Trim(ctx.Params("*"), "/") + manifestOptions, manifestOptionErr := buildGitManifestResponseOptions(ctx) + if manifestOptionErr != nil { + response := httputil.NewError("invalid_request", manifestOptionErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + manifestResponse, err := git.BuildGitManifestResponse(projectID, refName, path, repo, hash, manifestOptions) + if err != nil { + statusCode := http.StatusNotFound + if strings.Contains(strings.ToLower(err.Error()), "cursor") || strings.Contains(strings.ToLower(err.Error()), "limit") { + statusCode = http.StatusBadRequest + } + response := httputil.NewError("not_found", fmt.Sprintf("failed to read git manifest: %s", err), statusCode, map[string]any{"project_id": projectID, "ref": refName, "path": path}, nil) + if statusCode == http.StatusBadRequest { + response = httputil.NewError("invalid_request", err.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID, "ref": refName, "path": path}, nil) + } + response.WriteLog(handler.logger) + return response.Write(ctx) + } + return httputil.JSON(manifestResponse, http.StatusOK).Write(ctx) +} + +func parseOptionalBoolQuery(ctx fiber.Ctx, key string, defaultValue bool) (bool, error) { + value := strings.TrimSpace(ctx.Query(key)) + if value == "" { + return defaultValue, nil + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf("%s must be true or false", key) + } + return parsed, nil +} + +func buildGitTreeResponseOptions(ctx fiber.Ctx) (git.GitTreeResponseOptions, error) { + view := strings.TrimSpace(ctx.Query("view")) + if strings.EqualFold(view, "manifest") { + return git.GitTreeResponseOptions{}, nil + } + + includeSize, err := parseOptionalBoolQuery(ctx, "include_size", false) + if err != nil { + return git.GitTreeResponseOptions{}, err + } + includeLastModified, err := parseOptionalBoolQuery(ctx, "include_last_modified", false) + if err != nil { + return git.GitTreeResponseOptions{}, err + } + includeLFSPointer, err := parseOptionalBoolQuery(ctx, "include_lfs_pointer", false) + if err != nil { + return git.GitTreeResponseOptions{}, err + } + + limitValue := strings.TrimSpace(ctx.Query("limit")) + limit := 0 + if limitValue != "" { + parsedLimit, parseErr := strconv.Atoi(limitValue) + if parseErr != nil || parsedLimit < 0 { + return git.GitTreeResponseOptions{}, fmt.Errorf("limit must be a non-negative integer") + } + limit = parsedLimit + } + + return git.GitTreeResponseOptions{ + IncludeSize: includeSize, + IncludeLastModified: includeLastModified, + IncludeLFSPointer: includeLFSPointer, + Limit: limit, + }, nil +} + +func buildGitManifestResponseOptions(ctx fiber.Ctx) (git.GitManifestResponseOptions, error) { + filesOnly, err := parseOptionalBoolQuery(ctx, "files_only", true) + if err != nil { + return git.GitManifestResponseOptions{}, err + } + limit := 5000 + limitValue := strings.TrimSpace(ctx.Query("limit")) + if limitValue != "" { + parsedLimit, parseErr := strconv.Atoi(limitValue) + if parseErr != nil || parsedLimit <= 0 { + return git.GitManifestResponseOptions{}, fmt.Errorf("limit must be a positive integer") + } + if parsedLimit > 20000 { + return git.GitManifestResponseOptions{}, fmt.Errorf("limit must be less than or equal to 20000") + } + limit = parsedLimit + } + return git.GitManifestResponseOptions{ + Limit: limit, + Cursor: strings.TrimSpace(ctx.Query("cursor")), + FilesOnly: filesOnly, + }, nil +} + func (handler *Handler) handleGitProjectFileGET(ctx fiber.Ctx) error { organization, project, projectID, _, identity, errResponse := handler.resolveGitProject(ctx) if errResponse != nil { From e65d7b2ac01b53d4f11233d2bd87f4febb2a5e85 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 26 Jun 2026 09:57:46 -0700 Subject: [PATCH 11/36] patch an issue related to directories with spaces in their name --- internal/server/http/git/repository.go | 57 +++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/internal/server/http/git/repository.go b/internal/server/http/git/repository.go index 94e1ee4..370f82d 100644 --- a/internal/server/http/git/repository.go +++ b/internal/server/http/git/repository.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "net/url" "strconv" "strings" "time" @@ -91,10 +92,16 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { if refName == "" { refName = state.DefaultBranch.String } + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } return httputil.JSON(&git.GitProjectTreeResponse{ ProjectID: projectID, Ref: refName, - Path: strings.Trim(ctx.Params("*"), "/"), + Path: path, EntryCount: 0, Entries: []git.GitTreeEntry{}, }, http.StatusOK).Write(ctx) @@ -105,7 +112,12 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - path := strings.Trim(ctx.Params("*"), "/") + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } treeOptions, treeOptionErr := buildGitTreeResponseOptions(ctx) if treeOptionErr != nil { response := httputil.NewError("invalid_request", treeOptionErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) @@ -157,10 +169,16 @@ func (handler *Handler) handleGitProjectManifestGET(ctx fiber.Ctx) error { if refName == "" { refName = state.DefaultBranch.String } + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } return httputil.JSON(&git.GitProjectManifestResponse{ ProjectID: projectID, Ref: refName, - Path: strings.Trim(ctx.Params("*"), "/"), + Path: path, EntryCount: 0, HasMore: false, Entries: []git.GitTreeEntry{}, @@ -172,7 +190,12 @@ func (handler *Handler) handleGitProjectManifestGET(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - path := strings.Trim(ctx.Params("*"), "/") + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } manifestOptions, manifestOptionErr := buildGitManifestResponseOptions(ctx) if manifestOptionErr != nil { response := httputil.NewError("invalid_request", manifestOptionErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) @@ -279,7 +302,12 @@ func (handler *Handler) handleGitProjectFileGET(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - path := strings.Trim(ctx.Params("*"), "/") + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } requestedRef := strings.TrimSpace(ctx.Query("ref")) metadata, contentBytes, err := handler.gitService.GetGitHubFileMetadata(ctx.Context(), authorizationHeader, organization, project, identity, requestedRef, path) if err != nil { @@ -310,7 +338,12 @@ func (handler *Handler) handleGitProjectDownloadGET(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - path := strings.Trim(ctx.Params("*"), "/") + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } requestedRef := strings.TrimSpace(ctx.Query("ref")) metadata, _, err := handler.gitService.GetGitHubFileMetadata(ctx.Context(), authorizationHeader, organization, project, identity, requestedRef, path) if err != nil { @@ -333,3 +366,15 @@ func (handler *Handler) handleGitProjectDownloadGET(ctx fiber.Ctx) error { } return ctx.Redirect().To(strings.TrimSpace(metadata.GetDownloadURL())) } + +func decodeGitWildcardPath(ctx fiber.Ctx) (string, error) { + rawPath := strings.Trim(ctx.Params("*"), "/") + if rawPath == "" { + return "", nil + } + decodedPath, err := url.PathUnescape(rawPath) + if err != nil { + return "", fmt.Errorf("invalid git path") + } + return strings.Trim(decodedPath, "/"), nil +} From 1f9bc46341cb0e475bf95ba09ff4479b5a7c3f90 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 1 Jul 2026 14:23:01 -0700 Subject: [PATCH 12/36] fix up audit methods in gecko --- internal/git/domain/types.go | 7 + internal/git/service.go | 13 + internal/git/storage_analytics.go | 1997 +++++++++++++++++ internal/git/storage_analytics_pipeline.go | 628 ++++++ internal/git/storage_analytics_test.go | 1334 +++++++++++ internal/git/storage_index.go | 335 +++ internal/git/types.go | 256 +++ internal/integrations/syfon/adapter.go | 793 +++++++ internal/integrations/syfon/adapter_test.go | 233 ++ internal/server/http/git/handler.go | 6 + internal/server/http/git/installation.go | 6 +- internal/server/http/git/register.go | 6 + internal/server/http/git/storage_analytics.go | 322 +++ internal/server/http/shared/handler.go | 5 +- internal/server/server.go | 4 +- 15 files changed, 5939 insertions(+), 6 deletions(-) create mode 100644 internal/git/storage_analytics.go create mode 100644 internal/git/storage_analytics_pipeline.go create mode 100644 internal/git/storage_analytics_test.go create mode 100644 internal/git/storage_index.go create mode 100644 internal/integrations/syfon/adapter_test.go create mode 100644 internal/server/http/git/storage_analytics.go diff --git a/internal/git/domain/types.go b/internal/git/domain/types.go index 8804f0d..8886710 100644 --- a/internal/git/domain/types.go +++ b/internal/git/domain/types.go @@ -39,6 +39,13 @@ type StorageBucket struct { Resources []string } +type StorageBucketScope struct { + Bucket string + Organization string + ProjectID string + Path string +} + type StorageConfig struct { Bucket string Provider string diff --git a/internal/git/service.go b/internal/git/service.go index 54af114..d284813 100644 --- a/internal/git/service.go +++ b/internal/git/service.go @@ -64,6 +64,19 @@ func (service *GitService) RefreshProject(ctx context.Context, projectID string, if err := SyncRepositoryMirror(ctx, cloneURL, state.MirrorPath, &githttp.BasicAuth{Username: "x-access-token", Password: accessToken}); err != nil { return nil, state, err } + repo, err := OpenRepository(state.MirrorPath) + if err != nil { + return nil, state, fmt.Errorf("open refreshed git mirror: %w", err) + } + if !RepositoryIsEmpty(repo) { + refName, hash, err := ResolveGitReference(repo, repoMetadata.DefaultBranch, repoMetadata.DefaultBranch) + if err != nil { + return nil, state, fmt.Errorf("resolve refreshed git ref: %w", err) + } + if err := PersistRepoAnalyticsIndex(ctx, state.MirrorPath, repo, refName, hash); err != nil { + return nil, state, fmt.Errorf("persist repo analytics index: %w", err) + } + } updated := *state updated.InstallationTarget = sql.NullString{String: identity.Owner, Valid: identity.Owner != ""} updated.InstallationTargetType = sql.NullString{String: "Organization", Valid: identity.Owner != ""} diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go new file mode 100644 index 0000000..ed5d4c2 --- /dev/null +++ b/internal/git/storage_analytics.go @@ -0,0 +1,1997 @@ +package git + +import ( + "context" + "fmt" + "path" + "sort" + "strings" + "sync" + "time" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +const cleanupInactiveDays = 30 +const projectJoinCacheTTL = 45 * time.Second + +type storageAnalyticsBackend interface { + ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) + ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) + ListProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) + ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) + ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) + BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]gintegrationsyfon.ProjectRecord, error) + ListProjectFileUsage(ctx context.Context, authorizationHeader string, organization string, project string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) + ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectBucketObject, error) + BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) + BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error + DeleteProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, objectURLs []string) ([]gintegrationsyfon.ProjectBucketDeleteResult, error) + BulkUpdateAccessMethods(ctx context.Context, authorizationHeader string, updates map[string][]gintegrationsyfon.ProjectAccessMethod) error +} + +type StorageAnalyticsService struct { + storage storageAnalyticsBackend + projectJoinMu sync.RWMutex + projectJoinCache map[string]cachedProjectJoinState +} + +func NewStorageAnalyticsService(storage storageAnalyticsBackend) *StorageAnalyticsService { + if storage == nil { + return nil + } + return &StorageAnalyticsService{ + storage: storage, + projectJoinCache: map[string]cachedProjectJoinState{}, + } +} + +type RepoInventoryFile struct { + RepoPath string + Name string + Checksum string + Size int64 +} + +type projectRecordState struct { + gintegrationsyfon.ProjectRecord + CanonicalAccessURLs []string + Usage gintegrationsyfon.FileUsage + AccessProbes []gintegrationsyfon.BulkStorageProbeResult +} + +type storageAggregate struct { + name string + path string + rowType string + fileCount int + recordCount int + totalBytes int64 + downloadCount int64 + lastDownload *time.Time + latestUpdate *time.Time + duplicateCount int +} + +type projectDiffAuditModel struct { + Findings []GitProjectDiffFinding + Summary GitProjectDiffSummary + PathPrefix string +} + +type cleanupFindingModel struct { + Public GitStorageCleanupFinding + DeleteObjectIDs []string + DeleteStorageData bool + DeleteBucketObjects []string + UpdateAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod + Manual bool +} + +type cleanupAuditModel struct { + Findings []cleanupFindingModel + PublicFindings []GitStorageCleanupFinding + Summary GitStorageCleanupAuditSummary + ExpectedPathCount int + IncludesRepoManifest bool + PathPrefix string +} + +type chainAuditModel struct { + Findings []GitStorageChainFinding + Summary GitStorageChainAuditSummary + PathPrefix string +} + +type cachedProjectJoinState struct { + expiresAt time.Time + recordsByChecksum map[string][]projectRecordState + usageByObjectID map[string]gintegrationsyfon.FileUsage +} + +func BuildGitRepoInventory(ref string, gitSubpath string, repo *gogit.Repository, hash plumbing.Hash) ([]RepoInventoryFile, error) { + index, err := buildRepoAnalyticsIndex(ref, repo, hash) + if err != nil { + return nil, err + } + return filterRepoInventoryFiles(index, gitSubpath) +} + +func (service *StorageAnalyticsService) BuildStorageSummary(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitStorageSummaryResponse, error) { + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + if err != nil { + return nil, err + } + directory, err := repoDirectoryAggregate(index, gitSubpath) + if err != nil { + return nil, err + } + summaryAgg := summarizeSubtree(gitSubpath, inventory, recordsByChecksum, usageByObjectID, directory.DirectChildCount) + return &GitStorageSummaryResponse{ + Path: summaryAgg.path, + FileCount: summaryAgg.fileCount, + RecordCount: summaryAgg.recordCount, + DirectChildCount: directory.DirectChildCount, + TotalBytes: summaryAgg.totalBytes, + DownloadCount: summaryAgg.downloadCount, + LastDownloadTime: formatOptionalTime(summaryAgg.lastDownload), + LatestUpdateTime: formatOptionalTime(summaryAgg.latestUpdate), + DuplicatePathCount: summaryAgg.duplicateCount, + }, nil +} + +func (service *StorageAnalyticsService) BuildStorageChildren(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string) (*GitStorageChildrenResponse, error) { + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + if err != nil { + return nil, err + } + directory, err := repoDirectoryAggregate(index, gitSubpath) + if err != nil { + return nil, err + } + aggregates := aggregateImmediateChildren(gitSubpath, inventory, recordsByChecksum, usageByObjectID, cloneDirectoryChildren(directory.Children)) + sortStorageAggregates(aggregates, sortBy, sortOrder) + if limit > 0 && len(aggregates) > limit { + aggregates = aggregates[:limit] + } + items := make([]GitStorageChildResponseItem, 0, len(aggregates)) + for _, agg := range aggregates { + items = append(items, GitStorageChildResponseItem{ + Name: agg.name, + Path: agg.path, + Type: agg.rowType, + FileCount: agg.fileCount, + RecordCount: agg.recordCount, + TotalBytes: agg.totalBytes, + DownloadCount: agg.downloadCount, + LastDownloadTime: formatOptionalTime(agg.lastDownload), + LatestUpdateTime: formatOptionalTime(agg.latestUpdate), + }) + } + return &GitStorageChildrenResponse{Items: items}, nil +} + +func (service *StorageAnalyticsService) BuildProjectDiffAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitProjectDiffAuditResponse, error) { + _, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + if err != nil { + return nil, err + } + allProjectRecords, err := service.listProjectRecordStates(ctx, authorizationHeader, organization, project, usageByObjectID) + if err != nil { + return nil, err + } + model := buildProjectDiffAuditModel(gitSubpath, inventory, recordsByChecksum, allProjectRecords) + return &GitProjectDiffAuditResponse{ + Findings: model.Findings, + Summary: model.Summary, + PathPrefix: model.PathPrefix, + }, nil +} + +func (service *StorageAnalyticsService) BuildStorageCleanupAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, selectedRepoPaths []string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, checkStorage bool) (*GitStorageCleanupAuditResponse, *cleanupAuditModel, error) { + baseInputs, err := service.loadStorageAuditBaseInputs(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + if err != nil { + return nil, nil, err + } + recordSet, err := service.loadScopedProjectRecords(ctx, authorizationHeader, organization, project, baseInputs) + if err != nil { + return nil, nil, err + } + storageView, err := service.loadStorageAuditStorageView(ctx, authorizationHeader, organization, project, recordSet, checkStorage, checkStorage) + if err != nil { + return nil, nil, err + } + model := buildCleanupAuditModel(gitSubpath, baseInputs.inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL, selectedRepoPaths, checkStorage) + return &GitStorageCleanupAuditResponse{ + Findings: model.PublicFindings, + Summary: model.Summary, + ExpectedPathCount: model.ExpectedPathCount, + IncludesRepoManifest: model.IncludesRepoManifest, + PathPrefix: model.PathPrefix, + }, model, nil +} + +func (service *StorageAnalyticsService) BuildStorageChainAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitStorageChainAuditResponse, error) { + inventory, err := service.loadStorageChainInventory(ctx, ref, gitSubpath, mirrorPath, repo, hash) + if err != nil { + return nil, err + } + recordSet, err := service.loadProjectAuditRecordSet(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + storageView, err := service.loadStorageChainView(ctx, authorizationHeader, organization, project, recordSet) + if err != nil { + return nil, err + } + model := buildStorageChainAuditModel(gitSubpath, inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL) + model.Summary.BucketInventoryAvailable = storageView.bucketInventoryAvailable + model.Summary.BucketInventoryError = storageView.bucketInventoryError + return &GitStorageChainAuditResponse{ + Findings: append([]GitStorageChainFinding(nil), model.Findings...), + Groups: summarizeChainIssueGroups(model.Findings), + Summary: model.Summary, + PathPrefix: model.PathPrefix, + }, nil +} + +func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, selectedRepoPaths []string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, checkStorage bool, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool, dryRun bool) (*GitStorageCleanupApplyResponse, error) { + _, model, err := service.BuildStorageCleanupAudit(ctx, authorizationHeader, organization, project, ref, gitSubpath, selectedRepoPaths, mirrorPath, repo, hash, checkStorage) + if err != nil { + return nil, err + } + toDelete := make([]string, 0) + toDeleteWithStorage := make([]string, 0) + toDeleteBucketObjects := make([]string, 0) + toUpdate := make(map[string][]gintegrationsyfon.ProjectAccessMethod) + repoDeletePaths := make([]string, 0) + deletedBucketObjectURLs := make([]string, 0) + updatedRecordIDs := make([]string, 0) + manualPaths := make([]string, 0) + skippedPaths := make([]string, 0) + for _, finding := range model.Findings { + switch finding.Public.Kind { + case "repo_orphan_live_object", "repo_orphan_stale_record": + if deleteRepoOrphans { + toDelete = append(toDelete, finding.DeleteObjectIDs...) + if finding.DeleteStorageData { + toDeleteWithStorage = append(toDeleteWithStorage, finding.DeleteObjectIDs...) + } + repoDeletePaths = append(repoDeletePaths, finding.Public.NormalizedPath) + } else { + skippedPaths = append(skippedPaths, finding.Public.NormalizedPath) + } + case "stale_duplicate_record": + if deleteStaleDuplicates { + toDelete = append(toDelete, finding.DeleteObjectIDs...) + } else { + skippedPaths = append(skippedPaths, finding.Public.NormalizedPath) + } + case "broken_bucket_mapping": + if repairBrokenBucketMappings { + toDelete = append(toDelete, finding.DeleteObjectIDs...) + for objectID, methods := range finding.UpdateAccessMethods { + if trimmed := strings.TrimSpace(objectID); trimmed != "" { + toUpdate[trimmed] = append([]gintegrationsyfon.ProjectAccessMethod(nil), methods...) + updatedRecordIDs = append(updatedRecordIDs, trimmed) + } + } + } else { + skippedPaths = append(skippedPaths, finding.Public.NormalizedPath) + } + case "bucket_only_object": + if deleteBucketOnlyObjects { + toDeleteBucketObjects = append(toDeleteBucketObjects, finding.DeleteBucketObjects...) + } else { + skippedPaths = append(skippedPaths, finding.Public.NormalizedPath) + } + default: + manualPaths = append(manualPaths, finding.Public.NormalizedPath) + } + } + toDelete = uniqueStrings(toDelete) + toDeleteWithStorage = uniqueStrings(toDeleteWithStorage) + toDeleteMetadataOnly := differenceStrings(toDelete, toDeleteWithStorage) + toDeleteBucketObjects = uniqueStrings(toDeleteBucketObjects) + repoDeletePaths = uniqueStrings(repoDeletePaths) + deletedBucketObjectURLs = uniqueStrings(deletedBucketObjectURLs) + updatedRecordIDs = uniqueStrings(updatedRecordIDs) + manualPaths = uniqueStrings(manualPaths) + skippedPaths = uniqueStrings(skippedPaths) + purgeResults := make([]GitStorageCleanupPurgeResult, 0, len(toDelete)+len(toDeleteBucketObjects)) + if dryRun { + for _, objectID := range toDelete { + purgeResults = append(purgeResults, GitStorageCleanupPurgeResult{ + ObjectID: objectID, + Success: nil, + Status: "dry_run", + }) + } + return &GitStorageCleanupApplyResponse{ + DeletedRecordIDs: toDelete, + DeletedBucketObjectURLs: toDeleteBucketObjects, + UpdatedRecordIDs: updatedRecordIDs, + PurgeResults: purgeResults, + RepoDeletePaths: repoDeletePaths, + ManualPaths: manualPaths, + SkippedPaths: skippedPaths, + DryRun: true, + }, nil + } + if len(toUpdate) > 0 { + if err := service.storage.BulkUpdateAccessMethods(ctx, authorizationHeader, toUpdate); err != nil { + return nil, fmt.Errorf("update syfon access methods: %w", err) + } + } + if len(toDeleteMetadataOnly) > 0 { + if err := service.storage.BulkDeleteObjects(ctx, authorizationHeader, toDeleteMetadataOnly, false); err != nil { + return nil, fmt.Errorf("delete syfon objects: %w", err) + } + } + if len(toDeleteWithStorage) > 0 { + if err := service.storage.BulkDeleteObjects(ctx, authorizationHeader, toDeleteWithStorage, true); err != nil { + return nil, fmt.Errorf("delete syfon objects: %w", err) + } + } + if len(toDeleteBucketObjects) > 0 { + results, err := service.storage.DeleteProjectBucketObjects(ctx, authorizationHeader, organization, project, toDeleteBucketObjects) + if err != nil { + return nil, fmt.Errorf("delete syfon project bucket objects: %w", err) + } + for _, result := range results { + if strings.EqualFold(strings.TrimSpace(result.Status), "deleted") { + deletedBucketObjectURLs = append(deletedBucketObjectURLs, result.ObjectURL) + continue + } + purgeResults = append(purgeResults, GitStorageCleanupPurgeResult{ + ObjectID: result.ObjectURL, + Success: boolPtr(false), + Status: strings.TrimSpace(result.Status), + Error: strings.TrimSpace(result.Error), + }) + } + deletedBucketObjectURLs = uniqueStrings(deletedBucketObjectURLs) + } + if len(toUpdate) > 0 || len(toDelete) > 0 { + service.evictProjectJoinCache(organization, project) + } + for _, objectID := range toDelete { + success := true + purgeResults = append(purgeResults, GitStorageCleanupPurgeResult{ + ObjectID: objectID, + Success: &success, + Status: "deleted", + }) + } + return &GitStorageCleanupApplyResponse{ + DeletedRecordIDs: toDelete, + DeletedBucketObjectURLs: deletedBucketObjectURLs, + UpdatedRecordIDs: updatedRecordIDs, + PurgeResults: purgeResults, + RepoDeletePaths: repoDeletePaths, + ManualPaths: manualPaths, + SkippedPaths: skippedPaths, + DryRun: false, + }, nil +} + +func (service *StorageAnalyticsService) loadJoinState(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, []RepoInventoryFile, map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { + index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) + if err != nil { + return nil, nil, nil, nil, err + } + inventory, err := filterRepoInventoryFiles(index, gitSubpath) + if err != nil { + return nil, nil, nil, nil, err + } + recordsByChecksum, usageByObjectID, err := service.loadProjectJoinCache(ctx, authorizationHeader, organization, project, hash, index.sidecar.Files) + if err != nil { + return nil, nil, nil, nil, err + } + return index, inventory, recordsByChecksum, usageByObjectID, nil +} + +func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context, authorizationHeader string, organization string, project string, hash plumbing.Hash, inventory []RepoInventoryFile) (map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { + cacheKey := service.projectJoinCacheKey(organization, project, hash.String()) + service.projectJoinMu.RLock() + cached, ok := service.projectJoinCache[cacheKey] + service.projectJoinMu.RUnlock() + if ok && time.Now().Before(cached.expiresAt) { + return cached.recordsByChecksum, cached.usageByObjectID, nil + } + checksums := make([]string, 0, len(inventory)) + for _, item := range inventory { + checksums = append(checksums, item.Checksum) + } + recordsByChecksumRaw, err := service.storage.BulkGetProjectRecordsByChecksum(ctx, authorizationHeader, organization, project, checksums) + if err != nil { + return nil, nil, fmt.Errorf("lookup syfon project records by checksum: %w", err) + } + usageByObjectID, err := service.storage.ListProjectFileUsage(ctx, authorizationHeader, organization, project, cleanupInactiveDays) + if err != nil { + return nil, nil, fmt.Errorf("list syfon project file usage: %w", err) + } + recordsByChecksum := make(map[string][]projectRecordState, len(recordsByChecksumRaw)) + for _, records := range recordsByChecksumRaw { + for _, record := range records { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + record.Checksum = normalizedChecksum + recordsByChecksum[normalizedChecksum] = append(recordsByChecksum[normalizedChecksum], projectRecordState{ + ProjectRecord: record, + Usage: usageByObjectID[record.ObjectID], + }) + } + } + service.projectJoinMu.Lock() + service.projectJoinCache[cacheKey] = cachedProjectJoinState{ + expiresAt: time.Now().Add(projectJoinCacheTTL), + recordsByChecksum: recordsByChecksum, + usageByObjectID: usageByObjectID, + } + service.projectJoinMu.Unlock() + return recordsByChecksum, usageByObjectID, nil +} + +func (service *StorageAnalyticsService) projectJoinCacheKey(organization string, project string, commitHash string) string { + return strings.TrimSpace(organization) + "/" + strings.TrimSpace(project) + "::" + strings.TrimSpace(commitHash) +} + +func (service *StorageAnalyticsService) evictProjectJoinCache(organization string, project string) { + prefix := strings.TrimSpace(organization) + "/" + strings.TrimSpace(project) + "::" + service.projectJoinMu.Lock() + defer service.projectJoinMu.Unlock() + for key := range service.projectJoinCache { + if strings.HasPrefix(key, prefix) { + delete(service.projectJoinCache, key) + } + } +} + +func (service *StorageAnalyticsService) listProjectRecordStates(ctx context.Context, authorizationHeader string, organization string, project string, usageByObjectID map[string]gintegrationsyfon.FileUsage) (map[string][]projectRecordState, error) { + projectRecords, err := service.storage.ListProjectRecords(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("list syfon project records: %w", err) + } + out := make(map[string][]projectRecordState) + for _, record := range projectRecords { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + record.Checksum = normalizedChecksum + out[normalizedChecksum] = append(out[normalizedChecksum], projectRecordState{ + ProjectRecord: record, + Usage: usageByObjectID[record.ObjectID], + }) + } + return out, nil +} + +func (service *StorageAnalyticsService) loadProjectStorageScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + buckets, err := service.storage.ListBuckets(ctx, authorizationHeader) + if err != nil { + return nil, fmt.Errorf("list syfon buckets: %w", err) + } + bucketNames := make([]string, 0, len(buckets)) + for bucket := range buckets { + bucketNames = append(bucketNames, bucket) + } + sort.Strings(bucketNames) + scopes := make([]domain.StorageBucketScope, 0) + for _, bucket := range bucketNames { + items, err := service.storage.ListBucketScopes(ctx, authorizationHeader, bucket) + if err != nil { + return nil, fmt.Errorf("list syfon bucket scopes for %s: %w", bucket, err) + } + for _, scope := range items { + if !strings.EqualFold(strings.TrimSpace(scope.Organization), organization) { + continue + } + scopeProject := strings.TrimSpace(scope.ProjectID) + if scopeProject != "" && !strings.EqualFold(scopeProject, project) { + continue + } + scopes = append(scopes, scope) + } + } + sort.SliceStable(scopes, func(i, j int) bool { + iProject := strings.TrimSpace(scopes[i].ProjectID) + jProject := strings.TrimSpace(scopes[j].ProjectID) + if iProject == "" && jProject != "" { + return true + } + if iProject != "" && jProject == "" { + return false + } + if scopes[i].Bucket != scopes[j].Bucket { + return scopes[i].Bucket < scopes[j].Bucket + } + return scopes[i].Path < scopes[j].Path + }) + return scopes, nil +} + +func applyScopedStorageMappings(recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, scopes []domain.StorageBucketScope) (map[string][]projectRecordState, map[string][]projectRecordState) { + attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + clone.CanonicalAccessURLs = canonicalizeRecordAccessURLs(record.AccessURLs, scopes) + states = append(states, clone) + } + out[checksum] = states + } + return out + } + return attach(recordsByChecksum), attach(allProjectRecords) +} + +func (service *StorageAnalyticsService) attachStorageProbes(ctx context.Context, authorizationHeader string, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState) (map[string][]projectRecordState, map[string][]projectRecordState, error) { + items := make([]gintegrationsyfon.BulkStorageProbeItem, 0) + itemKeys := map[string]string{} + recordProbeKeysByObjectID := map[string][]string{} + + for _, group := range allProjectRecords { + for _, record := range group { + for _, accessURL := range probeAccessURLsForRecord(record) { + normalizedURL := strings.TrimSpace(accessURL) + if normalizedURL == "" { + continue + } + key := storageProbeRequestKey(normalizedURL, record.Size, record.Checksum) + recordProbeKeysByObjectID[record.ObjectID] = append(recordProbeKeysByObjectID[record.ObjectID], key) + if _, ok := itemKeys[key]; ok { + continue + } + itemKeys[key] = key + expectedSize := record.Size + items = append(items, gintegrationsyfon.BulkStorageProbeItem{ + ID: key, + ObjectURL: normalizedURL, + ExpectedSizeBytes: &expectedSize, + ExpectedSHA256: strings.TrimSpace(record.Checksum), + }) + } + } + } + + resultsByKey := map[string]gintegrationsyfon.BulkStorageProbeResult{} + if len(items) > 0 { + results, err := service.storage.BulkProbeStorageObjects(ctx, authorizationHeader, items) + if err != nil { + return nil, nil, fmt.Errorf("probe syfon storage objects: %w", err) + } + for _, result := range results { + resultsByKey[strings.TrimSpace(result.ID)] = result + } + } + + attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + keys := uniqueStrings(recordProbeKeysByObjectID[record.ObjectID]) + probes := make([]gintegrationsyfon.BulkStorageProbeResult, 0, len(keys)) + for _, key := range keys { + if result, ok := resultsByKey[key]; ok { + probes = append(probes, result) + } + } + clone.AccessProbes = probes + states = append(states, clone) + } + out[checksum] = states + } + return out + } + + return attach(recordsByChecksum), attach(allProjectRecords), nil +} + +func summarizeSubtree(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, usageByObjectID map[string]gintegrationsyfon.FileUsage, directChildCount int) storageAggregate { + agg := storageAggregate{ + path: normalizeRepoSubpath(gitSubpath), + rowType: "directory", + } + for _, item := range inventory { + agg.fileCount++ + agg.totalBytes += item.Size + matches := recordsByChecksum[normalizeAnalyticsChecksum(item.Checksum)] + agg.recordCount += len(matches) + if len(matches) > 1 { + agg.duplicateCount++ + } + for _, record := range matches { + applyUsage(&agg, record) + } + } + _ = usageByObjectID + _ = directChildCount + return agg +} + +func aggregateImmediateChildren(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, usageByObjectID map[string]gintegrationsyfon.FileUsage, aggregates []storageAggregate) []storageAggregate { + root := normalizeRepoSubpath(gitSubpath) + if aggregates == nil { + aggregates = make([]storageAggregate, 0) + } + aggregateLookup := make(map[string]*storageAggregate, len(aggregates)) + for index := range aggregates { + aggregateLookup[aggregates[index].path] = &aggregates[index] + } + for _, item := range inventory { + childName, childPath, childType := immediateChild(root, item.RepoPath) + if childPath == "" { + continue + } + wasPrecomputed := true + agg := aggregateLookup[childPath] + if agg == nil { + wasPrecomputed = false + agg = &storageAggregate{ + name: childName, + path: childPath, + rowType: childType, + } + aggregates = append(aggregates, *agg) + agg = &aggregates[len(aggregates)-1] + aggregateLookup[childPath] = agg + } + if !wasPrecomputed { + agg.fileCount++ + agg.totalBytes += item.Size + } + matches := recordsByChecksum[normalizeAnalyticsChecksum(item.Checksum)] + agg.recordCount += len(matches) + if len(matches) > 1 { + agg.duplicateCount++ + } + for _, record := range matches { + applyUsage(agg, record) + } + } + _ = usageByObjectID + return aggregates +} + +func buildProjectDiffAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState) *projectDiffAuditModel { + findings := make([]GitProjectDiffFinding, 0) + countsByKind := map[string]int{ + "duplicate_syfon_paths": 0, + "syfon_missing_in_repo": 0, + "repo_missing_in_syfon": 0, + "unknown": 0, + } + matchedPathCount := 0 + scannedRecordCount := 0 + repoChecksums := make(map[string]struct{}, len(inventory)) + for _, item := range inventory { + normalizedChecksum := normalizeAnalyticsChecksum(item.Checksum) + repoChecksums[normalizedChecksum] = struct{}{} + matches := recordsByChecksum[normalizedChecksum] + scannedRecordCount += len(matches) + if len(matches) > 0 { + matchedPathCount++ + } + if len(matches) == 0 { + evidence := buildFindingEvidence(item.Checksum, []string{item.RepoPath}, nil, "not_checked") + findings = append(findings, GitProjectDiffFinding{ + Kind: "repo_missing_in_syfon", + NormalizedPath: item.RepoPath, + Checksum: item.Checksum, + SourcePaths: []string{item.RepoPath}, + ObjectIDs: []string{}, + RecordCount: 0, + SizeBytes: item.Size, + RecommendedAction: "No Syfon record matched this Git-tracked checksum. Bucket presence is not part of this check; review ingest or metadata mapping for this path.", + Evidence: evidence, + }) + countsByKind["repo_missing_in_syfon"]++ + continue + } + if len(matches) > 1 { + evidence := buildFindingEvidence(item.Checksum, []string{item.RepoPath}, matches, "not_checked") + findings = append(findings, GitProjectDiffFinding{ + Kind: "duplicate_syfon_paths", + NormalizedPath: item.RepoPath, + Checksum: item.Checksum, + SourcePaths: recordSourcePaths(matches), + ObjectIDs: recordObjectIDs(matches), + RecordCount: len(matches), + SizeBytes: aggregateMatchedSize(matches, item.Size), + DownloadCount: aggregateMatchedDownloads(matches), + LastDownload: formatOptionalTime(latestMatchedDownload(matches)), + RecommendedAction: "Review duplicate Syfon records before deleting anything.", + Evidence: evidence, + }) + countsByKind["duplicate_syfon_paths"]++ + } + } + seenOrphanChecksums := map[string]struct{}{} + for checksum, matches := range allProjectRecords { + if len(matches) == 0 { + continue + } + if _, ok := repoChecksums[checksum]; ok { + continue + } + if _, ok := seenOrphanChecksums[checksum]; ok { + continue + } + seenOrphanChecksums[checksum] = struct{}{} + sourcePaths := recordSourcePaths(matches) + evidence := buildFindingEvidence(checksum, nil, matches, "not_checked") + findings = append(findings, GitProjectDiffFinding{ + Kind: "syfon_missing_in_repo", + NormalizedPath: orphanDisplayPath(checksum, sourcePaths), + Checksum: checksum, + SourcePaths: sourcePaths, + ObjectIDs: recordObjectIDs(matches), + RecordCount: len(matches), + SizeBytes: aggregateMatchedSize(matches, 0), + DownloadCount: aggregateMatchedDownloads(matches), + LastDownload: formatOptionalTime(latestMatchedDownload(matches)), + RecommendedAction: "Prepare delete to verify storage before removing Syfon-only records.", + Evidence: evidence, + }) + countsByKind["syfon_missing_in_repo"]++ + } + return &projectDiffAuditModel{ + Findings: findings, + Summary: GitProjectDiffSummary{ + CountsByKind: countsByKind, + TotalFindings: len(findings), + IndexedPathCount: matchedPathCount, + ExpectedPathCount: len(inventory), + MatchedPathCount: matchedPathCount, + IncludesRepoManifest: true, + ScannedRecordCount: scannedRecordCount, + }, + PathPrefix: normalizeRepoSubpath(gitSubpath), + } +} + +func buildCleanupAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, selectedRepoPaths []string, checkStorage bool) *cleanupAuditModel { + allowed := make(map[string]struct{}, len(selectedRepoPaths)) + for _, path := range selectedRepoPaths { + if normalized := normalizeRepoSubpath(path); normalized != "" { + allowed[normalized] = struct{}{} + } + } + includePath := func(path string) bool { + if len(allowed) == 0 { + return true + } + _, ok := allowed[normalizeRepoSubpath(path)] + return ok + } + findings := make([]cleanupFindingModel, 0) + countsByKind := map[string]int{ + "bucket_only_object": 0, + "stale_duplicate_record": 0, + "live_duplicate_conflict": 0, + "broken_access_url_error": 0, + "broken_bucket_mapping": 0, + "repo_orphan_live_object": 0, + "repo_orphan_stale_record": 0, + "storage_object_missing": 0, + "storage_validation_mismatch": 0, + "storage_probe_error": 0, + "unknown": 0, + } + repoChecksums := make(map[string]struct{}, len(inventory)) + referencedBucketURLs := map[string]struct{}{} + for _, item := range inventory { + normalizedChecksum := normalizeAnalyticsChecksum(item.Checksum) + repoChecksums[normalizedChecksum] = struct{}{} + matches := recordsByChecksum[normalizedChecksum] + if !includePath(item.RepoPath) || len(matches) == 0 { + continue + } + if len(matches) > 1 { + sortedMatches := append([]projectRecordState(nil), matches...) + sort.SliceStable(sortedMatches, func(i, j int) bool { + return compareRecordState(sortedMatches[i], sortedMatches[j]) > 0 + }) + if compareRecordState(sortedMatches[0], sortedMatches[1]) == 0 { + public := buildCleanupFinding("live_duplicate_conflict", item.RepoPath, sortedMatches, false, "record", "Manual review required for live duplicate records.") + findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + countsByKind["live_duplicate_conflict"]++ + } else { + candidates := make([]projectRecordState, 0, len(sortedMatches)-1) + deleteIDs := make([]string, 0, len(sortedMatches)-1) + for _, record := range sortedMatches[1:] { + candidates = append(candidates, record) + deleteIDs = append(deleteIDs, record.ObjectID) + } + public := buildCleanupFinding("stale_duplicate_record", item.RepoPath, sortedMatches, true, "record", "Delete stale duplicate records") + findings = append(findings, cleanupFindingModel{Public: public, DeleteObjectIDs: deleteIDs}) + countsByKind["stale_duplicate_record"]++ + } + continue + } + if checkStorage { + for _, bucketURL := range matchedBucketObjectURLs(matches[0], bucketObjectsByURL) { + referencedBucketURLs[bucketURL] = struct{}{} + } + switch storageFindingKind := classifyStorageFinding(matches[0], bucketObjectsByURL); storageFindingKind { + case storageFindingBrokenAccessURL: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Manual review required for broken access URLs") + findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + countsByKind[string(storageFindingKind)]++ + case storageFindingBrokenBucketMap: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Fix or remove the Syfon access URL because no bucket mapping is configured for it") + repairDeleteIDs, repairUpdates := brokenBucketMappingRepairPlan(matches) + findings = append(findings, cleanupFindingModel{ + Public: public, + DeleteObjectIDs: repairDeleteIDs, + UpdateAccessMethods: repairUpdates, + Manual: len(repairDeleteIDs) == 0 && len(repairUpdates) == 0, + }) + countsByKind[string(storageFindingKind)]++ + case storageFindingObjectMissing: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Storage object is missing for this Syfon access URL") + findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + countsByKind[string(storageFindingKind)]++ + case storageFindingValidationMismatch: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Storage metadata does not match the Syfon record") + findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + countsByKind[string(storageFindingKind)]++ + case storageFindingProbeError: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Manual review required for storage probe errors") + findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + countsByKind[string(storageFindingKind)]++ + } + } + } + for checksum, matches := range allProjectRecords { + if len(matches) == 0 { + continue + } + if _, ok := repoChecksums[checksum]; ok { + continue + } + displayPath := orphanDisplayPath(checksum, recordSourcePaths(matches)) + if !includePath(displayPath) { + continue + } + kind := "repo_orphan_stale_record" + if checkStorage && recordsReferenceBucketObject(matches, bucketObjectsByURL) { + kind = "repo_orphan_live_object" + } else if recordsContainLiveUsage(matches) { + kind = "repo_orphan_live_object" + } + if checkStorage { + for _, match := range matches { + for _, bucketURL := range matchedBucketObjectURLs(match, bucketObjectsByURL) { + referencedBucketURLs[bucketURL] = struct{}{} + } + } + } + public := buildCleanupFinding(kind, displayPath, matches, true, "record", repoOrphanAction(kind)) + findings = append(findings, cleanupFindingModel{ + Public: public, + DeleteObjectIDs: recordObjectIDs(matches), + DeleteStorageData: kind == "repo_orphan_live_object", + }) + countsByKind[kind]++ + } + if checkStorage { + bucketURLs := make([]string, 0, len(bucketObjectsByURL)) + for objectURL := range bucketObjectsByURL { + bucketURLs = append(bucketURLs, objectURL) + } + sort.Strings(bucketURLs) + for _, objectURL := range bucketURLs { + if _, ok := referencedBucketURLs[objectURL]; ok { + continue + } + if !includePath(objectURL) { + continue + } + public := buildBucketOnlyFinding(bucketObjectsByURL[objectURL]) + findings = append(findings, cleanupFindingModel{ + Public: public, + DeleteBucketObjects: []string{objectURL}, + }) + countsByKind["bucket_only_object"]++ + } + } + sort.Slice(findings, func(i, j int) bool { + return findings[i].Public.NormalizedPath < findings[j].Public.NormalizedPath + }) + publicFindings := make([]GitStorageCleanupFinding, 0, len(findings)) + repoDeleteCandidateCount := 0 + manualFindingCount := 0 + repoOrphanCount := 0 + staleDuplicateCount := 0 + for _, finding := range findings { + publicFindings = append(publicFindings, finding.Public) + if finding.Public.RepoDeleteCandidate { + repoDeleteCandidateCount++ + } + if finding.Manual { + manualFindingCount++ + } + if finding.Public.Kind == "repo_orphan_live_object" || finding.Public.Kind == "repo_orphan_stale_record" { + repoOrphanCount++ + } + if finding.Public.Kind == "stale_duplicate_record" { + staleDuplicateCount++ + } + } + return &cleanupAuditModel{ + Findings: findings, + PublicFindings: publicFindings, + Summary: GitStorageCleanupAuditSummary{ + CountsByKind: countsByKind, + TotalFindings: len(publicFindings), + ManualFindingCount: manualFindingCount, + RepoDeleteCandidateCount: repoDeleteCandidateCount, + StaleDuplicateCount: staleDuplicateCount, + RepoOrphanCount: repoOrphanCount, + }, + ExpectedPathCount: len(inventory), + IncludesRepoManifest: true, + PathPrefix: normalizeRepoSubpath(gitSubpath), + } +} + +func bucketObjectHasCompleteChain(matches []projectRecordState, repoPathsByChecksum map[string][]string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + for _, match := range matches { + if classifyStorageFinding(match, bucketObjectsByURL) != storageFindingNone { + continue + } + if len(uniqueStrings(repoPathsByChecksum[normalizeAnalyticsChecksum(match.Checksum)])) > 0 { + return true + } + } + return false +} + +func buildCleanupFinding(kind string, normalizedPath string, matches []projectRecordState, repoDeleteCandidate bool, cleanupScope string, action string) GitStorageCleanupFinding { + records := make([]GitStorageCleanupRecordAudit, 0, len(matches)) + var latestUpdate *time.Time + var latestDownload *time.Time + var totalBytes int64 + var totalDownloads int64 + checksum := "" + for _, match := range matches { + if checksum == "" { + checksum = strings.TrimSpace(match.Checksum) + } + records = append(records, GitStorageCleanupRecordAudit{ + ObjectID: match.ObjectID, + Checksum: strings.TrimSpace(match.Checksum), + NormalizedPath: normalizedPath, + CleanupScope: cleanupScope, + AccessProbes: accessProbesForRecord(match), + Status: accessStatusForRecord(match), + Error: accessErrorForRecord(match), + SizeBytes: match.Size, + LastUpdated: formatOptionalTime(match.UpdatedAt), + DownloadCount: match.Usage.DownloadCount, + LastDownload: formatOptionalTime(match.Usage.LastDownloadTime), + }) + totalBytes += match.Size + totalDownloads += match.Usage.DownloadCount + latestUpdate = laterTime(latestUpdate, match.UpdatedAt) + latestDownload = laterTime(latestDownload, match.Usage.LastDownloadTime) + } + bucketEvaluation := "not_checked" + if cleanupScope == "access_url" { + bucketEvaluation = "probed" + } + return GitStorageCleanupFinding{ + Kind: kind, + NormalizedPath: normalizedPath, + Checksum: checksum, + ObjectIDs: recordObjectIDs(matches), + Records: records, + RecommendedAction: action, + RepoDeleteCandidate: repoDeleteCandidate, + CleanupScope: cleanupScope, + SizeBytes: totalBytes, + LastUpdated: formatOptionalTime(latestUpdate), + DownloadCount: totalDownloads, + LastDownload: formatOptionalTime(latestDownload), + Evidence: buildFindingEvidence(checksum, nil, matches, bucketEvaluation), + } +} + +func repoOrphanAction(kind string) string { + if kind == "repo_orphan_live_object" { + return "Delete Syfon record and purge storage object" + } + return "Delete stale Syfon record" +} + +func compareRecordState(left projectRecordState, right projectRecordState) int { + if left.Usage.DownloadCount != right.Usage.DownloadCount { + if left.Usage.DownloadCount > right.Usage.DownloadCount { + return 1 + } + return -1 + } + if compareOptionalTime(left.Usage.LastDownloadTime, right.Usage.LastDownloadTime) != 0 { + return compareOptionalTime(left.Usage.LastDownloadTime, right.Usage.LastDownloadTime) + } + return compareOptionalTime(left.UpdatedAt, right.UpdatedAt) +} + +func compareOptionalTime(left *time.Time, right *time.Time) int { + switch { + case left == nil && right == nil: + return 0 + case left == nil: + return -1 + case right == nil: + return 1 + case left.After(*right): + return 1 + case left.Before(*right): + return -1 + default: + return 0 + } +} + +func recordHasBrokenAccess(record projectRecordState) bool { + if len(record.AccessURLs) == 0 { + return true + } + for _, accessURL := range record.AccessURLs { + if strings.TrimSpace(accessURL) != "" { + return false + } + } + return true +} + +func recordsContainLiveUsage(matches []projectRecordState) bool { + for _, match := range matches { + if match.Usage.DownloadCount > 0 || match.Usage.UploadCount > 0 || match.Usage.LastAccessTime != nil || match.Usage.LastDownloadTime != nil || match.Usage.LastUploadTime != nil { + return true + } + } + return false +} + +func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) storageFindingKind { + if recordHasBrokenAccess(record) { + return storageFindingBrokenAccessURL + } + if kind := classifyRawAccessURLFindings(record); kind != storageFindingNone { + return kind + } + bucketMatches := matchedBucketObjectURLs(record, bucketObjectsByURL) + if inventoryHasValidationMismatch(record, bucketMatches, bucketObjectsByURL) { + return storageFindingValidationMismatch + } + if len(record.AccessProbes) == 0 { + if len(bucketMatches) > 0 { + return storageFindingNone + } + if len(bucketObjectsByURL) > 0 && hasCanonicalBucketURL(record) { + return storageFindingProbeError + } + return storageFindingNone + } + hasPresent := false + hasMissing := false + hasBrokenBucketMapping := false + hasProbeError := false + hasBucketMatch := len(bucketMatches) > 0 + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Status) == "present" { + hasPresent = true + } + switch strings.TrimSpace(probe.ErrorKind) { + case "credential_missing": + hasBrokenBucketMapping = true + } + switch strings.TrimSpace(probe.Status) { + case "not_found": + hasMissing = true + case "forbidden", "unsupported", "invalid", "error": + hasProbeError = true + } + if strings.TrimSpace(probe.ValidationStatus) == "mismatched" { + return storageFindingValidationMismatch + } + } + if hasBucketMatch || hasPresent { + return storageFindingNone + } + if hasBrokenBucketMapping { + return storageFindingBrokenBucketMap + } + if hasMissing { + return storageFindingObjectMissing + } + if hasProbeError { + return storageFindingProbeError + } + return storageFindingNone +} + +func inventoryHasValidationMismatch(record projectRecordState, bucketObjectURLs []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + if len(bucketObjectURLs) == 0 { + return false + } + checksum := normalizeAnalyticsChecksum(record.Checksum) + for _, objectURL := range bucketObjectURLs { + item, ok := bucketObjectsByURL[objectURL] + if !ok { + continue + } + if record.Size > 0 && item.SizeBytes > 0 && item.SizeBytes != record.Size { + return true + } + if checksum != "" { + metaSHA := normalizeAnalyticsChecksum(item.MetaSHA256) + if metaSHA != "" && metaSHA != checksum { + return true + } + } + } + return false +} + +func accessProbesForRecord(record projectRecordState) []GitStorageCleanupAccessProbe { + if len(record.AccessProbes) > 0 { + probes := make([]GitStorageCleanupAccessProbe, 0, len(record.AccessProbes)) + for _, probe := range record.AccessProbes { + exists := probe.Exists + probes = append(probes, GitStorageCleanupAccessProbe{ + URL: probe.ObjectURL, + Provider: probe.Provider, + Bucket: probe.Bucket, + Key: probe.Key, + Path: probe.Path, + Exists: &exists, + Status: probe.Status, + Error: probe.Error, + ErrorKind: probe.ErrorKind, + SizeBytes: probe.SizeBytes, + MetaSHA256: probe.MetaSHA256, + ETag: probe.ETag, + LastModified: probe.LastModified, + ValidationStatus: probe.ValidationStatus, + SizeMatch: probe.SizeMatch, + SHA256Match: probe.SHA256Match, + ValidationMismatches: append([]string(nil), probe.ValidationMismatches...), + }) + } + return probes + } + if len(record.AccessURLs) == 0 { + return []GitStorageCleanupAccessProbe{{ + URL: "", + Status: "missing", + Error: "no access URLs present", + ErrorKind: "missing_access_url", + }} + } + probes := make([]GitStorageCleanupAccessProbe, 0, len(record.AccessURLs)) + for _, accessURL := range record.AccessURLs { + if strings.TrimSpace(accessURL) == "" { + probes = append(probes, GitStorageCleanupAccessProbe{ + URL: accessURL, + Status: "missing", + Error: "blank access URL", + ErrorKind: "missing_access_url", + }) + continue + } + probes = append(probes, GitStorageCleanupAccessProbe{ + URL: accessURL, + Status: "present", + }) + } + return probes +} + +func accessStatusForRecord(record projectRecordState) string { + switch classifyStorageFinding(record, nil) { + case storageFindingBrokenAccessURL, storageFindingObjectMissing: + return "missing" + case storageFindingBrokenBucketMap, storageFindingProbeError: + return "error" + case storageFindingValidationMismatch: + return "mismatched" + } + return "present" +} + +func accessErrorForRecord(record projectRecordState) string { + switch classifyStorageFinding(record, nil) { + case storageFindingBrokenAccessURL: + return "no usable access URL present" + case storageFindingBrokenBucketMap: + return "no Syfon bucket mapping is configured for this access URL" + case storageFindingObjectMissing: + return "storage object not found" + case storageFindingValidationMismatch: + return "storage metadata does not match the Syfon record" + case storageFindingProbeError: + return "storage probe failed" + } + return "" +} + +func buildBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStorageCleanupFinding { + objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL) + return GitStorageCleanupFinding{ + Kind: "bucket_only_object", + NormalizedPath: objectURL, + ObjectIDs: []string{}, + Records: []GitStorageCleanupRecordAudit{}, + RecommendedAction: "Review and delete bucket object that has no Syfon record", + RepoDeleteCandidate: false, + CleanupScope: "bucket_object", + SizeBytes: item.SizeBytes, + LastUpdated: strings.TrimSpace(item.LastModified), + Evidence: &GitAuditEvidence{ + AccessURLs: []string{objectURL}, + Buckets: uniqueStrings([]string{item.Bucket}), + Keys: uniqueStrings([]string{item.Key}), + ProbeStatuses: []string{"enumerated"}, + BucketEvaluation: "enumerated", + }, + } +} + +func buildChainBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStorageChainFinding { + objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL) + return GitStorageChainFinding{ + Kind: "bucket_only_object", + NormalizedPath: objectURL, + ObjectIDs: []string{}, + AccessURLs: []string{objectURL}, + BucketObjectURL: objectURL, + ResolvedBucket: strings.TrimSpace(item.Bucket), + ResolvedKey: strings.TrimSpace(item.Key), + ProbeStatus: "enumerated", + RecordCount: 0, + SizeBytes: item.SizeBytes, + RecommendedAction: "Bucket object exists, but no Syfon record matched it.", + Evidence: &GitAuditEvidence{ + AccessURLs: []string{objectURL}, + BucketObjectURLs: []string{objectURL}, + Buckets: uniqueStrings([]string{item.Bucket}), + Keys: uniqueStrings([]string{item.Key}), + ProbeStatuses: []string{"enumerated"}, + BucketEvaluation: "enumerated", + }, + } +} + +func buildChainRecordFindings(kind string, record projectRecordState, gitPaths []string, bucketObjectURLs []string, action string) []GitStorageChainFinding { + paths := uniqueStrings(gitPaths) + if len(paths) == 0 { + displayPath := orphanDisplayPath(strings.TrimSpace(record.Checksum), append(bucketObjectURLs, record.AccessURLs...)) + if displayPath == "" { + displayPath = strings.TrimSpace(record.Checksum) + } + paths = []string{displayPath} + } + objectIDs := uniqueStrings([]string{record.ObjectID}) + accessURLs := uniqueStrings(record.AccessURLs) + evidence := buildFindingEvidence(strings.TrimSpace(record.Checksum), gitPaths, []projectRecordState{record}, "enumerated_and_probed") + if evidence != nil { + evidence.BucketObjectURLs = uniqueStrings(append(evidence.BucketObjectURLs, bucketObjectURLs...)) + } + primaryProbe := selectChainProbe(record, bucketObjectURLs) + findings := make([]GitStorageChainFinding, 0, len(paths)) + for _, path := range paths { + findings = append(findings, GitStorageChainFinding{ + Kind: kind, + NormalizedPath: path, + Checksum: strings.TrimSpace(record.Checksum), + SourcePaths: uniqueStrings(gitPaths), + ObjectIDs: objectIDs, + AccessURLs: accessURLs, + BucketObjectURL: primaryProbe.bucketObjectURL, + ResolvedBucket: primaryProbe.probe.Bucket, + ResolvedKey: primaryProbe.probe.Key, + ProbeStatus: primaryProbe.probe.Status, + ErrorKind: primaryProbe.probe.ErrorKind, + Error: chainFindingError(kind, record, primaryProbe.probe), + RecordCount: 1, + SizeBytes: record.Size, + RecommendedAction: action, + Evidence: evidence, + }) + } + return findings +} + +type chainProbeSelection struct { + bucketObjectURL string + probe GitStorageCleanupAccessProbe +} + +func selectChainProbe(record projectRecordState, bucketObjectURLs []string) chainProbeSelection { + probes := accessProbesForRecord(record) + if len(bucketObjectURLs) > 0 { + targets := map[string]struct{}{} + for _, bucketURL := range bucketObjectURLs { + targets[strings.TrimSpace(bucketURL)] = struct{}{} + } + for _, probe := range probes { + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.URL) + if _, ok := targets[objectURL]; ok { + return chainProbeSelection{bucketObjectURL: objectURL, probe: probe} + } + } + } + for _, probe := range probes { + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.URL) + if probe.Status != "" || probe.ErrorKind != "" || objectURL != "" || probe.URL != "" { + return chainProbeSelection{bucketObjectURL: objectURL, probe: probe} + } + } + return chainProbeSelection{} +} + +func chainFindingError(kind string, record projectRecordState, probe GitStorageCleanupAccessProbe) string { + if trimmed := strings.TrimSpace(probe.Error); trimmed != "" { + return trimmed + } + switch kind { + case "syfon_broken_bucket_mapping": + return "no configured Syfon bucket mapping matched this access URL" + case "syfon_missing_bucket_object", "syfon_git_no_bucket": + return "mapped bucket object was not found" + case "git_syfon_metadata_mismatch": + return "bucket metadata did not match the Syfon record" + case "probe_error": + return accessErrorForRecord(record) + default: + return "" + } +} + +func recordsReferenceBucketObject(matches []projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + for _, match := range matches { + if len(matchedBucketObjectURLs(match, bucketObjectsByURL)) > 0 { + return true + } + } + return false +} + +func matchedBucketObjectURLs(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []string { + if len(bucketObjectsByURL) == 0 { + return nil + } + matches := make([]string, 0) + for _, objectURL := range recordBucketURLs(record) { + if _, ok := bucketObjectsByURL[objectURL]; ok { + matches = append(matches, objectURL) + } + } + return uniqueStrings(matches) +} + +func hasCanonicalBucketURL(record projectRecordState) bool { + return len(recordBucketURLs(record)) > 0 +} + +func recordBucketURLs(record projectRecordState) []string { + out := make([]string, 0) + for _, probe := range record.AccessProbes { + if objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL); objectURL != "" { + out = append(out, objectURL) + } + } + for _, accessURL := range accessURLsForStorage(record) { + if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { + out = append(out, objectURL) + } + } + return uniqueStrings(out) +} + +func accessURLsForStorage(record projectRecordState) []string { + if len(record.CanonicalAccessURLs) > 0 { + return record.CanonicalAccessURLs + } + return record.AccessURLs +} + +func probeAccessURLsForRecord(record projectRecordState) []string { + return uniqueStrings(append(rawAccessURLsForRecord(record), accessURLsForStorage(record)...)) +} + +func rawAccessURLsForRecord(record projectRecordState) []string { + out := make([]string, 0, len(record.AccessURLs)) + for _, accessURL := range record.AccessURLs { + if trimmed := strings.TrimSpace(accessURL); trimmed != "" { + out = append(out, trimmed) + } + } + return uniqueStrings(out) +} + +func brokenBucketMappingRepairPlan(records []projectRecordState) ([]string, map[string][]gintegrationsyfon.ProjectAccessMethod) { + deleteIDs := make([]string, 0) + updateAccessMethods := make(map[string][]gintegrationsyfon.ProjectAccessMethod) + for _, record := range records { + remainingMethods, shouldDelete, ok := repairBrokenBucketMappingRecord(record) + if !ok { + continue + } + if shouldDelete { + deleteIDs = append(deleteIDs, record.ObjectID) + continue + } + updateAccessMethods[record.ObjectID] = remainingMethods + } + return uniqueStrings(deleteIDs), updateAccessMethods +} + +func repairBrokenBucketMappingRecord(record projectRecordState) ([]gintegrationsyfon.ProjectAccessMethod, bool, bool) { + if len(record.AccessMethods) == 0 || len(record.AccessProbes) == 0 { + return nil, false, false + } + probesByURL := make(map[string][]gintegrationsyfon.BulkStorageProbeResult, len(record.AccessProbes)) + for _, probe := range record.AccessProbes { + probesByURL[strings.TrimSpace(probe.ObjectURL)] = append(probesByURL[strings.TrimSpace(probe.ObjectURL)], probe) + } + remaining := make([]gintegrationsyfon.ProjectAccessMethod, 0, len(record.AccessMethods)) + removedAny := false + for _, method := range record.AccessMethods { + if !accessURLHasBrokenBucketMapping(method.URL, probesByURL) { + remaining = append(remaining, method) + continue + } + removedAny = true + } + if !removedAny { + return nil, false, false + } + if len(remaining) == 0 { + return nil, true, true + } + return remaining, false, true +} + +func accessURLHasBrokenBucketMapping(accessURL string, probesByURL map[string][]gintegrationsyfon.BulkStorageProbeResult) bool { + probes := probesByURL[strings.TrimSpace(accessURL)] + if len(probes) == 0 { + return false + } + hasBrokenBucketMapping := false + for _, probe := range probes { + if strings.TrimSpace(probe.Status) == "present" { + return false + } + if strings.TrimSpace(probe.ErrorKind) == "credential_missing" { + hasBrokenBucketMapping = true + } + } + return hasBrokenBucketMapping +} + +func classifyRawAccessURLFindings(record projectRecordState) storageFindingKind { + if len(record.AccessProbes) == 0 { + return storageFindingNone + } + probesByURL := make(map[string][]gintegrationsyfon.BulkStorageProbeResult, len(record.AccessProbes)) + for _, probe := range record.AccessProbes { + probesByURL[strings.TrimSpace(probe.ObjectURL)] = append(probesByURL[strings.TrimSpace(probe.ObjectURL)], probe) + } + for _, accessURL := range rawAccessURLsForRecord(record) { + probes := probesByURL[accessURL] + if len(probes) == 0 { + continue + } + hasPresent := false + hasMissing := false + hasBrokenBucketMapping := false + hasProbeError := false + for _, probe := range probes { + if strings.TrimSpace(probe.ValidationStatus) == "mismatched" { + return storageFindingValidationMismatch + } + if strings.TrimSpace(probe.Status) == "present" { + hasPresent = true + } + switch strings.TrimSpace(probe.ErrorKind) { + case "credential_missing": + hasBrokenBucketMapping = true + } + switch strings.TrimSpace(probe.Status) { + case "not_found": + hasMissing = true + case "forbidden", "unsupported", "invalid", "error": + hasProbeError = true + } + } + if hasPresent { + continue + } + switch { + case hasBrokenBucketMapping: + return storageFindingBrokenBucketMap + case hasMissing: + return storageFindingObjectMissing + case hasProbeError: + return storageFindingProbeError + } + } + return storageFindingNone +} + +func canonicalizeRecordAccessURLs(accessURLs []string, scopes []domain.StorageBucketScope) []string { + out := make([]string, 0, len(accessURLs)) + for _, accessURL := range accessURLs { + if objectURL := canonicalizeScopedStorageURL(accessURL, scopes); objectURL != "" { + out = append(out, objectURL) + continue + } + if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { + out = append(out, objectURL) + } + } + return uniqueStrings(out) +} + +func canonicalizeScopedStorageURL(accessURL string, scopes []domain.StorageBucketScope) string { + if len(scopes) == 0 { + return "" + } + _, key, ok := parseStorageURL(accessURL) + if !ok { + return "" + } + targetBucket := "" + prefixes := make([]string, 0, len(scopes)) + for _, scope := range scopes { + bucket, prefix, ok := parseStorageScopePath(scope.Path) + if !ok { + continue + } + if strings.TrimSpace(bucket) != "" { + targetBucket = strings.TrimSpace(bucket) + } + if strings.TrimSpace(prefix) != "" { + prefixes = append(prefixes, strings.Trim(strings.TrimSpace(prefix), "/")) + } + } + if targetBucket == "" { + return "" + } + normalizedKey := normalizeScopedStorageKeyForGecko(key, prefixes) + if normalizedKey == "" { + return "" + } + return canonicalStorageURL(targetBucket, normalizedKey, "") +} + +func parseStorageScopePath(raw string) (string, string, bool) { + return parseStorageURL(raw) +} + +func parseStorageURL(raw string) (string, string, bool) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "", "", false + } + if !strings.HasPrefix(strings.ToLower(trimmed), "s3://") { + return "", "", false + } + rest := strings.TrimPrefix(trimmed, "s3://") + rest = strings.TrimLeft(rest, "/") + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 { + return "", "", false + } + bucket := strings.TrimSpace(parts[0]) + key := strings.Trim(strings.TrimSpace(parts[1]), "/") + if bucket == "" || key == "" { + return "", "", false + } + return bucket, key, true +} + +func normalizeScopedStorageKeyForGecko(key string, prefixes []string) string { + key = strings.Trim(strings.TrimSpace(key), "/") + normalizedPrefixes := normalizedScopePrefixesForGecko(prefixes) + remainder := key + for _, prefix := range normalizedPrefixes { + remainder = trimLeadingStoragePrefixForGecko(remainder, prefix) + } + composedPrefix := strings.Join(normalizedPrefixes, "/") + switch { + case composedPrefix == "": + return remainder + case remainder == "": + return composedPrefix + default: + return path.Join(composedPrefix, remainder) + } +} + +func normalizedScopePrefixesForGecko(prefixes []string) []string { + out := make([]string, 0, len(prefixes)) + for _, prefix := range prefixes { + prefix = strings.Trim(strings.TrimSpace(prefix), "/") + if prefix == "" { + continue + } + if len(out) == 0 { + out = append(out, prefix) + continue + } + last := out[len(out)-1] + switch { + case prefix == last: + continue + case strings.HasPrefix(prefix, last+"/"): + out[len(out)-1] = prefix + case strings.HasPrefix(last, prefix+"/"): + continue + default: + out = append(out, prefix) + } + } + return out +} + +func trimLeadingStoragePrefixForGecko(key, prefix string) string { + key = strings.Trim(strings.TrimSpace(key), "/") + prefix = strings.Trim(strings.TrimSpace(prefix), "/") + if key == "" || prefix == "" { + return key + } + if key == prefix { + return "" + } + if strings.HasPrefix(key, prefix+"/") { + return strings.TrimPrefix(key, prefix+"/") + } + return key +} + +func canonicalStorageURL(bucket string, key string, objectURL string) string { + cleanBucket := strings.TrimSpace(bucket) + cleanKey := strings.Trim(strings.TrimSpace(key), "/") + if cleanBucket != "" && cleanKey != "" { + return "s3://" + cleanBucket + "/" + cleanKey + } + trimmed := strings.TrimSpace(objectURL) + if trimmed == "" { + return "" + } + if !strings.HasPrefix(strings.ToLower(trimmed), "s3://") { + return "" + } + rest := strings.TrimPrefix(trimmed, "s3://") + if strings.HasPrefix(rest, "/") { + rest = strings.TrimLeft(rest, "/") + } + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || strings.TrimSpace(parts[1]) == "" { + return "" + } + return "s3://" + strings.TrimSpace(parts[0]) + "/" + strings.Trim(strings.TrimSpace(parts[1]), "/") +} + +func normalizeAnalyticsChecksum(value string) string { + trimmed := strings.TrimSpace(value) + trimmed = strings.TrimPrefix(trimmed, "sha256:") + trimmed = strings.TrimPrefix(trimmed, "SHA256:") + if trimmed == "" { + return "" + } + return strings.ToLower(trimmed) +} + +func applyUsage(agg *storageAggregate, record projectRecordState) { + agg.downloadCount += record.Usage.DownloadCount + agg.lastDownload = laterTime(agg.lastDownload, record.Usage.LastDownloadTime) + agg.latestUpdate = laterTime(agg.latestUpdate, record.UpdatedAt) +} + +func aggregateMatchedSize(matches []projectRecordState, fallback int64) int64 { + var total int64 + for _, match := range matches { + if match.Size > 0 { + total += match.Size + } + } + if total == 0 { + return fallback + } + return total +} + +func aggregateMatchedDownloads(matches []projectRecordState) int64 { + var total int64 + for _, match := range matches { + total += match.Usage.DownloadCount + } + return total +} + +func latestMatchedDownload(matches []projectRecordState) *time.Time { + var latest *time.Time + for _, match := range matches { + latest = laterTime(latest, match.Usage.LastDownloadTime) + } + return latest +} + +func laterTime(current *time.Time, candidate *time.Time) *time.Time { + if candidate == nil { + return current + } + if current == nil || candidate.After(*current) { + copyTime := candidate.UTC() + return ©Time + } + return current +} + +func immediateChild(root string, repoPath string) (string, string, string) { + normalizedPath := normalizeRepoSubpath(repoPath) + if root != "" { + prefix := root + "/" + if !strings.HasPrefix(normalizedPath, prefix) { + return "", "", "" + } + normalizedPath = strings.TrimPrefix(normalizedPath, prefix) + } + if normalizedPath == "" { + return "", "", "" + } + parts := strings.Split(normalizedPath, "/") + childName := parts[0] + if len(parts) == 1 { + if root == "" { + return childName, childName, "file" + } + return childName, root + "/" + childName, "file" + } + if root == "" { + return childName, childName, "directory" + } + return childName, root + "/" + childName, "directory" +} + +func normalizeRepoSubpath(raw string) string { + return strings.Trim(strings.TrimSpace(raw), "/") +} + +func recordObjectIDs(matches []projectRecordState) []string { + out := make([]string, 0, len(matches)) + for _, match := range matches { + if trimmed := strings.TrimSpace(match.ObjectID); trimmed != "" { + out = append(out, trimmed) + } + } + return uniqueStrings(out) +} + +func buildFindingEvidence(checksum string, sourcePaths []string, matches []projectRecordState, bucketEvaluation string) *GitAuditEvidence { + evidence := &GitAuditEvidence{ + Checksum: strings.TrimSpace(checksum), + SourcePaths: uniqueStrings(sourcePaths), + ObjectIDs: []string{}, + AccessURLs: []string{}, + BucketObjectURLs: []string{}, + Buckets: []string{}, + Keys: []string{}, + ProbeStatuses: []string{}, + ValidationStates: []string{}, + ErrorKinds: []string{}, + Errors: []string{}, + BucketEvaluation: strings.TrimSpace(bucketEvaluation), + } + for _, match := range matches { + if objectID := strings.TrimSpace(match.ObjectID); objectID != "" { + evidence.ObjectIDs = append(evidence.ObjectIDs, objectID) + } + evidence.AccessURLs = append(evidence.AccessURLs, match.AccessURLs...) + if len(match.AccessProbes) == 0 { + continue + } + for _, probe := range match.AccessProbes { + if objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL); objectURL != "" { + evidence.BucketObjectURLs = append(evidence.BucketObjectURLs, objectURL) + } + if bucket := strings.TrimSpace(probe.Bucket); bucket != "" { + evidence.Buckets = append(evidence.Buckets, bucket) + } + if key := strings.TrimSpace(probe.Key); key != "" { + evidence.Keys = append(evidence.Keys, key) + } + if status := strings.TrimSpace(probe.Status); status != "" { + evidence.ProbeStatuses = append(evidence.ProbeStatuses, status) + } + if validation := strings.TrimSpace(probe.ValidationStatus); validation != "" { + evidence.ValidationStates = append(evidence.ValidationStates, validation) + } + if kind := strings.TrimSpace(probe.ErrorKind); kind != "" { + evidence.ErrorKinds = append(evidence.ErrorKinds, kind) + } + if err := strings.TrimSpace(probe.Error); err != "" { + evidence.Errors = append(evidence.Errors, err) + } + } + } + evidence.ObjectIDs = uniqueStrings(evidence.ObjectIDs) + evidence.AccessURLs = uniqueStrings(evidence.AccessURLs) + evidence.BucketObjectURLs = uniqueStrings(evidence.BucketObjectURLs) + evidence.Buckets = uniqueStrings(evidence.Buckets) + evidence.Keys = uniqueStrings(evidence.Keys) + evidence.ProbeStatuses = uniqueStrings(evidence.ProbeStatuses) + evidence.ValidationStates = uniqueStrings(evidence.ValidationStates) + evidence.ErrorKinds = uniqueStrings(evidence.ErrorKinds) + evidence.Errors = uniqueStrings(evidence.Errors) + if evidence.Checksum == "" && + len(evidence.SourcePaths) == 0 && + len(evidence.ObjectIDs) == 0 && + len(evidence.AccessURLs) == 0 && + len(evidence.BucketObjectURLs) == 0 && + len(evidence.Buckets) == 0 && + len(evidence.Keys) == 0 && + len(evidence.ProbeStatuses) == 0 && + len(evidence.ValidationStates) == 0 && + len(evidence.ErrorKinds) == 0 && + len(evidence.Errors) == 0 && + evidence.BucketEvaluation == "" { + return nil + } + return evidence +} + +func flattenRecordStates(recordsByChecksum map[string][]projectRecordState) []projectRecordState { + out := make([]projectRecordState, 0) + seen := map[string]struct{}{} + for _, group := range recordsByChecksum { + for _, record := range group { + objectID := strings.TrimSpace(record.ObjectID) + if objectID == "" { + continue + } + if _, ok := seen[objectID]; ok { + continue + } + seen[objectID] = struct{}{} + out = append(out, record) + } + } + sort.Slice(out, func(i, j int) bool { + return out[i].ObjectID < out[j].ObjectID + }) + return out +} + +func chainPathCount(gitPaths []string) int { + if len(gitPaths) == 0 { + return 1 + } + return len(gitPaths) +} + +func uniqueStrings(values []string) []string { + if len(values) == 0 { + return []string{} + } + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func storageProbeRequestKey(objectURL string, size int64, checksum string) string { + return strings.TrimSpace(objectURL) + "|" + fmt.Sprintf("%d", size) + "|" + strings.TrimSpace(checksum) +} + +func sortStorageAggregates(items []storageAggregate, sortBy string, sortOrder string) { + desc := !strings.EqualFold(strings.TrimSpace(sortOrder), "asc") + switch strings.ToLower(strings.TrimSpace(sortBy)) { + case "name": + sort.Slice(items, func(i, j int) bool { + left := strings.ToLower(items[i].name) + right := strings.ToLower(items[j].name) + if desc { + return left > right + } + return left < right + }) + default: + sort.Slice(items, func(i, j int) bool { + if items[i].totalBytes != items[j].totalBytes { + if desc { + return items[i].totalBytes > items[j].totalBytes + } + return items[i].totalBytes < items[j].totalBytes + } + if items[i].rowType != items[j].rowType { + return items[i].rowType == "directory" + } + return strings.ToLower(items[i].name) < strings.ToLower(items[j].name) + }) + } +} + +func orphanDisplayPath(checksum string, sourcePaths []string) string { + if len(sourcePaths) > 0 && strings.TrimSpace(sourcePaths[0]) != "" { + return strings.TrimSpace(sourcePaths[0]) + } + return "sha256/" + strings.TrimSpace(checksum) +} + +func recordSourcePaths(matches []projectRecordState) []string { + seen := make(map[string]struct{}) + out := make([]string, 0) + for _, match := range matches { + for _, accessURL := range match.AccessURLs { + normalized := strings.TrimSpace(accessURL) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + } + sort.Strings(out) + return out +} + +func formatOptionalTime(value *time.Time) string { + if value == nil { + return "" + } + return value.UTC().Format(time.RFC3339) +} + +func boolPtr(value bool) *bool { + return &value +} + +func differenceStrings(values []string, toRemove []string) []string { + if len(values) == 0 { + return []string{} + } + removeSet := make(map[string]struct{}, len(toRemove)) + for _, value := range toRemove { + removeSet[value] = struct{}{} + } + out := make([]string, 0, len(values)) + for _, value := range values { + if _, ok := removeSet[value]; ok { + continue + } + out = append(out, value) + } + return out +} diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go new file mode 100644 index 0000000..209cc20 --- /dev/null +++ b/internal/git/storage_analytics_pipeline.go @@ -0,0 +1,628 @@ +package git + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +type storageAuditBaseInputs struct { + index *repoAnalyticsIndex + inventory []RepoInventoryFile + recordsByChecksum map[string][]projectRecordState + usageByObjectID map[string]gintegrationsyfon.FileUsage +} + +type storageAuditRecordSet struct { + recordsByChecksum map[string][]projectRecordState + allProjectRecords map[string][]projectRecordState +} + +type storageAuditStorageView struct { + scopes []domain.StorageBucketScope + bucketObjects []gintegrationsyfon.ProjectBucketObject + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + recordsByChecksum map[string][]projectRecordState + allProjectRecords map[string][]projectRecordState + bucketInventoryAvailable bool + bucketInventoryError string +} + +type storageChainIndex struct { + inventory []RepoInventoryFile + allRecords []projectRecordState + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + repoPathsByChecksum map[string][]string + recordsByBucketURL map[string][]projectRecordState +} + +func (service *StorageAnalyticsService) loadStorageChainInventory(ctx context.Context, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) ([]RepoInventoryFile, error) { + index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) + if err != nil { + return nil, err + } + return filterRepoInventoryFiles(index, gitSubpath) +} + +type storageFindingKind string + +const ( + storageFindingNone storageFindingKind = "" + storageFindingBrokenAccessURL storageFindingKind = "broken_access_url_error" + storageFindingBrokenBucketMap storageFindingKind = "broken_bucket_mapping" + storageFindingObjectMissing storageFindingKind = "storage_object_missing" + storageFindingValidationMismatch storageFindingKind = "storage_validation_mismatch" + storageFindingProbeError storageFindingKind = "storage_probe_error" +) + +type chainAuditAccumulator struct { + findings []GitStorageChainFinding + summary GitStorageChainAuditSummary +} + +func newChainSummary(bucketObjectCount, syfonRecordCount, gitTrackedFileCount int) GitStorageChainAuditSummary { + return GitStorageChainAuditSummary{ + CountsByKind: map[string]int{ + "bucket_only_object": 0, + "bucket_syfon_no_git": 0, + "bucket_syfon_git_complete": 0, + "syfon_broken_bucket_mapping": 0, + "syfon_missing_bucket_object": 0, + "syfon_git_no_bucket": 0, + "git_only_no_syfon": 0, + "git_syfon_metadata_mismatch": 0, + "probe_error": 0, + }, + BucketObjectCount: bucketObjectCount, + SyfonRecordCount: syfonRecordCount, + GitTrackedFileCount: gitTrackedFileCount, + BucketInventoryAvailable: true, + } +} + +func (acc *chainAuditAccumulator) add(kind string, findings ...GitStorageChainFinding) { + acc.findings = append(acc.findings, findings...) + acc.summary.CountsByKind[kind] += len(findings) +} + +func (acc *chainAuditAccumulator) addCount(kind string, count int) { + acc.summary.CountsByKind[kind] += count +} + +func finalizeChainFindings(gitSubpath string, acc chainAuditAccumulator) *chainAuditModel { + sort.Slice(acc.findings, func(i, j int) bool { + if acc.findings[i].Kind == acc.findings[j].Kind { + return acc.findings[i].NormalizedPath < acc.findings[j].NormalizedPath + } + return acc.findings[i].Kind < acc.findings[j].Kind + }) + acc.summary.TotalFindings = len(acc.findings) + return &chainAuditModel{ + Findings: acc.findings, + Summary: acc.summary, + PathPrefix: normalizeRepoSubpath(gitSubpath), + } +} + +func summarizeChainIssueGroups(findings []GitStorageChainFinding) []GitStorageChainIssueGroup { + groups := make(map[string]*GitStorageChainIssueGroup) + groupPaths := make(map[string]map[string]struct{}) + groupObjects := make(map[string]map[string]struct{}) + for _, finding := range findings { + group := groups[finding.Kind] + if group == nil { + group = &GitStorageChainIssueGroup{Kind: finding.Kind} + groups[finding.Kind] = group + groupPaths[finding.Kind] = map[string]struct{}{} + groupObjects[finding.Kind] = map[string]struct{}{} + } + group.FindingCount++ + group.RecordCount += finding.RecordCount + group.TotalBytes += finding.SizeBytes + groupPaths[finding.Kind][finding.NormalizedPath] = struct{}{} + for _, objectID := range finding.ObjectIDs { + groupObjects[finding.Kind][objectID] = struct{}{} + } + } + kinds := make([]string, 0, len(groups)) + for kind := range groups { + kinds = append(kinds, kind) + } + sort.Strings(kinds) + out := make([]GitStorageChainIssueGroup, 0, len(kinds)) + for _, kind := range kinds { + group := *groups[kind] + group.PathCount = len(groupPaths[kind]) + group.ObjectCount = len(groupObjects[kind]) + out = append(out, group) + } + return out +} + +func (service *StorageAnalyticsService) loadStorageAuditBaseInputs(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*storageAuditBaseInputs, error) { + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + if err != nil { + return nil, err + } + return &storageAuditBaseInputs{ + index: index, + inventory: inventory, + recordsByChecksum: recordsByChecksum, + usageByObjectID: usageByObjectID, + }, nil +} + +func (service *StorageAnalyticsService) loadScopedProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string, base *storageAuditBaseInputs) (*storageAuditRecordSet, error) { + allProjectRecords, err := service.listProjectRecordStates(ctx, authorizationHeader, organization, project, base.usageByObjectID) + if err != nil { + return nil, err + } + return &storageAuditRecordSet{ + recordsByChecksum: base.recordsByChecksum, + allProjectRecords: allProjectRecords, + }, nil +} + +func (service *StorageAnalyticsService) loadProjectScopeMappings(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + return service.loadProjectStorageScopes(ctx, authorizationHeader, organization, project) +} + +func (service *StorageAnalyticsService) loadProjectChainScopeMappings(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + scopes, err := service.storage.ListProjectScopes(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("list syfon project scopes: %w", err) + } + sort.SliceStable(scopes, func(i, j int) bool { + iProject := strings.TrimSpace(scopes[i].ProjectID) + jProject := strings.TrimSpace(scopes[j].ProjectID) + if iProject == "" && jProject != "" { + return true + } + if iProject != "" && jProject == "" { + return false + } + if scopes[i].Bucket != scopes[j].Bucket { + return scopes[i].Bucket < scopes[j].Bucket + } + return scopes[i].Path < scopes[j].Path + }) + return scopes, nil +} + +func applyScopeCanonicalization(recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope) *storageAuditRecordSet { + if recordSet == nil { + return nil + } + recordsByChecksum, allProjectRecords := applyScopedStorageMappings(recordSet.recordsByChecksum, recordSet.allProjectRecords, scopes) + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: allProjectRecords, + } +} + +func (service *StorageAnalyticsService) loadProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string) (*storageAuditRecordSet, error) { + projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("list syfon project audit records: %w", err) + } + recordsByChecksum := make(map[string][]projectRecordState) + for _, record := range projectRecords { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + record.Checksum = normalizedChecksum + recordsByChecksum[normalizedChecksum] = append(recordsByChecksum[normalizedChecksum], projectRecordState{ + ProjectRecord: record, + }) + } + allProjectRecords := cloneRecordStateMap(recordsByChecksum) + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: allProjectRecords, + }, nil +} + +func (service *StorageAnalyticsService) loadProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { + bucketObjects, err := service.storage.ListProjectBucketObjects(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, nil, fmt.Errorf("list syfon project bucket objects: %w", err) + } + bucketObjectsByURL := make(map[string]gintegrationsyfon.ProjectBucketObject, len(bucketObjects)) + for _, item := range bucketObjects { + if objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL); objectURL != "" { + bucketObjectsByURL[objectURL] = item + } + } + return bucketObjects, bucketObjectsByURL, nil +} + +func synthesizeBucketInventoryFromProbes(allProjectRecords map[string][]projectRecordState) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { + if len(allProjectRecords) == 0 { + return []gintegrationsyfon.ProjectBucketObject{}, map[string]gintegrationsyfon.ProjectBucketObject{} + } + bucketObjectsByURL := make(map[string]gintegrationsyfon.ProjectBucketObject) + for _, group := range allProjectRecords { + for _, record := range group { + for _, probe := range record.AccessProbes { + if !strings.EqualFold(strings.TrimSpace(probe.Status), "present") { + continue + } + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL) + if objectURL == "" { + continue + } + if _, ok := bucketObjectsByURL[objectURL]; ok { + continue + } + bucketObjectsByURL[objectURL] = gintegrationsyfon.ProjectBucketObject{ + ObjectURL: objectURL, + Provider: strings.TrimSpace(probe.Provider), + Bucket: strings.TrimSpace(probe.Bucket), + Key: strings.TrimSpace(probe.Key), + Path: strings.TrimSpace(probe.Path), + SizeBytes: derefInt64(probe.SizeBytes), + MetaSHA256: strings.TrimSpace(probe.MetaSHA256), + ETag: strings.TrimSpace(probe.ETag), + LastModified: strings.TrimSpace(probe.LastModified), + } + } + } + } + bucketObjects := make([]gintegrationsyfon.ProjectBucketObject, 0, len(bucketObjectsByURL)) + for _, item := range bucketObjectsByURL { + bucketObjects = append(bucketObjects, item) + } + sort.Slice(bucketObjects, func(i, j int) bool { + return bucketObjects[i].ObjectURL < bucketObjects[j].ObjectURL + }) + return bucketObjects, bucketObjectsByURL +} + +func shouldDegradeBucketInventory(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(strings.TrimSpace(err.Error())) + return strings.Contains(message, "project-bucket") && + (strings.Contains(message, "status 403") || + strings.Contains(message, "status 409") || + strings.Contains(message, "permission denied") || + strings.Contains(message, "provider denied list access") || + strings.Contains(message, "bucket inventory request") || + strings.Contains(message, "bucket target may be missing or inaccessible")) +} + +func (service *StorageAnalyticsService) attachProjectStorageProbes(ctx context.Context, authorizationHeader string, recordSet *storageAuditRecordSet) (*storageAuditRecordSet, error) { + recordsByChecksum, allProjectRecords, err := service.attachStorageProbes(ctx, authorizationHeader, recordSet.recordsByChecksum, recordSet.allProjectRecords) + if err != nil { + return nil, err + } + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: allProjectRecords, + }, nil +} + +func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet) (*storageAuditStorageView, error) { + scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + recordSet = applyScopeCanonicalization(recordSet, scopes) + view := &storageAuditStorageView{ + scopes: scopes, + recordsByChecksum: recordSet.recordsByChecksum, + allProjectRecords: recordSet.allProjectRecords, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, + bucketInventoryAvailable: true, + } + bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project) + if err != nil { + if !shouldDegradeBucketInventory(err) { + return nil, err + } + view.bucketInventoryAvailable = false + view.bucketInventoryError = strings.TrimSpace(err.Error()) + probedRecordSet, probeErr := service.attachProjectStorageProbes(ctx, authorizationHeader, recordSet) + if probeErr != nil { + return nil, probeErr + } + view.recordsByChecksum = probedRecordSet.recordsByChecksum + view.allProjectRecords = probedRecordSet.allProjectRecords + view.bucketObjects, view.bucketObjectsByURL = synthesizeBucketInventoryFromProbes(probedRecordSet.allProjectRecords) + return view, nil + } + view.bucketObjects = bucketObjects + view.bucketObjectsByURL = bucketObjectsByURL + + probeCandidates := selectTargetedProbeRecordSet(recordSet, bucketObjectsByURL) + if probeCandidates != nil { + probedSubset, probeErr := service.attachProjectStorageProbes(ctx, authorizationHeader, probeCandidates) + if probeErr != nil { + return nil, probeErr + } + merged := mergeRecordSetProbes(recordSet, probedSubset) + view.recordsByChecksum = merged.recordsByChecksum + view.allProjectRecords = merged.allProjectRecords + } + return view, nil +} + +func (service *StorageAnalyticsService) loadStorageAuditStorageView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, includeBucketInventory bool, includeProbes bool) (*storageAuditStorageView, error) { + scopes, err := service.loadProjectScopeMappings(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + recordSet = applyScopeCanonicalization(recordSet, scopes) + if includeProbes { + recordSet, err = service.attachProjectStorageProbes(ctx, authorizationHeader, recordSet) + if err != nil { + return nil, err + } + } + view := &storageAuditStorageView{ + scopes: scopes, + recordsByChecksum: recordSet.recordsByChecksum, + allProjectRecords: recordSet.allProjectRecords, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, + bucketInventoryAvailable: includeBucketInventory, + } + if includeBucketInventory { + bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project) + if err != nil { + if !includeProbes || !shouldDegradeBucketInventory(err) { + return nil, err + } + view.bucketInventoryAvailable = false + view.bucketInventoryError = strings.TrimSpace(err.Error()) + bucketObjects, bucketObjectsByURL = synthesizeBucketInventoryFromProbes(recordSet.allProjectRecords) + } + view.bucketObjects = bucketObjects + view.bucketObjectsByURL = bucketObjectsByURL + } + return view, nil +} + +func cloneRecordStateMap(input map[string][]projectRecordState) map[string][]projectRecordState { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + clone.CanonicalAccessURLs = append([]string(nil), record.CanonicalAccessURLs...) + clone.AccessProbes = append([]gintegrationsyfon.BulkStorageProbeResult(nil), record.AccessProbes...) + states = append(states, clone) + } + out[checksum] = states + } + return out +} + +func selectTargetedProbeRecordSet(recordSet *storageAuditRecordSet, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) *storageAuditRecordSet { + if recordSet == nil { + return nil + } + selected := make(map[string][]projectRecordState) + for checksum, group := range recordSet.allProjectRecords { + for _, record := range group { + if !recordNeedsTargetedProbe(record, bucketObjectsByURL) { + continue + } + selected[checksum] = append(selected[checksum], record) + } + } + if len(selected) == 0 { + return nil + } + return &storageAuditRecordSet{ + recordsByChecksum: cloneRecordStateMap(selected), + allProjectRecords: cloneRecordStateMap(selected), + } +} + +func recordNeedsTargetedProbe(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + if len(matchedBucketObjectURLs(record, bucketObjectsByURL)) == 0 { + return true + } + if !sameStringSet(rawAccessURLsForRecord(record), accessURLsForStorage(record)) { + return true + } + checksum := normalizeAnalyticsChecksum(record.Checksum) + if checksum == "" { + return false + } + matches := matchedBucketObjectURLs(record, bucketObjectsByURL) + for _, objectURL := range matches { + item, ok := bucketObjectsByURL[objectURL] + if !ok { + continue + } + if normalizeAnalyticsChecksum(item.MetaSHA256) != "" { + return false + } + } + return true +} + +func sameStringSet(left []string, right []string) bool { + a := uniqueStrings(left) + b := uniqueStrings(right) + if len(a) != len(b) { + return false + } + set := make(map[string]struct{}, len(a)) + for _, value := range a { + set[strings.TrimSpace(value)] = struct{}{} + } + for _, value := range b { + if _, ok := set[strings.TrimSpace(value)]; !ok { + return false + } + } + return true +} + +func mergeRecordSetProbes(base *storageAuditRecordSet, probed *storageAuditRecordSet) *storageAuditRecordSet { + if base == nil { + return probed + } + if probed == nil { + return base + } + probesByObjectID := make(map[string][]gintegrationsyfon.BulkStorageProbeResult) + for _, group := range probed.allProjectRecords { + for _, record := range group { + probesByObjectID[strings.TrimSpace(record.ObjectID)] = append([]gintegrationsyfon.BulkStorageProbeResult(nil), record.AccessProbes...) + } + } + attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + if probes, ok := probesByObjectID[strings.TrimSpace(record.ObjectID)]; ok { + clone.AccessProbes = append([]gintegrationsyfon.BulkStorageProbeResult(nil), probes...) + } + states = append(states, clone) + } + out[checksum] = states + } + return out + } + return &storageAuditRecordSet{ + recordsByChecksum: attach(base.recordsByChecksum), + allProjectRecords: attach(base.allProjectRecords), + } +} + +func derefInt64(value *int64) int64 { + if value == nil { + return 0 + } + return *value +} + +func buildStorageChainIndex(inventory []RepoInventoryFile, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) storageChainIndex { + repoPathsByChecksum := make(map[string][]string, len(inventory)) + for _, item := range inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" { + continue + } + repoPathsByChecksum[checksum] = append(repoPathsByChecksum[checksum], item.RepoPath) + } + allRecords := flattenRecordStates(allProjectRecords) + recordsByBucketURL := make(map[string][]projectRecordState) + for _, record := range allRecords { + for _, bucketURL := range recordBucketURLs(record) { + recordsByBucketURL[bucketURL] = append(recordsByBucketURL[bucketURL], record) + } + } + return storageChainIndex{ + inventory: inventory, + allRecords: allRecords, + bucketObjectsByURL: bucketObjectsByURL, + repoPathsByChecksum: repoPathsByChecksum, + recordsByBucketURL: recordsByBucketURL, + } +} + +func buildStorageChainAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) *chainAuditModel { + index := buildStorageChainIndex(inventory, allProjectRecords, bucketObjectsByURL) + acc := chainAuditAccumulator{ + findings: make([]GitStorageChainFinding, 0), + summary: newChainSummary(len(bucketObjectsByURL), len(index.allRecords), len(inventory)), + } + buildBucketOriginChainFindings(index, &acc) + buildSyfonOriginChainFindings(index, &acc) + buildGitOriginChainFindings(index, recordsByChecksum, allProjectRecords, &acc) + return finalizeChainFindings(gitSubpath, acc) +} + +func buildBucketOriginChainFindings(index storageChainIndex, acc *chainAuditAccumulator) { + bucketURLs := make([]string, 0, len(index.bucketObjectsByURL)) + for bucketURL := range index.bucketObjectsByURL { + bucketURLs = append(bucketURLs, bucketURL) + } + sort.Strings(bucketURLs) + for _, bucketURL := range bucketURLs { + item := index.bucketObjectsByURL[bucketURL] + if bucketObjectHasCompleteChain(index.recordsByBucketURL[bucketURL], index.repoPathsByChecksum, index.bucketObjectsByURL) { + acc.addCount("bucket_syfon_git_complete", 1) + continue + } + acc.add("bucket_only_object", buildChainBucketOnlyFinding(item)) + } +} + +func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccumulator) { + for _, record := range index.allRecords { + gitPaths := uniqueStrings(index.repoPathsByChecksum[normalizeAnalyticsChecksum(record.Checksum)]) + bucketMatches := matchedBucketObjectURLs(record, index.bucketObjectsByURL) + switch classifyStorageFinding(record, index.bucketObjectsByURL) { + case storageFindingBrokenBucketMap: + findings := buildChainRecordFindings("syfon_broken_bucket_mapping", record, gitPaths, bucketMatches, "Syfon record exists, but its access URL does not resolve to a configured bucket mapping.") + acc.findings = append(acc.findings, findings...) + acc.addCount("syfon_broken_bucket_mapping", chainPathCount(gitPaths)) + case storageFindingObjectMissing: + if len(gitPaths) > 0 { + findings := buildChainRecordFindings("syfon_git_no_bucket", record, gitPaths, bucketMatches, "Git and Syfon matched, but the mapped bucket object does not exist.") + acc.findings = append(acc.findings, findings...) + acc.addCount("syfon_git_no_bucket", len(gitPaths)) + continue + } + acc.add("syfon_missing_bucket_object", buildChainRecordFindings("syfon_missing_bucket_object", record, nil, bucketMatches, "Syfon record points to a mapped bucket location, but the object does not exist.")...) + case storageFindingValidationMismatch: + if len(gitPaths) > 0 { + findings := buildChainRecordFindings("git_syfon_metadata_mismatch", record, gitPaths, bucketMatches, "Bucket object exists, but its metadata does not match what Syfon expects.") + acc.findings = append(acc.findings, findings...) + acc.addCount("git_syfon_metadata_mismatch", len(gitPaths)) + continue + } + acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) + case storageFindingBrokenAccessURL, storageFindingProbeError: + findings := buildChainRecordFindings("probe_error", record, gitPaths, bucketMatches, "Bucket verification failed before Gecko could classify this record cleanly.") + acc.findings = append(acc.findings, findings...) + acc.addCount("probe_error", chainPathCount(gitPaths)) + case storageFindingNone: + if len(gitPaths) == 0 && len(bucketMatches) > 0 { + acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) + } + } + } +} + +func buildGitOriginChainFindings(index storageChainIndex, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, acc *chainAuditAccumulator) { + for _, item := range index.inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if len(allProjectRecords[checksum]) > 0 || len(recordsByChecksum[checksum]) > 0 { + continue + } + acc.add("git_only_no_syfon", GitStorageChainFinding{ + Kind: "git_only_no_syfon", + NormalizedPath: item.RepoPath, + Checksum: checksum, + SourcePaths: []string{item.RepoPath}, + ObjectIDs: []string{}, + RecordCount: 0, + SizeBytes: item.Size, + RecommendedAction: "Git checksum has no matching Syfon record. Bucket presence is not claimed by this finding.", + Evidence: &GitAuditEvidence{ + Checksum: checksum, + SourcePaths: []string{item.RepoPath}, + ObjectIDs: []string{}, + BucketEvaluation: "not_evaluated", + }, + }) + } +} diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go new file mode 100644 index 0000000..c4c9d00 --- /dev/null +++ b/internal/git/storage_analytics_test.go @@ -0,0 +1,1334 @@ +package git + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +type fakeStorageAnalyticsBackend struct { + projectRecords []gintegrationsyfon.ProjectRecord + bulkRecords map[string][]gintegrationsyfon.ProjectRecord + buckets map[string]domain.StorageBucket + bucketScopes map[string][]domain.StorageBucketScope + projectScopes []domain.StorageBucketScope + usageByObject map[string]gintegrationsyfon.FileUsage + probeResults map[string]gintegrationsyfon.BulkStorageProbeResult + bucketObjects []gintegrationsyfon.ProjectBucketObject + listProjectBucketObjectsErr error + listBucketsCalls int + listBucketScopesCalls int + listProjectAuditRecordsCalls int + listProjectScopesCalls int + listProjectFileUsageCalls int + listProjectBucketObjectsCalls int + probeCalls int + probeItems []gintegrationsyfon.BulkStorageProbeItem + deletedIDs []string + updatedAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod + deletedBucketObjects []string +} + +func (fake *fakeStorageAnalyticsBackend) ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) { + fake.listBucketsCalls++ + out := make(map[string]domain.StorageBucket, len(fake.buckets)) + for bucket, metadata := range fake.buckets { + out[bucket] = metadata + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) { + fake.listBucketScopesCalls++ + return append([]domain.StorageBucketScope(nil), fake.bucketScopes[bucket]...), nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) { + return append([]gintegrationsyfon.ProjectRecord(nil), fake.projectRecords...), nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) { + fake.listProjectAuditRecordsCalls++ + return append([]gintegrationsyfon.ProjectRecord(nil), fake.projectRecords...), nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + fake.listProjectScopesCalls++ + if fake.projectScopes != nil { + return append([]domain.StorageBucketScope(nil), fake.projectScopes...), nil + } + out := make([]domain.StorageBucketScope, 0) + for _, scopes := range fake.bucketScopes { + out = append(out, scopes...) + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]gintegrationsyfon.ProjectRecord, error) { + if fake.bulkRecords != nil { + out := make(map[string][]gintegrationsyfon.ProjectRecord, len(fake.bulkRecords)) + for checksum, records := range fake.bulkRecords { + out[checksum] = append([]gintegrationsyfon.ProjectRecord(nil), records...) + } + return out, nil + } + allowed := make(map[string]struct{}, len(checksums)) + for _, checksum := range checksums { + allowed[strings.TrimSpace(checksum)] = struct{}{} + } + out := make(map[string][]gintegrationsyfon.ProjectRecord) + for _, record := range fake.projectRecords { + if _, ok := allowed[record.Checksum]; !ok { + continue + } + if record.Organization != organization || record.Project != project { + continue + } + out[record.Checksum] = append(out[record.Checksum], record) + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectFileUsage(ctx context.Context, authorizationHeader string, organization string, project string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) { + fake.listProjectFileUsageCalls++ + out := make(map[string]gintegrationsyfon.FileUsage, len(fake.usageByObject)) + for objectID, usage := range fake.usageByObject { + out[objectID] = usage + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectBucketObject, error) { + fake.listProjectBucketObjectsCalls++ + if fake.listProjectBucketObjectsErr != nil { + return nil, fake.listProjectBucketObjectsErr + } + return append([]gintegrationsyfon.ProjectBucketObject(nil), fake.bucketObjects...), nil +} + +func (fake *fakeStorageAnalyticsBackend) BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) { + fake.probeCalls++ + fake.probeItems = append(fake.probeItems, items...) + out := make([]gintegrationsyfon.BulkStorageProbeResult, 0, len(items)) + for _, item := range items { + if result, ok := fake.probeResults[item.ID]; ok { + out = append(out, result) + continue + } + exists := true + out = append(out, gintegrationsyfon.BulkStorageProbeResult{ + ID: item.ID, + ObjectURL: item.ObjectURL, + Exists: exists, + Status: "present", + ValidationStatus: "matched", + SizeBytes: item.ExpectedSizeBytes, + MetaSHA256: item.ExpectedSHA256, + }) + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error { + fake.deletedIDs = append(fake.deletedIDs, objectIDs...) + return nil +} + +func (fake *fakeStorageAnalyticsBackend) DeleteProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, objectURLs []string) ([]gintegrationsyfon.ProjectBucketDeleteResult, error) { + fake.deletedBucketObjects = append(fake.deletedBucketObjects, objectURLs...) + results := make([]gintegrationsyfon.ProjectBucketDeleteResult, 0, len(objectURLs)) + for _, objectURL := range objectURLs { + results = append(results, gintegrationsyfon.ProjectBucketDeleteResult{ + ObjectURL: objectURL, + Status: "deleted", + }) + } + return results, nil +} + +func (fake *fakeStorageAnalyticsBackend) BulkUpdateAccessMethods(ctx context.Context, authorizationHeader string, updates map[string][]gintegrationsyfon.ProjectAccessMethod) error { + if fake.updatedAccessMethods == nil { + fake.updatedAccessMethods = map[string][]gintegrationsyfon.ProjectAccessMethod{} + } + for objectID, methods := range updates { + fake.updatedAccessMethods[objectID] = append([]gintegrationsyfon.ProjectAccessMethod(nil), methods...) + } + return nil +} + +func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/c.txt": lfsPointer("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", 300), + "data/e.txt": lfsPointer("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", 50), + "data/nested/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "plain.txt": "not lfs\n", + }) + inventory, err := BuildGitRepoInventory(refName, "data", repo, hash) + if err != nil { + t.Fatalf("build repo inventory: %v", err) + } + if len(inventory) != 4 { + t.Fatalf("expected 4 lfs files under data, got %+v", inventory) + } + if inventory[0].RepoPath != "data/a.txt" || inventory[3].RepoPath != "data/nested/b.txt" { + t.Fatalf("unexpected inventory ordering: %+v", inventory) + } + + now := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC) + older := now.Add(-48 * time.Hour) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: inventory[0].Checksum, Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-d", Checksum: inventory[1].Checksum, Organization: "org", Project: "proj", Size: 300, UpdatedAt: &older}, + {ObjectID: "obj-b-old", Checksum: inventory[3].Checksum, Organization: "org", Project: "proj", Size: 200, UpdatedAt: &older, AccessURLs: []string{"s3://bucket/b-old"}}, + {ObjectID: "obj-b-new", Checksum: inventory[3].Checksum, Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/b-new"}}, + {ObjectID: "obj-orphan", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj", Size: 500, UpdatedAt: &older, AccessURLs: []string{"s3://bucket/orphan"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 5, LastDownloadTime: ptrTime(now)}, + "obj-b-old": {ObjectID: "obj-b-old", DownloadCount: 0}, + "obj-b-new": {ObjectID: "obj-b-new", DownloadCount: 10, LastDownloadTime: ptrTime(now)}, + "obj-orphan": {ObjectID: "obj-orphan", DownloadCount: 0}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{}, + } + service := NewStorageAnalyticsService(backend) + + summary, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build storage summary: %v", err) + } + if summary.FileCount != 4 || summary.RecordCount != 4 || summary.DirectChildCount != 4 || summary.DuplicatePathCount != 1 { + t.Fatalf("unexpected summary: %+v", summary) + } + if summary.TotalBytes != 650 || summary.DownloadCount != 15 { + t.Fatalf("unexpected summary bytes/downloads: %+v", summary) + } + + children, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 10, "bytes", "desc") + if err != nil { + t.Fatalf("build storage children: %v", err) + } + if len(children.Items) != 4 { + t.Fatalf("expected 4 child rows, got %+v", children.Items) + } + if children.Items[0].Path != "data/c.txt" || children.Items[1].Path != "data/nested" { + t.Fatalf("unexpected child ordering: %+v", children.Items) + } + + diff, err := service.BuildProjectDiffAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build project diff audit: %v", err) + } + if diff.Summary.TotalFindings != 3 { + t.Fatalf("expected 3 diff findings, got %+v", diff) + } + assertHasDiffFinding(t, diff.Findings, "duplicate_syfon_paths", "data/nested/b.txt") + missingFinding := assertHasDiffFinding(t, diff.Findings, "repo_missing_in_syfon", "data/e.txt") + if missingFinding.Checksum != "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" { + t.Fatalf("expected checksum on repo_missing_in_syfon, got %+v", missingFinding) + } + if missingFinding.Evidence == nil || missingFinding.Evidence.Checksum != missingFinding.Checksum || missingFinding.Evidence.BucketEvaluation != "not_checked" { + t.Fatalf("expected evidence on repo_missing_in_syfon, got %+v", missingFinding) + } + orphanFinding := assertHasDiffFinding(t, diff.Findings, "syfon_missing_in_repo", "s3://bucket/orphan") + if len(orphanFinding.SourcePaths) != 1 || orphanFinding.SourcePaths[0] != "s3://bucket/orphan" { + t.Fatalf("expected orphan source path, got %+v", orphanFinding) + } + if orphanFinding.Checksum != "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" { + t.Fatalf("expected checksum on syfon_missing_in_repo, got %+v", orphanFinding) + } + if orphanFinding.Evidence == nil || len(orphanFinding.Evidence.AccessURLs) != 1 || orphanFinding.Evidence.AccessURLs[0] != "s3://bucket/orphan" { + t.Fatalf("expected orphan access URL evidence, got %+v", orphanFinding) + } + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + if backend.probeCalls != 1 { + t.Fatalf("expected one bulk storage probe call, got %d", backend.probeCalls) + } + if cleanup.Summary.TotalFindings != 3 { + t.Fatalf("expected 3 cleanup findings, got %+v", cleanup) + } + assertHasCleanupFinding(t, cleanup.Findings, "stale_duplicate_record", "data/nested/b.txt") + assertHasCleanupFinding(t, cleanup.Findings, "broken_access_url_error", "data/c.txt") + assertHasCleanupFinding(t, cleanup.Findings, "repo_orphan_stale_record", "s3://bucket/orphan") + + applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true, true, true, false, false, true) + if err != nil { + t.Fatalf("apply cleanup dry run: %v", err) + } + if !applyResult.DryRun { + t.Fatalf("expected dry run apply result, got %+v", applyResult) + } + if len(applyResult.DeletedRecordIDs) != 2 || !contains(applyResult.DeletedRecordIDs, "obj-b-old") || !contains(applyResult.DeletedRecordIDs, "obj-orphan") { + t.Fatalf("unexpected dry run delete ids: %+v", applyResult) + } + if len(backend.deletedIDs) != 0 { + t.Fatalf("dry run should not delete objects, got %+v", backend.deletedIDs) + } +} + +func TestApplyStorageCleanupRepairsBrokenBucketMappings(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://legacy/a", "s3://bucket/a"}, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{{URL: "s3://legacy/a"}, {URL: "s3://bucket/a"}}, + }, + { + ObjectID: "obj-b", + Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + Organization: "org", + Project: "proj", + Size: 200, + UpdatedAt: &now, + AccessURLs: []string{"s3://legacy/b"}, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{{URL: "s3://legacy/b"}}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://legacy/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://legacy/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://legacy/a", + Status: "error", + ErrorKind: "credential_missing", + }, + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey("s3://legacy/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://legacy/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://legacy/b", + Status: "error", + ErrorKind: "credential_missing", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", []string{"data/a.txt", "data/b.txt"}, mirrorPath, repo, hash, true, false, false, false, true, false) + if err != nil { + t.Fatalf("apply cleanup broken bucket mapping repair: %v", err) + } + if len(applyResult.UpdatedRecordIDs) != 1 || applyResult.UpdatedRecordIDs[0] != "obj-a" { + t.Fatalf("expected obj-a access methods to be updated, got %+v", applyResult.UpdatedRecordIDs) + } + if len(applyResult.DeletedRecordIDs) != 1 || applyResult.DeletedRecordIDs[0] != "obj-b" { + t.Fatalf("expected obj-b to be deleted, got %+v", applyResult.DeletedRecordIDs) + } + updatedMethods := backend.updatedAccessMethods["obj-a"] + if len(updatedMethods) != 1 || updatedMethods[0].URL != "s3://bucket/a" { + t.Fatalf("expected obj-a to retain only good access method, got %+v", updatedMethods) + } + if len(backend.deletedIDs) != 1 || backend.deletedIDs[0] != "obj-b" { + t.Fatalf("expected obj-b to be deleted, got %+v", backend.deletedIDs) + } +} + +func TestApplyStorageCleanupDeletesBucketOnlyObjects(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/a"}, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{{URL: "s3://bucket/a"}}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", SizeBytes: 100}, + {ObjectURL: "s3://bucket/orphan", Bucket: "bucket", Key: "orphan", SizeBytes: 25}, + }, + } + service := NewStorageAnalyticsService(backend) + + applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", []string{"s3://bucket/orphan"}, mirrorPath, repo, hash, true, false, false, true, false, false) + if err != nil { + t.Fatalf("apply cleanup bucket only delete: %v", err) + } + if len(applyResult.DeletedBucketObjectURLs) != 1 || applyResult.DeletedBucketObjectURLs[0] != "s3://bucket/orphan" { + t.Fatalf("expected orphan bucket object to be deleted, got %+v", applyResult.DeletedBucketObjectURLs) + } + if len(backend.deletedBucketObjects) != 1 || backend.deletedBucketObjects[0] != "s3://bucket/orphan" { + t.Fatalf("expected backend bucket delete call, got %+v", backend.deletedBucketObjects) + } +} + +func TestBuildStorageCleanupAuditReportsStorageProbeFailures(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/b"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Status: "not_found", + Exists: false, + ValidationStatus: "unverifiable", + }, + storageProbeRequestKey("s3://bucket/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://bucket/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://bucket/b", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + assertHasCleanupFinding(t, cleanup.Findings, "storage_object_missing", "data/a.txt") + assertHasCleanupFinding(t, cleanup.Findings, "storage_validation_mismatch", "data/b.txt") + var mismatchFinding GitStorageCleanupFinding + for _, finding := range cleanup.Findings { + if finding.Kind == "storage_validation_mismatch" && finding.NormalizedPath == "data/b.txt" { + mismatchFinding = finding + break + } + } + if len(mismatchFinding.Records) != 1 || len(mismatchFinding.Records[0].AccessProbes) != 1 { + t.Fatalf("expected probe details on mismatch finding, got %+v", mismatchFinding) + } + if mismatchFinding.Records[0].AccessProbes[0].ValidationStatus != "mismatched" { + t.Fatalf("expected mismatched validation status, got %+v", mismatchFinding.Records[0].AccessProbes[0]) + } + if mismatchFinding.Checksum != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" { + t.Fatalf("expected checksum on mismatch finding, got %+v", mismatchFinding) + } + if mismatchFinding.Evidence == nil || + len(mismatchFinding.Evidence.AccessURLs) != 1 || + mismatchFinding.Evidence.AccessURLs[0] != "s3://bucket/b" || + len(mismatchFinding.Evidence.ProbeStatuses) != 1 || + mismatchFinding.Evidence.ProbeStatuses[0] != "present" || + mismatchFinding.Evidence.BucketEvaluation != "probed" { + t.Fatalf("expected storage evidence on mismatch finding, got %+v", mismatchFinding) + } +} + +func TestBuildStorageCleanupAuditFlagsRecordWhenAnyAccessProbeIsDead(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + missingKey := storageProbeRequestKey("s3://bucket/legacy-a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + presentKey := storageProbeRequestKey("s3://bucket/current-a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/legacy-a", "s3://bucket/current-a"}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + missingKey: { + ID: missingKey, + ObjectURL: "s3://bucket/legacy-a", + Status: "not_found", + Exists: false, + ValidationStatus: "unverifiable", + }, + presentKey: { + ID: presentKey, + ObjectURL: "s3://bucket/current-a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + finding := assertHasCleanupFinding(t, cleanup.Findings, "storage_object_missing", "data/a.txt") + if finding.Evidence == nil || !contains(finding.Evidence.AccessURLs, "s3://bucket/legacy-a") { + t.Fatalf("expected dead raw access URL evidence, got %+v", finding) + } +} + +func TestBuildStorageCleanupAuditReportsBrokenBucketMappingSeparately(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + probeKey := storageProbeRequestKey("s3://bforepc-prod/path/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc-prod/path/a.txt"}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + probeKey: { + ID: probeKey, + ObjectURL: "s3://bforepc-prod/path/a.txt", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "bforepc-prod"`, + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + assertHasCleanupFinding(t, cleanup.Findings, "broken_bucket_mapping", "data/a.txt") + for _, finding := range cleanup.Findings { + if finding.Kind == "broken_bucket_mapping" && finding.NormalizedPath == "data/a.txt" { + if len(finding.Records) != 1 || finding.Records[0].Error != "no Syfon bucket mapping is configured for this access URL" { + t.Fatalf("expected broken bucket mapping detail, got %+v", finding) + } + if finding.Evidence == nil || + len(finding.Evidence.AccessURLs) != 1 || + finding.Evidence.AccessURLs[0] != "s3://bforepc-prod/path/a.txt" || + len(finding.Evidence.ErrorKinds) != 1 || + finding.Evidence.ErrorKinds[0] != "credential_missing" || + finding.Evidence.BucketEvaluation != "probed" { + t.Fatalf("expected broken bucket mapping evidence, got %+v", finding) + } + return + } + } + t.Fatalf("missing broken bucket mapping detail in %+v", cleanup.Findings) +} + +func TestBuildStorageCleanupAuditStartsFromBucketInventory(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-orphan", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj", Size: 50, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/orphan"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + {ObjectURL: "s3://bucket/orphan", Bucket: "bucket", Key: "orphan", Path: "orphan", SizeBytes: 50}, + {ObjectURL: "s3://bucket/loose", Bucket: "bucket", Key: "loose", Path: "loose", SizeBytes: 25}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey("s3://bucket/orphan", 50, "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"): { + ID: storageProbeRequestKey("s3://bucket/orphan", 50, "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"), + ObjectURL: "s3://bucket/orphan", + Bucket: "bucket", + Key: "orphan", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + assertHasCleanupFinding(t, cleanup.Findings, "repo_orphan_live_object", "s3://bucket/orphan") + assertHasCleanupFinding(t, cleanup.Findings, "bucket_only_object", "s3://bucket/loose") +} + +func TestBuildStorageChainAuditUsesBucketFirstFindingKinds(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/missing.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "data/git-only.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 300), + "data/bad-map.txt": lfsPointer("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", 400), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-missing", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/missing"}}, + {ObjectID: "obj-no-git", Checksum: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", Organization: "org", Project: "proj", Size: 150, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/no-git"}}, + {ObjectID: "obj-bad-map", Checksum: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Organization: "org", Project: "proj", Size: 400, UpdatedAt: &now, AccessURLs: []string{"s3://legacy-bucket/bad-map"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + {ObjectURL: "s3://bucket/no-git", Bucket: "bucket", Key: "no-git", Path: "no-git", SizeBytes: 150}, + {ObjectURL: "s3://bucket/loose", Bucket: "bucket", Key: "loose", Path: "loose", SizeBytes: 25}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey("s3://bucket/missing", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://bucket/missing", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://bucket/missing", + Bucket: "bucket", + Key: "missing", + Status: "not_found", + Exists: false, + ValidationStatus: "unverifiable", + }, + storageProbeRequestKey("s3://bucket/no-git", 150, "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"): { + ID: storageProbeRequestKey("s3://bucket/no-git", 150, "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"), + ObjectURL: "s3://bucket/no-git", + Bucket: "bucket", + Key: "no-git", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey("s3://legacy-bucket/bad-map", 400, "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"): { + ID: storageProbeRequestKey("s3://legacy-bucket/bad-map", 400, "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"), + ObjectURL: "s3://legacy-bucket/bad-map", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "legacy-bucket"`, + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + findings := loadAllChainFindings(t, service, "org", "proj", chain) + assertHasChainFinding(t, findings, "bucket_only_object", "s3://bucket/loose") + assertHasChainFinding(t, findings, "bucket_syfon_no_git", "s3://bucket/no-git") + assertHasChainFinding(t, findings, "syfon_git_no_bucket", "data/missing.txt") + assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "data/bad-map.txt") + assertHasChainFinding(t, findings, "git_only_no_syfon", "data/git-only.txt") + if chain.Summary.BucketObjectCount != 3 || chain.Summary.SyfonRecordCount != 4 || chain.Summary.GitTrackedFileCount != 4 { + t.Fatalf("unexpected chain summary totals: %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditUsesProjectAuditSourcesAndTargetsProbes(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/missing.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/present"}}, + {ObjectID: "obj-missing", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/missing"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present", Bucket: "bucket", Key: "present", Path: "present", SizeBytes: 100, MetaSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/missing", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://bucket/missing", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://bucket/missing", + Bucket: "bucket", + Key: "missing", + Status: "not_found", + Exists: false, + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + findings := loadAllChainFindings(t, service, "org", "proj", chain) + assertHasChainFinding(t, findings, "syfon_git_no_bucket", "data/missing.txt") + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected one project audit record call, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.listProjectScopesCalls != 1 { + t.Fatalf("expected one project scope call, got %d", backend.listProjectScopesCalls) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected no project file usage calls, got %d", backend.listProjectFileUsageCalls) + } + if backend.listBucketsCalls != 0 || backend.listBucketScopesCalls != 0 { + t.Fatalf("expected no bucket-wide scope discovery, got listBuckets=%d listBucketScopes=%d", backend.listBucketsCalls, backend.listBucketScopesCalls) + } + if backend.probeCalls != 1 || len(backend.probeItems) != 1 || backend.probeItems[0].ObjectURL != "s3://bucket/missing" { + t.Fatalf("expected one targeted probe for missing object, got calls=%d items=%+v", backend.probeCalls, backend.probeItems) + } +} + +func TestBuildStorageChainAuditSurfacesMetadataMismatchAndEvidence(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 999}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + findings := loadAllChainFindings(t, service, "org", "proj", chain) + finding := assertHasChainFinding(t, findings, "git_syfon_metadata_mismatch", "data/a.txt") + if finding.Evidence == nil || + len(finding.Evidence.BucketObjectURLs) != 1 || + finding.Evidence.BucketObjectURLs[0] != "s3://bucket/a" || + len(finding.Evidence.ValidationStates) != 1 || + finding.Evidence.ValidationStates[0] != "mismatched" { + t.Fatalf("expected metadata mismatch evidence, got %+v", finding) + } +} + +func TestBuildStorageChainAuditUsesScopedProjectRecordsForGitJoin(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + bulkRecords: map[string][]gintegrationsyfon.ProjectRecord{}, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + for _, finding := range loadAllChainFindings(t, service, "org", "proj", chain) { + if finding.Kind == "git_only_no_syfon" && finding.NormalizedPath == "data/a.txt" { + t.Fatalf("did not expect scoped Syfon record to be reclassified as git-only when bulk checksum lookup is empty: %+v", finding) + } + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected one fully connected bucket->syfon->git chain, got summary %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/slide.ome.tiff": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "HTAN_INT", + Project: "BForePC", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc-prod/OHSU/slide.ome.tiff"}, + }, + }, + buckets: map[string]domain.StorageBucket{ + "bforepc": {Bucket: "bforepc", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bforepc": {{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", Bucket: "bforepc", Key: "bforepc-prod/OHSU/slide.ome.tiff", Path: "slide.ome.tiff", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bforepc-prod/OHSU/slide.ome.tiff", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bforepc-prod/OHSU/slide.ome.tiff", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bforepc-prod/OHSU/slide.ome.tiff", + Status: "not_found", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "bforepc-prod"`, + ValidationStatus: "unverifiable", + }, + storageProbeRequestKey("s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", + Bucket: "bforepc", + Key: "bforepc-prod/OHSU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { + t.Fatalf("expected stale raw scoped URL to surface as broken bucket mapping, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 0 { + t.Fatalf("expected stale raw scoped URL to block clean-chain count, got %+v", chain.Summary) + } + findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) + finding := assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "data/slide.ome.tiff") + if finding.Evidence == nil || !contains(finding.Evidence.AccessURLs, "s3://bforepc-prod/OHSU/slide.ome.tiff") { + t.Fatalf("expected raw stale access URL in evidence, got %+v", finding) + } + if len(backend.probeItems) != 2 { + t.Fatalf("expected raw and canonical probes, got %+v", backend.probeItems) + } + if !containsProbeTarget(backend.probeItems, "s3://bforepc-prod/OHSU/slide.ome.tiff") || !containsProbeTarget(backend.probeItems, "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff") { + t.Fatalf("expected probe targets to include raw and canonical URLs, got %+v", backend.probeItems) + } +} + +func TestBuildStorageChainAuditNormalizesChecksumJoinKeys(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/a.txt"}, + }, + }, + bulkRecords: map[string][]gintegrationsyfon.ProjectRecord{ + "sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {{ + ObjectID: "obj-a", + Checksum: "SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/a.txt"}, + }}, + }, + buckets: map[string]domain.StorageBucket{ + "bucket": {Bucket: "bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bucket": {{ + Bucket: "bucket", + Organization: "org", + ProjectID: "proj", + Path: "s3://bucket", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a.txt", Bucket: "bucket", Key: "a.txt", Path: "a.txt", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a.txt", + Bucket: "bucket", + Key: "a.txt", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["git_only_no_syfon"]; got != 0 { + t.Fatalf("expected normalized checksum join to avoid git-only false positive, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected one normalized complete chain, got %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditDegradesWhenProjectBucketListDenied(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/prefix/a.txt"}, + }, + }, + buckets: map[string]domain.StorageBucket{ + "bucket": {Bucket: "bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bucket": {{ + Bucket: "bucket", + Organization: "org", + ProjectID: "proj", + Path: "s3://bucket/prefix", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + listProjectBucketObjectsErr: fmt.Errorf("list syfon project bucket objects: syfon POST /data/inspect/project-bucket failed with status 409: provider rejected bucket inventory request for s3://bucket/prefix; mapped bucket target may be missing or inaccessible"), + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/prefix/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/prefix/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/prefix/a.txt", + Bucket: "bucket", + Key: "prefix/a.txt", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected one connected chain when bucket inventory degrades to probes, got %+v", chain.Summary) + } + if chain.Summary.BucketInventoryAvailable { + t.Fatalf("expected bucket inventory to be marked unavailable, got %+v", chain.Summary) + } + if !strings.Contains(chain.Summary.BucketInventoryError, "mapped bucket target may be missing or inaccessible") { + t.Fatalf("expected bucket inventory error detail, got %+v", chain.Summary) + } + for _, finding := range loadAllChainFindings(t, service, "org", "proj", chain) { + if finding.Kind == "git_only_no_syfon" { + t.Fatalf("did not expect git-only miss after degraded inventory fallback: %+v", finding) + } + } + if backend.listProjectBucketObjectsCalls != 1 { + t.Fatalf("expected one bucket inventory attempt, got %d", backend.listProjectBucketObjectsCalls) + } + if backend.probeCalls != 1 { + t.Fatalf("expected probe fallback to run once, got %d", backend.probeCalls) + } +} + +func TestBuildStorageChainAuditReturnsFullFindings(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/git-only.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 50), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + buckets: map[string]domain.StorageBucket{ + "bucket": {Bucket: "bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bucket": {{ + Bucket: "bucket", + Organization: "org", + ProjectID: "proj", + Path: "s3://bucket", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if len(chain.Findings) == 0 { + t.Fatalf("expected chain audit findings in response, got %+v", chain) + } + if len(chain.Groups) == 0 { + t.Fatalf("expected grouped summary rows, got %+v", chain) + } + if finding := assertHasChainFinding(t, chain.Findings, "git_only_no_syfon", "data/git-only.txt"); finding.NormalizedPath != "data/git-only.txt" { + t.Fatalf("unexpected chain finding: %+v", finding) + } +} + +func TestBuildStorageCleanupAuditTreatsMissingProbeEvidenceAsProbeError(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://legacy-bucket/a"}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/other", Bucket: "bucket", Key: "other", Path: "other", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://legacy-bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): {}, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + assertHasCleanupFinding(t, cleanup.Findings, "storage_probe_error", "data/a.txt") + for _, finding := range cleanup.Findings { + if finding.NormalizedPath == "data/a.txt" && finding.Kind == "storage_object_missing" { + t.Fatalf("did not expect missing-object classification without a real not_found probe, got %+v", finding) + } + } +} + +func TestBuildStorageCleanupAuditSkipsBucketStagesWhenCheckStorageDisabled(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + buckets: map[string]domain.StorageBucket{ + "bucket": {Bucket: "bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bucket": {{ + Bucket: "bucket", + Organization: "org", + ProjectID: "proj", + Path: "s3://bucket", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, false) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + if len(cleanup.Findings) != 0 { + t.Fatalf("expected no cleanup findings without storage checks, got %+v", cleanup.Findings) + } + if backend.listProjectBucketObjectsCalls != 0 { + t.Fatalf("expected bucket inventory to be skipped when check_storage=false, got %d calls", backend.listProjectBucketObjectsCalls) + } + if backend.probeCalls != 0 { + t.Fatalf("expected storage probes to be skipped when check_storage=false, got %d calls", backend.probeCalls) + } +} + +func TestPersistRepoAnalyticsIndexAndLoadExistingDirectoryWithoutLFSFiles(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "plain/notes.txt": "plain text only\n", + "plain/nested/x.md": "still plain\n", + }) + if err := PersistRepoAnalyticsIndex(context.Background(), mirrorPath, repo, refName, hash); err != nil { + t.Fatalf("persist repo analytics index: %v", err) + } + sidecar, err := readRepoAnalyticsIndexSidecar(mirrorPath) + if err != nil { + t.Fatalf("read repo analytics sidecar: %v", err) + } + if sidecar.CommitHash != hash.String() { + t.Fatalf("unexpected sidecar hash: %+v", sidecar) + } + index, err := loadOrBuildRepoAnalyticsIndex(context.Background(), mirrorPath, refName, repo, hash) + if err != nil { + t.Fatalf("load repo analytics index: %v", err) + } + directory, err := repoDirectoryAggregate(index, "plain") + if err != nil { + t.Fatalf("lookup plain directory aggregate: %v", err) + } + if directory.FileCount != 0 || directory.DirectChildCount != 0 { + t.Fatalf("expected zero-lfs directory aggregate, got %+v", directory) + } + filtered, err := filterRepoInventoryFiles(index, "plain") + if err != nil { + t.Fatalf("filter plain directory inventory: %v", err) + } + if len(filtered) != 0 { + t.Fatalf("expected no lfs files under plain directory, got %+v", filtered) + } +} + +func buildAnalyticsMirror(t *testing.T, files map[string]string) (*gogit.Repository, string, string, plumbing.Hash) { + t.Helper() + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + for filePath, content := range files { + fullPath := filepath.Join(sourcePath, filepath.FromSlash(filePath)) + if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filePath, err) + } + if err := os.WriteFile(fullPath, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", filePath, err) + } + if _, err := worktree.Add(filePath); err != nil { + t.Fatalf("add %s: %v", filePath, err) + } + } + if _, err := worktree.Commit("seed analytics repo", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit analytics repo: %v", err) + } + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve ref: %v", err) + } + return mirrorRepo, mirrorPath, refName, hash +} + +func lfsPointer(checksum string, size int64) string { + return strings.Join([]string{ + "version https://git-lfs.github.com/spec/v1", + "oid sha256:" + checksum, + "size " + strconv.FormatInt(size, 10), + "", + }, "\n") +} + +func ptrTime(value time.Time) *time.Time { + copyValue := value + return ©Value +} + +func contains(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func containsProbeTarget(values []gintegrationsyfon.BulkStorageProbeItem, target string) bool { + for _, value := range values { + if value.ObjectURL == target { + return true + } + } + return false +} + +func assertHasDiffFinding(t *testing.T, findings []GitProjectDiffFinding, kind string, path string) GitProjectDiffFinding { + t.Helper() + for _, finding := range findings { + if finding.Kind == kind && finding.NormalizedPath == path { + return finding + } + } + t.Fatalf("missing diff finding kind=%s path=%s in %+v", kind, path, findings) + return GitProjectDiffFinding{} +} + +func assertHasCleanupFinding(t *testing.T, findings []GitStorageCleanupFinding, kind string, path string) GitStorageCleanupFinding { + t.Helper() + for _, finding := range findings { + if finding.Kind == kind && finding.NormalizedPath == path { + return finding + } + } + t.Fatalf("missing cleanup finding kind=%s path=%s in %+v", kind, path, findings) + return GitStorageCleanupFinding{} +} + +func loadAllChainFindings(t *testing.T, service *StorageAnalyticsService, organization string, project string, chain *GitStorageChainAuditResponse) []GitStorageChainFinding { + t.Helper() + _, _ = service, organization + _ = project + if chain == nil { + t.Fatalf("expected chain audit response, got %+v", chain) + } + return append([]GitStorageChainFinding(nil), chain.Findings...) +} + +func assertHasChainFinding(t *testing.T, findings []GitStorageChainFinding, kind string, path string) GitStorageChainFinding { + t.Helper() + for _, finding := range findings { + if finding.Kind == kind && finding.NormalizedPath == path { + return finding + } + } + t.Fatalf("missing chain finding kind=%s path=%s in %+v", kind, path, findings) + return GitStorageChainFinding{} +} diff --git a/internal/git/storage_index.go b/internal/git/storage_index.go new file mode 100644 index 0000000..795cbe4 --- /dev/null +++ b/internal/git/storage_index.go @@ -0,0 +1,335 @@ +package git + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" +) + +const repoAnalyticsIndexSchemaVersion = 1 + +var repoAnalyticsIndexCache = &repoAnalyticsIndexMemoryCache{ + entries: map[string]*repoAnalyticsIndex{}, +} + +type repoAnalyticsIndexMemoryCache struct { + mu sync.RWMutex + entries map[string]*repoAnalyticsIndex +} + +type repoAnalyticsIndex struct { + sidecar GitRepoAnalyticsIndexSidecar + directoryLookup map[string]GitRepoAnalyticsDirectory +} + +func repoAnalyticsCacheKey(mirrorPath string, hash plumbing.Hash) string { + return strings.TrimSpace(mirrorPath) + "::" + hash.String() +} + +func (cache *repoAnalyticsIndexMemoryCache) get(mirrorPath string, hash plumbing.Hash) *repoAnalyticsIndex { + cache.mu.RLock() + defer cache.mu.RUnlock() + entry := cache.entries[repoAnalyticsCacheKey(mirrorPath, hash)] + if entry == nil { + return nil + } + return entry +} + +func (cache *repoAnalyticsIndexMemoryCache) put(mirrorPath string, hash plumbing.Hash, index *repoAnalyticsIndex) { + if index == nil { + return + } + cache.mu.Lock() + defer cache.mu.Unlock() + prefix := strings.TrimSpace(mirrorPath) + "::" + for key := range cache.entries { + if strings.HasPrefix(key, prefix) && key != repoAnalyticsCacheKey(mirrorPath, hash) { + delete(cache.entries, key) + } + } + cache.entries[repoAnalyticsCacheKey(mirrorPath, hash)] = index +} + +func RepoAnalyticsIndexPath(mirrorPath string) string { + return mirrorPath + ".analytics.index.json" +} + +func PersistRepoAnalyticsIndex(ctx context.Context, mirrorPath string, repo *gogit.Repository, ref string, hash plumbing.Hash) error { + index, err := buildRepoAnalyticsIndex(ref, repo, hash) + if err != nil { + return err + } + if err := writeRepoAnalyticsIndexSidecar(mirrorPath, index.sidecar); err != nil { + return err + } + repoAnalyticsIndexCache.put(mirrorPath, hash, index) + _ = ctx + return nil +} + +func loadOrBuildRepoAnalyticsIndex(_ context.Context, mirrorPath string, ref string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, error) { + if cached := repoAnalyticsIndexCache.get(mirrorPath, hash); cached != nil { + return cached, nil + } + sidecar, err := readRepoAnalyticsIndexSidecar(mirrorPath) + if err == nil && sidecar.SchemaVersion == repoAnalyticsIndexSchemaVersion && sidecar.CommitHash == hash.String() { + index := repoAnalyticsIndexFromSidecar(sidecar) + repoAnalyticsIndexCache.put(mirrorPath, hash, index) + return index, nil + } + index, buildErr := buildRepoAnalyticsIndex(ref, repo, hash) + if buildErr != nil { + return nil, buildErr + } + if writeErr := writeRepoAnalyticsIndexSidecar(mirrorPath, index.sidecar); writeErr != nil { + return nil, writeErr + } + repoAnalyticsIndexCache.put(mirrorPath, hash, index) + return index, nil +} + +func readRepoAnalyticsIndexSidecar(mirrorPath string) (GitRepoAnalyticsIndexSidecar, error) { + sidecarPath := RepoAnalyticsIndexPath(mirrorPath) + contentBytes, err := os.ReadFile(sidecarPath) + if err != nil { + return GitRepoAnalyticsIndexSidecar{}, fmt.Errorf("read repo analytics index sidecar: %w", err) + } + sidecar := GitRepoAnalyticsIndexSidecar{} + if err := json.Unmarshal(contentBytes, &sidecar); err != nil { + return GitRepoAnalyticsIndexSidecar{}, fmt.Errorf("decode repo analytics index sidecar: %w", err) + } + return sidecar, nil +} + +func writeRepoAnalyticsIndexSidecar(mirrorPath string, sidecar GitRepoAnalyticsIndexSidecar) error { + sidecarPath := RepoAnalyticsIndexPath(mirrorPath) + if err := os.MkdirAll(filepath.Dir(sidecarPath), 0o755); err != nil { + return fmt.Errorf("create repo analytics sidecar dir: %w", err) + } + contentBytes, err := json.Marshal(sidecar) + if err != nil { + return fmt.Errorf("encode repo analytics index sidecar: %w", err) + } + tempPath := sidecarPath + ".tmp" + if err := os.WriteFile(tempPath, contentBytes, 0o644); err != nil { + return fmt.Errorf("write repo analytics index temp sidecar: %w", err) + } + if err := os.Rename(tempPath, sidecarPath); err != nil { + return fmt.Errorf("move repo analytics index sidecar into place: %w", err) + } + return nil +} + +func repoAnalyticsIndexFromSidecar(sidecar GitRepoAnalyticsIndexSidecar) *repoAnalyticsIndex { + directoryLookup := make(map[string]GitRepoAnalyticsDirectory, len(sidecar.Directories)) + for _, directory := range sidecar.Directories { + directoryLookup[directory.Path] = directory + } + return &repoAnalyticsIndex{ + sidecar: sidecar, + directoryLookup: directoryLookup, + } +} + +func buildRepoAnalyticsIndex(ref string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, error) { + commit, err := repo.CommitObject(hash) + if err != nil { + return nil, fmt.Errorf("load commit for ref %s: %w", ref, err) + } + tree, err := commit.Tree() + if err != nil { + return nil, fmt.Errorf("load git tree for ref %s: %w", ref, err) + } + files := make([]RepoInventoryFile, 0) + knownDirectories := map[string]struct{}{"": {}} + if err := walkGitRepoInventory("", tree, &files, knownDirectories); err != nil { + return nil, err + } + sort.Slice(files, func(i, j int) bool { + return files[i].RepoPath < files[j].RepoPath + }) + directories := buildRepoAnalyticsDirectories(files, knownDirectories) + sidecar := GitRepoAnalyticsIndexSidecar{ + SchemaVersion: repoAnalyticsIndexSchemaVersion, + CommitHash: hash.String(), + RefName: ref, + GeneratedAt: time.Now().UTC(), + Files: files, + Directories: directories, + } + return repoAnalyticsIndexFromSidecar(sidecar), nil +} + +func buildRepoAnalyticsDirectories(files []RepoInventoryFile, knownDirectories map[string]struct{}) []GitRepoAnalyticsDirectory { + type directoryBuilder struct { + GitRepoAnalyticsDirectory + children map[string]*GitRepoAnalyticsChild + } + builders := make(map[string]*directoryBuilder, len(knownDirectories)) + ensureDirectory := func(path string) *directoryBuilder { + builder := builders[path] + if builder != nil { + return builder + } + builder = &directoryBuilder{ + GitRepoAnalyticsDirectory: GitRepoAnalyticsDirectory{Path: path}, + children: map[string]*GitRepoAnalyticsChild{}, + } + builders[path] = builder + return builder + } + for path := range knownDirectories { + ensureDirectory(path) + } + for _, file := range files { + parentPath := "" + parts := strings.Split(file.RepoPath, "/") + for index, part := range parts { + directory := ensureDirectory(parentPath) + directory.FileCount++ + directory.TotalBytes += file.Size + childPath := part + if parentPath != "" { + childPath = parentPath + "/" + part + } + childType := "file" + if index < len(parts)-1 { + childType = "directory" + } + child := directory.children[childPath] + if child == nil { + child = &GitRepoAnalyticsChild{ + Name: part, + Path: childPath, + Type: childType, + } + directory.children[childPath] = child + } + child.FileCount++ + child.TotalBytes += file.Size + if childType == "directory" { + parentPath = childPath + ensureDirectory(parentPath) + } + } + } + paths := make([]string, 0, len(builders)) + for path := range builders { + paths = append(paths, path) + } + sort.Strings(paths) + directories := make([]GitRepoAnalyticsDirectory, 0, len(paths)) + for _, path := range paths { + builder := builders[path] + childPaths := make([]string, 0, len(builder.children)) + for childPath := range builder.children { + childPaths = append(childPaths, childPath) + } + sort.Strings(childPaths) + builder.Children = make([]GitRepoAnalyticsChild, 0, len(childPaths)) + for _, childPath := range childPaths { + builder.Children = append(builder.Children, *builder.children[childPath]) + } + builder.DirectChildCount = len(builder.Children) + directories = append(directories, builder.GitRepoAnalyticsDirectory) + } + return directories +} + +func filterRepoInventoryFiles(index *repoAnalyticsIndex, gitSubpath string) ([]RepoInventoryFile, error) { + normalizedPath := normalizeRepoSubpath(gitSubpath) + if _, ok := index.directoryLookup[normalizedPath]; !ok { + return nil, fmt.Errorf("load git tree path %s: directory not found", normalizedPath) + } + if normalizedPath == "" { + return append([]RepoInventoryFile(nil), index.sidecar.Files...), nil + } + prefix := normalizedPath + "/" + filtered := make([]RepoInventoryFile, 0) + for _, item := range index.sidecar.Files { + if strings.HasPrefix(item.RepoPath, prefix) { + filtered = append(filtered, item) + } + } + return filtered, nil +} + +func repoDirectoryAggregate(index *repoAnalyticsIndex, gitSubpath string) (GitRepoAnalyticsDirectory, error) { + normalizedPath := normalizeRepoSubpath(gitSubpath) + directory, ok := index.directoryLookup[normalizedPath] + if !ok { + return GitRepoAnalyticsDirectory{}, fmt.Errorf("load git tree path %s: directory not found", normalizedPath) + } + return directory, nil +} + +func cloneDirectoryChildren(children []GitRepoAnalyticsChild) []storageAggregate { + out := make([]storageAggregate, 0, len(children)) + for _, child := range children { + out = append(out, storageAggregate{ + name: child.Name, + path: child.Path, + rowType: child.Type, + fileCount: child.FileCount, + totalBytes: child.TotalBytes, + }) + } + return out +} + +func walkGitRepoInventory(prefix string, tree *object.Tree, files *[]RepoInventoryFile, knownDirectories map[string]struct{}) error { + for _, entry := range tree.Entries { + entryPath := entry.Name + if prefix != "" { + entryPath = prefix + "/" + entry.Name + } + if entry.Mode == filemode.Dir { + knownDirectories[entryPath] = struct{}{} + childTree, err := tree.Tree(entry.Name) + if err != nil { + return fmt.Errorf("load nested git tree %s: %w", entryPath, err) + } + if err := walkGitRepoInventory(entryPath, childTree, files, knownDirectories); err != nil { + return err + } + continue + } + file, err := tree.File(entry.Name) + if err != nil { + continue + } + reader, err := file.Reader() + if err != nil { + continue + } + contentBytes, readErr := io.ReadAll(io.LimitReader(reader, 2048)) + _ = reader.Close() + if readErr != nil { + continue + } + pointer := ParseGitLFSPointer(contentBytes) + if pointer == nil { + continue + } + *files = append(*files, RepoInventoryFile{ + RepoPath: entryPath, + Name: entry.Name, + Checksum: pointer.OID, + Size: pointer.Size, + }) + } + return nil +} diff --git a/internal/git/types.go b/internal/git/types.go index 5c854c6..e5e31fd 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -280,6 +280,262 @@ type GitLFSPointerInfo struct { Size int64 `json:"size"` } +type GitRepoAnalyticsChild struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + FileCount int `json:"file_count"` + TotalBytes int64 `json:"total_bytes"` +} + +type GitRepoAnalyticsDirectory struct { + Path string `json:"path"` + DirectChildCount int `json:"direct_child_count"` + FileCount int `json:"file_count"` + TotalBytes int64 `json:"total_bytes"` + Children []GitRepoAnalyticsChild `json:"children"` +} + +type GitRepoAnalyticsIndexSidecar struct { + SchemaVersion int `json:"schema_version"` + CommitHash string `json:"commit_hash"` + RefName string `json:"ref_name"` + GeneratedAt time.Time `json:"generated_at"` + Files []RepoInventoryFile `json:"files"` + Directories []GitRepoAnalyticsDirectory `json:"directories"` +} + +type GitStorageSummaryResponse struct { + Path string `json:"path"` + FileCount int `json:"file_count"` + RecordCount int `json:"record_count"` + DirectChildCount int `json:"direct_child_count"` + TotalBytes int64 `json:"total_bytes"` + DownloadCount int64 `json:"download_count"` + LastDownloadTime string `json:"last_download_time,omitempty"` + LatestUpdateTime string `json:"latest_update_time,omitempty"` + DuplicatePathCount int `json:"duplicate_path_count"` +} + +type GitStorageChildResponseItem struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + FileCount int `json:"file_count"` + RecordCount int `json:"record_count"` + TotalBytes int64 `json:"total_bytes"` + DownloadCount int64 `json:"download_count"` + LastDownloadTime string `json:"last_download_time,omitempty"` + LatestUpdateTime string `json:"latest_update_time,omitempty"` +} + +type GitStorageChildrenResponse struct { + Items []GitStorageChildResponseItem `json:"items"` +} + +type GitProjectDiffAuditRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` +} + +type GitAuditEvidence struct { + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` + Buckets []string `json:"buckets,omitempty"` + Keys []string `json:"keys,omitempty"` + ProbeStatuses []string `json:"probe_statuses,omitempty"` + ValidationStates []string `json:"validation_states,omitempty"` + ErrorKinds []string `json:"error_kinds,omitempty"` + Errors []string `json:"errors,omitempty"` + BucketEvaluation string `json:"bucket_evaluation,omitempty"` +} + +type GitProjectDiffFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids"` + RecordCount int `json:"record_count"` + SizeBytes int64 `json:"size_bytes,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` + RecommendedAction string `json:"recommended_action"` + Evidence *GitAuditEvidence `json:"evidence,omitempty"` +} + +type GitProjectDiffSummary struct { + CountsByKind map[string]int `json:"counts_by_kind"` + TotalFindings int `json:"total_findings"` + IndexedPathCount int `json:"indexed_path_count"` + ExpectedPathCount int `json:"expected_path_count"` + MatchedPathCount int `json:"matched_path_count"` + IncludesRepoManifest bool `json:"includes_repo_manifest"` + ScannedRecordCount int `json:"scanned_record_count"` +} + +type GitProjectDiffAuditResponse struct { + Findings []GitProjectDiffFinding `json:"findings"` + Summary GitProjectDiffSummary `json:"summary"` + PathPrefix string `json:"path_prefix"` +} + +type GitStorageCleanupAuditRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + CheckStorage bool `json:"check_storage,omitempty"` + SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` +} + +type GitStorageChainAuditRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` +} + +type GitStorageChainFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids"` + AccessURLs []string `json:"access_urls,omitempty"` + BucketObjectURL string `json:"bucket_object_url,omitempty"` + ResolvedBucket string `json:"resolved_bucket,omitempty"` + ResolvedKey string `json:"resolved_key,omitempty"` + ProbeStatus string `json:"probe_status,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + Error string `json:"error,omitempty"` + RecordCount int `json:"record_count"` + SizeBytes int64 `json:"size_bytes,omitempty"` + RecommendedAction string `json:"recommended_action"` + Evidence *GitAuditEvidence `json:"evidence,omitempty"` +} + +type GitStorageChainAuditSummary struct { + CountsByKind map[string]int `json:"counts_by_kind"` + TotalFindings int `json:"total_findings"` + BucketObjectCount int `json:"bucket_object_count"` + SyfonRecordCount int `json:"syfon_record_count"` + GitTrackedFileCount int `json:"git_tracked_file_count"` + BucketInventoryAvailable bool `json:"bucket_inventory_available"` + BucketInventoryError string `json:"bucket_inventory_error,omitempty"` +} + +type GitStorageChainIssueGroup struct { + Kind string `json:"kind"` + FindingCount int `json:"finding_count"` + PathCount int `json:"path_count"` + RecordCount int `json:"record_count"` + ObjectCount int `json:"object_count"` + TotalBytes int64 `json:"total_bytes,omitempty"` +} + +type GitStorageChainAuditResponse struct { + Findings []GitStorageChainFinding `json:"findings"` + Groups []GitStorageChainIssueGroup `json:"groups,omitempty"` + Summary GitStorageChainAuditSummary `json:"summary"` + PathPrefix string `json:"path_prefix"` +} + +type GitStorageCleanupAccessProbe struct { + URL string `json:"url"` + Provider string `json:"provider,omitempty"` + Bucket string `json:"bucket,omitempty"` + Key string `json:"key,omitempty"` + Path string `json:"path,omitempty"` + Exists *bool `json:"exists,omitempty"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + MetaSHA256 string `json:"meta_sha256,omitempty"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` + ValidationStatus string `json:"validation_status,omitempty"` + SizeMatch *bool `json:"size_match,omitempty"` + SHA256Match *bool `json:"sha256_match,omitempty"` + ValidationMismatches []string `json:"validation_mismatches,omitempty"` +} + +type GitStorageCleanupRecordAudit struct { + ObjectID string `json:"object_id"` + Checksum string `json:"checksum,omitempty"` + NormalizedPath string `json:"normalized_path,omitempty"` + CleanupScope string `json:"cleanup_scope"` + AccessProbes []GitStorageCleanupAccessProbe `json:"access_probes"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` + SizeBytes int64 `json:"size,omitempty"` + LastUpdated string `json:"updated_time,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` +} + +type GitStorageCleanupFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + ObjectIDs []string `json:"object_ids"` + Records []GitStorageCleanupRecordAudit `json:"records"` + RecommendedAction string `json:"recommended_action"` + RepoDeleteCandidate bool `json:"repo_delete_candidate"` + CleanupScope string `json:"cleanup_scope"` + SizeBytes int64 `json:"total_bytes,omitempty"` + LastUpdated string `json:"last_updated,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` + Evidence *GitAuditEvidence `json:"evidence,omitempty"` +} + +type GitStorageCleanupAuditSummary struct { + CountsByKind map[string]int `json:"counts_by_kind"` + TotalFindings int `json:"total_findings"` + ManualFindingCount int `json:"manual_finding_count"` + RepoDeleteCandidateCount int `json:"repo_delete_candidate_count"` + StaleDuplicateCount int `json:"stale_duplicate_count"` + RepoOrphanCount int `json:"repo_orphan_count"` +} + +type GitStorageCleanupAuditResponse struct { + Findings []GitStorageCleanupFinding `json:"findings"` + Summary GitStorageCleanupAuditSummary `json:"summary"` + ExpectedPathCount int `json:"expected_path_count"` + IncludesRepoManifest bool `json:"includes_repo_manifest"` + PathPrefix string `json:"path_prefix"` +} + +type GitStorageCleanupApplyRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + DeleteRepoOrphans bool `json:"delete_repo_orphans,omitempty"` + DeleteStaleDuplicates bool `json:"delete_stale_duplicates,omitempty"` + DeleteBucketOnlyObjects bool `json:"delete_bucket_only_objects,omitempty"` + RepairBrokenBucketMappings bool `json:"repair_broken_bucket_mappings,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` +} + +type GitStorageCleanupPurgeResult struct { + ObjectID string `json:"object_id"` + Success *bool `json:"success"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` +} + +type GitStorageCleanupApplyResponse struct { + DeletedRecordIDs []string `json:"deleted_record_ids"` + DeletedBucketObjectURLs []string `json:"deleted_bucket_object_urls"` + UpdatedRecordIDs []string `json:"updated_record_ids"` + PurgeResults []GitStorageCleanupPurgeResult `json:"purge_results"` + RepoDeletePaths []string `json:"repo_delete_paths"` + ManualPaths []string `json:"manual_paths"` + SkippedPaths []string `json:"skipped_paths"` + DryRun bool `json:"dry_run"` +} + type GitUploadSessionFileManifest struct { Name string `json:"name"` Size int64 `json:"size"` diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index 453137a..d89a172 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -2,24 +2,109 @@ package syfon import ( "context" + "encoding/json" "fmt" + "io" "net/http" "net/url" "path" + "sort" + "strconv" "strings" + "time" "github.com/calypr/gecko/internal/git/domain" "github.com/calypr/syfon/apigen/client/bucketapi" + drsapi "github.com/calypr/syfon/apigen/client/drs" + internalapi "github.com/calypr/syfon/apigen/client/internalapi" + metricsapi "github.com/calypr/syfon/apigen/client/metricsapi" syfonservices "github.com/calypr/syfon/client/services" ) const refreshAuthzHeader = "X-Syfon-Refresh-Authz" +const bulkStorageProbeBatchSize = 200 type Manager struct { baseURL string client *http.Client } +type ProjectRecord struct { + ObjectID string + Checksum string + Organization string + Project string + Size int64 + UpdatedAt *time.Time + CreatedAt *time.Time + AccessURLs []string + AccessMethods []ProjectAccessMethod +} + +type ProjectAccessMethod struct { + AccessID string + Type string + URL string + Headers []string +} + +type BulkStorageProbeItem struct { + ID string + ObjectURL string + ExpectedSizeBytes *int64 + ExpectedSHA256 string +} + +type BulkStorageProbeResult struct { + ID string + ObjectURL string + Provider string + Bucket string + Key string + Path string + Exists bool + Status string + Error string + ErrorKind string + SizeBytes *int64 + MetaSHA256 string + ETag string + LastModified string + ValidationStatus string + SizeMatch *bool + SHA256Match *bool + ValidationMismatches []string +} + +type ProjectBucketObject struct { + ObjectURL string + Provider string + Bucket string + Key string + Path string + SizeBytes int64 + MetaSHA256 string + ETag string + LastModified string +} + +type FileUsage struct { + ObjectID string + Name string + Size int64 + DownloadCount int64 + UploadCount int64 + LastAccessTime *time.Time + LastDownloadTime *time.Time + LastUploadTime *time.Time +} + +type ProjectBucketDeleteResult struct { + ObjectURL string + Status string + Error string +} + func NewManager(baseURL string, client *http.Client) *Manager { httpClient := client if httpClient == nil { @@ -114,6 +199,28 @@ func (manager *Manager) AddScope(ctx context.Context, authorizationHeader string return nil } +func (manager *Manager) ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) { + requestPath := "/data/buckets/" + url.PathEscape(strings.TrimSpace(bucket)) + "/scopes" + var scopes []struct { + Organization string `json:"organization"` + ProjectId string `json:"project_id"` + Path *string `json:"path"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, requestPath, nil, nil, &scopes); err != nil { + return nil, fmt.Errorf("request syfon bucket scopes: %w", err) + } + out := make([]domain.StorageBucketScope, 0, len(scopes)) + for _, scope := range scopes { + out = append(out, domain.StorageBucketScope{ + Bucket: strings.TrimSpace(bucket), + Organization: strings.TrimSpace(scope.Organization), + ProjectID: strings.TrimSpace(scope.ProjectId), + Path: stringValue(scope.Path), + }) + } + return out, nil +} + func (manager *Manager) CleanupProject(ctx context.Context, authorizationHeader string, organization string, project string) error { dataBaseURL, err := manager.dataAPIBaseURL() if err != nil { @@ -141,6 +248,473 @@ func (manager *Manager) CleanupProject(ctx context.Context, authorizationHeader return nil } +func (manager *Manager) ListProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]ProjectRecord, error) { + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) + page := 1 + out := make([]ProjectRecord, 0) + for { + params.Set("limit", "1000") + params.Set("page", strconv.Itoa(page)) + var response internalapi.ListRecordsResponse + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, "/index", params, nil, &response); err != nil { + return nil, fmt.Errorf("list syfon project records page %d: %w", page, err) + } + records := []internalapi.InternalRecord{} + if response.Records != nil { + records = *response.Records + } + if len(records) == 0 { + break + } + for _, record := range records { + projectRecord, ok := projectRecordFromInternal(record) + if ok { + out = append(out, projectRecord) + } + } + if len(records) < 1000 { + break + } + page++ + } + return out, nil +} + +func (manager *Manager) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]ProjectRecord, error) { + requestBody := struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + }{ + Organization: strings.TrimSpace(organization), + Project: strings.TrimSpace(project), + } + var response struct { + Items []struct { + ObjectID string `json:"object_id"` + Checksum string `json:"checksum"` + Organization string `json:"organization"` + Project string `json:"project"` + Size int64 `json:"size"` + CreatedTime string `json:"created_time"` + UpdatedTime string `json:"updated_time"` + AccessURLs []string `json:"access_urls"` + AccessMethods []struct { + AccessID string `json:"access_id"` + Type string `json:"type"` + URL string `json:"url"` + Headers []string `json:"headers"` + } `json:"access_methods"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-records", nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("list syfon project audit records: %w", err) + } + out := make([]ProjectRecord, 0, len(response.Items)) + for _, item := range response.Items { + checksum := normalizeSHA256(item.Checksum) + if checksum == "" { + continue + } + accessURLs := make([]string, 0, len(item.AccessURLs)) + for _, raw := range item.AccessURLs { + if trimmed := strings.TrimSpace(raw); trimmed != "" { + accessURLs = append(accessURLs, trimmed) + } + } + accessMethods := make([]ProjectAccessMethod, 0, len(item.AccessMethods)) + for _, method := range item.AccessMethods { + accessMethods = append(accessMethods, ProjectAccessMethod{ + AccessID: strings.TrimSpace(method.AccessID), + Type: strings.TrimSpace(method.Type), + URL: strings.TrimSpace(method.URL), + Headers: append([]string(nil), method.Headers...), + }) + } + out = append(out, ProjectRecord{ + ObjectID: strings.TrimSpace(item.ObjectID), + Checksum: checksum, + Organization: strings.TrimSpace(item.Organization), + Project: strings.TrimSpace(item.Project), + Size: item.Size, + CreatedAt: parseOptionalTime(optionalString(item.CreatedTime)), + UpdatedAt: parseOptionalTime(optionalString(item.UpdatedTime)), + AccessURLs: accessURLs, + AccessMethods: accessMethods, + }) + } + return out, nil +} + +func (manager *Manager) ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + requestBody := struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + }{ + Organization: strings.TrimSpace(organization), + Project: strings.TrimSpace(project), + } + var response struct { + Items []struct { + Bucket string `json:"bucket"` + Organization string `json:"organization"` + ProjectID string `json:"project_id"` + Path string `json:"path"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-scopes", nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("list syfon project scopes: %w", err) + } + out := make([]domain.StorageBucketScope, 0, len(response.Items)) + for _, item := range response.Items { + out = append(out, domain.StorageBucketScope{ + Bucket: strings.TrimSpace(item.Bucket), + Organization: strings.TrimSpace(item.Organization), + ProjectID: strings.TrimSpace(item.ProjectID), + Path: strings.TrimSpace(item.Path), + }) + } + return out, nil +} + +func (manager *Manager) BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]ProjectRecord, error) { + normalized := dedupeChecksums(checksums) + out := make(map[string][]ProjectRecord, len(normalized)) + if len(normalized) == 0 { + return out, nil + } + const batchSize = 200 + for start := 0; start < len(normalized); start += batchSize { + end := start + batchSize + if end > len(normalized) { + end = len(normalized) + } + requestBody := internalapi.BulkHashesRequest{Hashes: normalized[start:end]} + var response struct { + Results map[string][]internalapi.InternalRecord `json:"results"` + Records *[]internalapi.InternalRecord `json:"records"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/index/bulk/hashes", nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("bulk lookup syfon checksums: %w", err) + } + if len(response.Results) > 0 { + for _, records := range response.Results { + for _, record := range records { + projectRecord, ok := projectRecordFromInternal(record) + if !ok { + continue + } + if !recordMatchesScope(projectRecord, organization, project) { + continue + } + out[projectRecord.Checksum] = append(out[projectRecord.Checksum], projectRecord) + } + } + continue + } + if response.Records != nil { + for _, record := range *response.Records { + projectRecord, ok := projectRecordFromInternal(record) + if !ok { + continue + } + if !recordMatchesScope(projectRecord, organization, project) { + continue + } + out[projectRecord.Checksum] = append(out[projectRecord.Checksum], projectRecord) + } + } + } + return out, nil +} + +func (manager *Manager) ListProjectFileUsage(ctx context.Context, authorizationHeader string, organization string, project string, inactiveDays int) (map[string]FileUsage, error) { + out := make(map[string]FileUsage) + offset := 0 + for { + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) + params.Set("limit", "1000") + params.Set("offset", strconv.Itoa(offset)) + if inactiveDays > 0 { + params.Set("inactive_days", strconv.Itoa(inactiveDays)) + } + var response metricsapi.MetricsListResponse + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, "/index/v1/metrics/files", params, nil, &response); err != nil { + return nil, fmt.Errorf("list syfon metrics files offset %d: %w", offset, err) + } + items := []metricsapi.FileUsage{} + if response.Data != nil { + items = *response.Data + } + if len(items) == 0 { + break + } + for _, item := range items { + objectID := strings.TrimSpace(stringValue(item.ObjectId)) + if objectID == "" { + continue + } + out[objectID] = FileUsage{ + ObjectID: objectID, + Name: stringValue(item.Name), + Size: int64Value(item.Size), + DownloadCount: int64Value(item.DownloadCount), + UploadCount: int64Value(item.UploadCount), + LastAccessTime: item.LastAccessTime, + LastDownloadTime: item.LastDownloadTime, + LastUploadTime: item.LastUploadTime, + } + } + if len(items) < 1000 { + break + } + offset += len(items) + } + return out, nil +} + +func (manager *Manager) BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error { + normalized := dedupeStrings(objectIDs) + if len(normalized) == 0 { + return nil + } + requestBody := drsapi.BulkDeleteObjectsJSONRequestBody{BulkObjectIds: normalized} + if deleteStorageData { + requestBody.DeleteStorageData = &deleteStorageData + } + resp, err := manager.drsClient(authorizationHeader) + if err != nil { + return err + } + response, err := resp.BulkDeleteObjectsWithResponse(ctx, requestBody) + if err != nil { + return fmt.Errorf("bulk delete syfon objects: %w", err) + } + if response.StatusCode() != http.StatusOK && response.StatusCode() != http.StatusNoContent { + return fmt.Errorf("bulk delete syfon objects failed with status %d", response.StatusCode()) + } + return nil +} + +func (manager *Manager) DeleteProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, objectURLs []string) ([]ProjectBucketDeleteResult, error) { + requestBody := struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + ObjectURLs []string `json:"object_urls"` + }{ + Organization: strings.TrimSpace(organization), + Project: strings.TrimSpace(project), + ObjectURLs: dedupeStrings(objectURLs), + } + if len(requestBody.ObjectURLs) == 0 { + return []ProjectBucketDeleteResult{}, nil + } + var response struct { + Items []struct { + ObjectURL string `json:"object_url"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-bucket/delete", nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("delete syfon project bucket objects: %w", err) + } + out := make([]ProjectBucketDeleteResult, 0, len(response.Items)) + for _, item := range response.Items { + out = append(out, ProjectBucketDeleteResult{ + ObjectURL: strings.TrimSpace(item.ObjectURL), + Status: strings.TrimSpace(item.Status), + Error: strings.TrimSpace(item.Error), + }) + } + return out, nil +} + +func (manager *Manager) BulkUpdateAccessMethods(ctx context.Context, authorizationHeader string, updates map[string][]ProjectAccessMethod) error { + if len(updates) == 0 { + return nil + } + resp, err := manager.drsClient(authorizationHeader) + if err != nil { + return err + } + const batchSize = 200 + objectIDs := make([]string, 0, len(updates)) + for objectID := range updates { + if trimmed := strings.TrimSpace(objectID); trimmed != "" { + objectIDs = append(objectIDs, trimmed) + } + } + sort.Strings(objectIDs) + for start := 0; start < len(objectIDs); start += batchSize { + end := start + batchSize + if end > len(objectIDs) { + end = len(objectIDs) + } + body := drsapi.BulkUpdateAccessMethodsJSONRequestBody{ + Updates: make([]struct { + AccessMethods []drsapi.AccessMethod `json:"access_methods"` + ObjectId string `json:"object_id"` + }, 0, end-start), + } + for _, objectID := range objectIDs[start:end] { + methods := projectAccessMethodsToDRS(updates[objectID]) + if len(methods) == 0 { + continue + } + body.Updates = append(body.Updates, struct { + AccessMethods []drsapi.AccessMethod `json:"access_methods"` + ObjectId string `json:"object_id"` + }{ + ObjectId: objectID, + AccessMethods: methods, + }) + } + if len(body.Updates) == 0 { + continue + } + response, err := resp.BulkUpdateAccessMethodsWithResponse(ctx, body) + if err != nil { + return fmt.Errorf("bulk update syfon access methods: %w", err) + } + if response.StatusCode() != http.StatusOK { + return fmt.Errorf("bulk update syfon access methods failed with status %d", response.StatusCode()) + } + } + return nil +} + +func (manager *Manager) BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []BulkStorageProbeItem) ([]BulkStorageProbeResult, error) { + if len(items) == 0 { + return []BulkStorageProbeResult{}, nil + } + out := make([]BulkStorageProbeResult, 0, len(items)) + for start := 0; start < len(items); start += bulkStorageProbeBatchSize { + end := start + bulkStorageProbeBatchSize + if end > len(items) { + end = len(items) + } + requestBody := struct { + Items []struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` + } `json:"items"` + }{Items: make([]struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` + }, 0, end-start)} + for _, item := range items[start:end] { + requestBody.Items = append(requestBody.Items, struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` + }{ + ID: strings.TrimSpace(item.ID), + ObjectURL: strings.TrimSpace(item.ObjectURL), + ExpectedSizeBytes: item.ExpectedSizeBytes, + ExpectedSHA256: strings.TrimSpace(item.ExpectedSHA256), + }) + } + var response struct { + Items []struct { + ID string `json:"id"` + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Path string `json:"path"` + Exists bool `json:"exists"` + Status string `json:"status"` + Error string `json:"error"` + ErrorKind string `json:"error_kind"` + SizeBytes *int64 `json:"size_bytes"` + MetaSHA256 string `json:"meta_sha256"` + ETag string `json:"etag"` + LastModified string `json:"last_modified"` + ValidationStatus string `json:"validation_status"` + SizeMatch *bool `json:"size_match"` + SHA256Match *bool `json:"sha256_match"` + ValidationMismatches []string `json:"validation_mismatches"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/bulk", nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("bulk probe syfon storage objects: %w", err) + } + for _, item := range response.Items { + out = append(out, BulkStorageProbeResult{ + ID: strings.TrimSpace(item.ID), + ObjectURL: strings.TrimSpace(item.ObjectURL), + Provider: strings.TrimSpace(item.Provider), + Bucket: strings.TrimSpace(item.Bucket), + Key: strings.TrimSpace(item.Key), + Path: strings.TrimSpace(item.Path), + Exists: item.Exists, + Status: strings.TrimSpace(item.Status), + Error: strings.TrimSpace(item.Error), + ErrorKind: strings.TrimSpace(item.ErrorKind), + SizeBytes: item.SizeBytes, + MetaSHA256: strings.TrimSpace(item.MetaSHA256), + ETag: strings.TrimSpace(item.ETag), + LastModified: strings.TrimSpace(item.LastModified), + ValidationStatus: strings.TrimSpace(item.ValidationStatus), + SizeMatch: item.SizeMatch, + SHA256Match: item.SHA256Match, + ValidationMismatches: append([]string(nil), item.ValidationMismatches...), + }) + } + } + return out, nil +} + +func (manager *Manager) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string) ([]ProjectBucketObject, error) { + requestBody := struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + }{ + Organization: strings.TrimSpace(organization), + Project: strings.TrimSpace(project), + } + var response struct { + Items []struct { + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + MetaSHA256 string `json:"meta_sha256"` + ETag string `json:"etag"` + LastModified string `json:"last_modified"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-bucket", nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("list syfon project bucket objects: %w", err) + } + out := make([]ProjectBucketObject, 0, len(response.Items)) + for _, item := range response.Items { + out = append(out, ProjectBucketObject{ + ObjectURL: strings.TrimSpace(item.ObjectURL), + Provider: strings.TrimSpace(item.Provider), + Bucket: strings.TrimSpace(item.Bucket), + Key: strings.TrimSpace(item.Key), + Path: strings.TrimSpace(item.Path), + SizeBytes: item.SizeBytes, + MetaSHA256: strings.TrimSpace(item.MetaSHA256), + ETag: strings.TrimSpace(item.ETag), + LastModified: strings.TrimSpace(item.LastModified), + }) + } + return out, nil +} + func (manager *Manager) bucketsService(authorizationHeader string) (*syfonservices.BucketsService, error) { clientBaseURL, err := manager.clientBaseURL() if err != nil { @@ -160,6 +734,25 @@ func (manager *Manager) bucketsService(authorizationHeader string) (*syfonservic return syfonservices.NewBucketsService(client), nil } +func (manager *Manager) drsClient(authorizationHeader string) (*drsapi.ClientWithResponses, error) { + clientBaseURL, err := manager.clientBaseURL() + if err != nil { + return nil, err + } + client, err := drsapi.NewClientWithResponses(clientBaseURL, + drsapi.WithHTTPClient(manager.client), + drsapi.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error { + req.Header.Set("Authorization", authorizationHeader) + req.Header.Set(refreshAuthzHeader, "true") + return nil + }), + ) + if err != nil { + return nil, fmt.Errorf("create syfon drs client: %w", err) + } + return client, nil +} + func (manager *Manager) clientBaseURL() (string, error) { if manager.baseURL == "" { return "", fmt.Errorf("SYFON_DATA_API_BASE_URL is not configured") @@ -223,3 +816,203 @@ func bucketPath(provider string, bucket string, prefix string) string { return "s3://" + cleanBucket + "/" + cleanPrefix } } + +func (manager *Manager) requestJSON(ctx context.Context, authorizationHeader string, method string, requestPath string, query url.Values, requestBody any, out any) error { + baseURL, err := manager.clientBaseURL() + if err != nil { + return err + } + queryURL, err := url.Parse(strings.TrimRight(baseURL, "/") + requestPath) + if err != nil { + return fmt.Errorf("parse syfon request url: %w", err) + } + if len(query) > 0 { + queryURL.RawQuery = query.Encode() + } + var body io.Reader + if requestBody != nil { + bodyBytes, marshalErr := json.Marshal(requestBody) + if marshalErr != nil { + return fmt.Errorf("marshal syfon request body: %w", marshalErr) + } + body = strings.NewReader(string(bodyBytes)) + } + req, err := http.NewRequestWithContext(ctx, method, queryURL.String(), body) + if err != nil { + return fmt.Errorf("build syfon request: %w", err) + } + req.Header.Set("Authorization", authorizationHeader) + req.Header.Set(refreshAuthzHeader, "true") + if requestBody != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := manager.client.Do(req) + if err != nil { + return fmt.Errorf("request syfon %s %s: %w", method, requestPath, err) + } + defer resp.Body.Close() + if resp.StatusCode >= http.StatusBadRequest { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("syfon %s %s failed with status %d: %s", method, requestPath, resp.StatusCode, strings.TrimSpace(string(bodyBytes))) + } + if out == nil { + return nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("decode syfon %s %s response: %w", method, requestPath, err) + } + return nil +} + +func projectRecordFromInternal(record internalapi.InternalRecord) (ProjectRecord, bool) { + checksum := "" + if record.Hashes != nil { + checksum = normalizeSHA256((*record.Hashes)["sha256"]) + } + if checksum == "" { + return ProjectRecord{}, false + } + accessURLs := make([]string, 0) + accessMethods := make([]ProjectAccessMethod, 0) + if record.AccessMethods != nil { + for _, method := range *record.AccessMethods { + projectMethod := ProjectAccessMethod{ + Type: strings.TrimSpace(string(method.Type)), + Headers: []string{}, + } + if method.AccessId != nil { + projectMethod.AccessID = strings.TrimSpace(*method.AccessId) + } + if method.AccessUrl != nil { + projectMethod.URL = strings.TrimSpace(method.AccessUrl.Url) + accessURLs = append(accessURLs, projectMethod.URL) + if method.AccessUrl.Headers != nil { + projectMethod.Headers = append([]string(nil), (*method.AccessUrl.Headers)...) + } + } + accessMethods = append(accessMethods, projectMethod) + } + } + return ProjectRecord{ + ObjectID: strings.TrimSpace(record.Did), + Checksum: checksum, + Organization: stringValue(record.Organization), + Project: stringValue(record.Project), + Size: int64Value(record.Size), + UpdatedAt: parseOptionalTime(record.UpdatedTime), + CreatedAt: parseOptionalTime(record.CreatedTime), + AccessURLs: accessURLs, + AccessMethods: accessMethods, + }, true +} + +func projectAccessMethodsToDRS(methods []ProjectAccessMethod) []drsapi.AccessMethod { + out := make([]drsapi.AccessMethod, 0, len(methods)) + for _, method := range methods { + accessMethod := drsapi.AccessMethod{} + if trimmed := strings.TrimSpace(method.AccessID); trimmed != "" { + accessMethod.AccessId = &trimmed + } + if trimmed := strings.TrimSpace(method.Type); trimmed != "" { + accessMethod.Type = drsapi.AccessMethodType(trimmed) + } + if trimmed := strings.TrimSpace(method.URL); trimmed != "" { + accessURL := struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: trimmed} + if len(method.Headers) > 0 { + headers := append([]string(nil), method.Headers...) + accessURL.Headers = &headers + } + accessMethod.AccessUrl = &accessURL + } + out = append(out, accessMethod) + } + return out +} + +func stringValue(value *string) string { + if value == nil { + return "" + } + return strings.TrimSpace(*value) +} + +func int64Value(value *int64) int64 { + if value == nil { + return 0 + } + return *value +} + +func parseOptionalTime(value *string) *time.Time { + if value == nil || strings.TrimSpace(*value) == "" { + return nil + } + parsed, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(*value)) + if err != nil { + return nil + } + parsed = parsed.UTC() + return &parsed +} + +func optionalString(value string) *string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return nil + } + return &trimmed +} + +func dedupeChecksums(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + normalized := normalizeSHA256(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + +func recordMatchesScope(record ProjectRecord, organization string, project string) bool { + recordOrg := strings.TrimSpace(record.Organization) + recordProject := strings.TrimSpace(record.Project) + if recordOrg == "" && recordProject == "" { + return true + } + return strings.EqualFold(recordOrg, organization) && strings.EqualFold(recordProject, project) +} + +func dedupeStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(strings.TrimPrefix(value, "sha256:")) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func normalizeSHA256(value string) string { + trimmed := strings.TrimSpace(strings.TrimPrefix(value, "sha256:")) + if trimmed == "" { + return "" + } + return strings.ToLower(trimmed) +} diff --git a/internal/integrations/syfon/adapter_test.go b/internal/integrations/syfon/adapter_test.go new file mode 100644 index 0000000..c2c4394 --- /dev/null +++ b/internal/integrations/syfon/adapter_test.go @@ -0,0 +1,233 @@ +package syfon + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "reflect" + "strconv" + "strings" + "testing" +) + +func TestBulkGetProjectRecordsByChecksumReadsResultsMap(t *testing.T) { + t.Helper() + + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/index/bulk/hashes" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Hashes []string `json:"hashes"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if !reflect.DeepEqual(req.Hashes, []string{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }) { + t.Fatalf("unexpected hashes payload: %#v", req.Hashes) + } + + org := "org" + project := "proj" + sizeA := int64(100) + sizeB := int64(200) + hashesA := map[string]string{"sha256": "sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} + hashesB := map[string]string{"sha256": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} + accessURLA := "s3://bucket/a" + accessURLB := "s3://bucket/b" + + response := map[string]any{ + "results": map[string]any{ + req.Hashes[0]: []any{ + map[string]any{ + "did": "obj-a", + "organization": org, + "project": project, + "size": sizeA, + "hashes": hashesA, + "access_methods": []any{ + map[string]any{ + "access_url": map[string]any{"url": accessURLA}, + }, + }, + }, + }, + req.Hashes[1]: []any{ + map[string]any{ + "did": "obj-b", + "organization": org, + "project": project, + "size": sizeB, + "hashes": hashesB, + "access_methods": []any{ + map[string]any{ + "access_url": map[string]any{"url": accessURLB}, + }, + }, + }, + }, + }, + } + body, err := json.Marshal(response) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + records, err := manager.BulkGetProjectRecordsByChecksum( + context.Background(), + "Bearer token", + "org", + "proj", + []string{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + ) + if err != nil { + t.Fatalf("BulkGetProjectRecordsByChecksum returned error: %v", err) + } + + if len(records) != 2 { + t.Fatalf("expected 2 checksum groups, got %#v", records) + } + if len(records["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]) != 1 { + t.Fatalf("expected first checksum match, got %#v", records) + } + if len(records["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"]) != 1 { + t.Fatalf("expected second checksum match, got %#v", records) + } + if got := records["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"][0].ObjectID; got != "obj-a" { + t.Fatalf("expected obj-a, got %q", got) + } + if got := records["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"][0].AccessURLs; !reflect.DeepEqual(got, []string{"s3://bucket/b"}) { + t.Fatalf("unexpected access urls: %#v", got) + } +} + +func TestBulkGetProjectRecordsByChecksumAllowsLegacyUnscopedResults(t *testing.T) { + t.Helper() + + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + response := map[string]any{ + "results": map[string]any{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": []any{ + map[string]any{ + "did": "obj-a", + "size": 100, + "hashes": map[string]string{"sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + }, + }, + }, + } + body, err := json.Marshal(response) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + records, err := manager.BulkGetProjectRecordsByChecksum( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + ) + if err != nil { + t.Fatalf("BulkGetProjectRecordsByChecksum returned error: %v", err) + } + if len(records["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]) != 1 { + t.Fatalf("expected fallback match for legacy unscoped result, got %#v", records) + } +} + +func TestBulkProbeStorageObjectsBatchesRequests(t *testing.T) { + t.Helper() + + requests := 0 + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/bulk" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + requests++ + var req struct { + Items []struct { + ID string `json:"id"` + ObjectURL string `json:"object_url"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes"` + ExpectedSHA256 string `json:"expected_sha256"` + } `json:"items"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if len(req.Items) == 0 || len(req.Items) > bulkStorageProbeBatchSize { + t.Fatalf("unexpected batch size %d", len(req.Items)) + } + respItems := make([]map[string]any, 0, len(req.Items)) + for _, item := range req.Items { + respItems = append(respItems, map[string]any{ + "id": item.ID, + "object_url": item.ObjectURL, + "exists": true, + "status": "present", + "validation_status": "matched", + }) + } + body, err := json.Marshal(map[string]any{"items": respItems}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + items := make([]BulkStorageProbeItem, 0, bulkStorageProbeBatchSize+1) + for i := 0; i < bulkStorageProbeBatchSize+1; i++ { + size := int64(i + 1) + items = append(items, BulkStorageProbeItem{ + ID: "item-" + strconv.Itoa(i), + ObjectURL: "s3://bucket/object-" + strconv.Itoa(i), + ExpectedSizeBytes: &size, + ExpectedSHA256: strings.Repeat("a", 64), + }) + } + + results, err := manager.BulkProbeStorageObjects(context.Background(), "Bearer token", items) + if err != nil { + t.Fatalf("BulkProbeStorageObjects returned error: %v", err) + } + if requests != 2 { + t.Fatalf("expected 2 bulk probe requests, got %d", requests) + } + if len(results) != len(items) { + t.Fatalf("expected %d results, got %d", len(items), len(results)) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} diff --git a/internal/server/http/git/handler.go b/internal/server/http/git/handler.go index 947fda8..94eb969 100644 --- a/internal/server/http/git/handler.go +++ b/internal/server/http/git/handler.go @@ -22,11 +22,16 @@ type Handler struct { gitService *git.GitService projectSetup *git.SetupService projectSync *git.ReconcileService + storageAnalytics *git.StorageAnalyticsService thumbnailStore thumbnail.Manager presentationStore presentation.Manager } func NewHandler(sharedHandler *shared.Handler) *Handler { + var storageAnalytics *git.StorageAnalyticsService + if sharedHandler.GitService != nil && sharedHandler.SyfonManager != nil { + storageAnalytics = git.NewStorageAnalyticsService(sharedHandler.SyfonManager) + } return &Handler{ Handler: sharedHandler, db: sharedHandler.DB, @@ -38,6 +43,7 @@ func NewHandler(sharedHandler *shared.Handler) *Handler { gitService: sharedHandler.GitService, projectSetup: sharedHandler.ProjectSetup, projectSync: sharedHandler.ProjectSync, + storageAnalytics: storageAnalytics, thumbnailStore: sharedHandler.ThumbnailStore, presentationStore: sharedHandler.PresentationStore, } diff --git a/internal/server/http/git/installation.go b/internal/server/http/git/installation.go index dc727ad..8d90e7d 100644 --- a/internal/server/http/git/installation.go +++ b/internal/server/http/git/installation.go @@ -46,7 +46,7 @@ func (handler *Handler) handleGitOrganizationInitConnectPOST(ctx fiber.Ctx) erro if redirectPath == "" { redirectPath = "/git" } - connectCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + connectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() redirectURL, err := handler.gitService.RequestInstallationURL( connectCtx, @@ -149,7 +149,7 @@ func (handler *Handler) handleGitOrganizationConnectPOST(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - connectCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + connectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() installation, err := handler.gitService.RequestOrganizationInstallationStatus( connectCtx, @@ -262,7 +262,7 @@ func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - connectCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + connectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() projectCfg, errResponse := handler.loadProjectConfig(connectCtx, projectID) if errResponse != nil { diff --git a/internal/server/http/git/register.go b/internal/server/http/git/register.go index 53da818..4d18c3b 100644 --- a/internal/server/http/git/register.go +++ b/internal/server/http/git/register.go @@ -33,6 +33,8 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler gitGroup.Get("/projects/:orgTitle/:projectTitle/manifest/*", projectReadAuth, handler.handleGitProjectManifestGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/file/*", projectReadAuth, handler.handleGitProjectFileGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/download/*", projectReadAuth, handler.handleGitProjectDownloadGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/storage/summary", projectReadAuth, handler.handleGitProjectStorageSummaryGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/storage/children", projectReadAuth, handler.handleGitProjectStorageChildrenGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/thumbnail", handler.handleGitProjectThumbnailGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/presentationConfig", projectConfigReadAuth, handler.handleGitProjectPresentationConfigGET) @@ -47,6 +49,10 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler projectGitWrite.Post("/presentationConfig", projectConfigWriteAuth, handler.handleGitProjectPresentationConfigPUT) projectGitWrite.Post("/edit-connect", projectWriteAuth, handler.handleGitProjectEditConnectPOST) projectGitWrite.Post("/update", projectWriteAuth, handler.handleGitProjectUpdatePOST) + projectGitWrite.Post("/repair/project-diff/audit", projectReadAuth, handler.handleGitProjectDiffAuditPOST) + projectGitWrite.Post("/repair/storage-chain/audit", projectReadAuth, handler.handleGitProjectStorageChainAuditPOST) + projectGitWrite.Post("/repair/storage-cleanup/audit", projectReadAuth, handler.handleGitProjectStorageCleanupAuditPOST) + projectGitWrite.Post("/repair/storage-cleanup/apply", projectReadAuth, handler.handleGitProjectStorageCleanupApplyPOST) projectGitWrite.Post("/uploads/session", projectWriteAuth, handler.handleGitProjectUploadSessionPOST) projectGitWrite.Get("/uploads/session/:sessionID", projectWriteAuth, handler.handleGitProjectUploadSessionGET) projectGitWrite.Post("/uploads/session/:sessionID/files", projectWriteAuth, handler.handleGitProjectUploadSessionFilesPOST) diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go new file mode 100644 index 0000000..ba103c4 --- /dev/null +++ b/internal/server/http/git/storage_analytics.go @@ -0,0 +1,322 @@ +package git + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/calypr/gecko/apierror" + gitcore "github.com/calypr/gecko/internal/git" + "github.com/calypr/gecko/internal/httputil" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/gofiber/fiber/v3" +) + +func (handler *Handler) handleGitProjectStorageSummaryGET(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + gitSubpath := normalizeAnalyticsSubpath(strings.TrimSpace(ctx.Query("git_subpath"))) + response, err := handler.storageAnalytics.BuildStorageSummary(ctx.Context(), projectCtx.authorizationHeader, projectCtx.organization, projectCtx.project, projectCtx.refName, gitSubpath, projectCtx.mirrorPath, projectCtx.repo, projectCtx.hash) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectStorageChildrenGET(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + gitSubpath := normalizeAnalyticsSubpath(strings.TrimSpace(ctx.Query("git_subpath"))) + limit := 1000 + if rawLimit := strings.TrimSpace(ctx.Query("limit")); rawLimit != "" { + parsed, err := strconv.Atoi(rawLimit) + if err != nil || parsed <= 0 { + response := httputil.NewError("invalid_request", "limit must be a positive integer", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + limit = parsed + } + response, err := handler.storageAnalytics.BuildStorageChildren( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + gitSubpath, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + limit, + strings.TrimSpace(ctx.Query("sort_by")), + strings.TrimSpace(ctx.Query("sort_order")), + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectDiffAuditPOST(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + requestBody := gitcore.GitProjectDiffAuditRequest{} + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + projectCtx.applyRequestRef(requestBody.Ref) + gitSubpath := normalizeAnalyticsSubpath(requestBody.GitSubpath) + response, err := handler.storageAnalytics.BuildProjectDiffAudit(ctx.Context(), projectCtx.authorizationHeader, projectCtx.organization, projectCtx.project, projectCtx.refName, gitSubpath, projectCtx.mirrorPath, projectCtx.repo, projectCtx.hash) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectStorageCleanupAuditPOST(ctx fiber.Ctx) error { + projectCtx, requestBody, errResponse := handler.parseCleanupAnalyticsRequest(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + response, _, err := handler.storageAnalytics.BuildStorageCleanupAudit( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + requestBody.GitSubpath, + requestBody.SelectedRepoPaths, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + requestBody.CheckStorage, + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, requestBody.GitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + requestBody := gitcore.GitStorageChainAuditRequest{} + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + projectCtx.applyRequestRef(requestBody.Ref) + gitSubpath := normalizeAnalyticsSubpath(requestBody.GitSubpath) + response, err := handler.storageAnalytics.BuildStorageChainAudit( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + gitSubpath, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectStorageCleanupApplyPOST(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + requestBody := gitcore.GitStorageCleanupApplyRequest{} + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + projectCtx.applyRequestRef(requestBody.Ref) + requestBody.GitSubpath = normalizeAnalyticsSubpath(requestBody.GitSubpath) + requestBody.SelectedRepoPaths = normalizeAnalyticsPathList(requestBody.SelectedRepoPaths) + response, err := handler.storageAnalytics.ApplyStorageCleanup( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + requestBody.GitSubpath, + requestBody.SelectedRepoPaths, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + true, + requestBody.DeleteRepoOrphans, + requestBody.DeleteStaleDuplicates, + requestBody.DeleteBucketOnlyObjects, + requestBody.RepairBrokenBucketMappings, + requestBody.DryRun, + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, requestBody.GitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +type gitAnalyticsContext struct { + organization string + project string + projectID string + authorizationHeader string + defaultBranch string + refName string + mirrorPath string + repo *gogit.Repository + hash plumbing.Hash +} + +func (ctx *gitAnalyticsContext) applyRequestRef(ref string) { + ref = strings.TrimSpace(ref) + if ref == "" || ref == ctx.refName { + return + } + refName, hash, err := gitcore.ResolveGitReference(ctx.repo, ref, ctx.defaultBranch) + if err != nil { + return + } + ctx.refName = refName + ctx.hash = hash +} + +func (handler *Handler) parseCleanupAnalyticsRequest(ctx fiber.Ctx) (*gitAnalyticsContext, gitcore.GitStorageCleanupAuditRequest, *httputil.ErrorResponse) { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return nil, gitcore.GitStorageCleanupAuditRequest{}, errResponse + } + requestBody := gitcore.GitStorageCleanupAuditRequest{} + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return nil, gitcore.GitStorageCleanupAuditRequest{}, errResponse + } + projectCtx.applyRequestRef(requestBody.Ref) + requestBody.GitSubpath = normalizeAnalyticsSubpath(requestBody.GitSubpath) + requestBody.SelectedRepoPaths = normalizeAnalyticsPathList(requestBody.SelectedRepoPaths) + return projectCtx, requestBody, nil +} + +func (handler *Handler) resolveGitAnalyticsContext(ctx fiber.Ctx) (*gitAnalyticsContext, *httputil.ErrorResponse) { + if handler.storageAnalytics == nil { + response := httputil.NewError("internal_error", "storage analytics service is not configured", http.StatusInternalServerError, nil, nil) + response.WriteLog(handler.logger) + return nil, response + } + organization, project, projectID, _, identity, errResponse := handler.resolveGitProject(ctx) + if errResponse != nil { + return nil, errResponse + } + state, err := handler.loadGitProjectState(projectID, identity) + if err != nil { + response := httputil.NewError("database_error", fmt.Sprintf("failed to read git state: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } + if state == nil || state.MirrorPath == "" { + response := httputil.NewError("conflict", fmt.Sprintf("project %s has not been refreshed yet", projectID), http.StatusConflict, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } + authorizationHeader := strings.TrimSpace(ctx.Get("Authorization")) + if authorizationHeader != "" { + refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + if err != nil { + handler.logger.Warning("failed to warm git mirror for %s analytics: %v", projectID, err) + } + } + repo, err := gitcore.OpenRepository(state.MirrorPath) + if err != nil { + response := httputil.NewError("integration_error", fmt.Sprintf("failed to open git mirror: %s", err), http.StatusBadGateway, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } + if gitcore.RepositoryIsEmpty(repo) { + response := httputil.NewError("conflict", fmt.Sprintf("project %s has no Git content yet", projectID), http.StatusConflict, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } + defaultBranch := state.DefaultBranch.String + refName, hash, err := gitcore.ResolveGitReference(repo, strings.TrimSpace(ctx.Query("ref")), defaultBranch) + if err != nil { + response := httputil.NewError("not_found", fmt.Sprintf("failed to resolve git ref: %s", err), http.StatusNotFound, map[string]any{"project_id": projectID, "ref": ctx.Query("ref")}, nil) + response.WriteLog(handler.logger) + return nil, response + } + return &gitAnalyticsContext{ + organization: organization, + project: project, + projectID: projectID, + authorizationHeader: authorizationHeader, + defaultBranch: defaultBranch, + refName: refName, + mirrorPath: state.MirrorPath, + repo: repo, + hash: hash, + }, nil +} + +func parseOptionalAnalyticsBody(ctx fiber.Ctx, target any, details map[string]any) *httputil.ErrorResponse { + body := ctx.Body() + if len(body) == 0 { + return nil + } + return httputil.ParseJSONBody(body, target, details) +} + +func normalizeAnalyticsSubpath(raw string) string { + return strings.Trim(strings.TrimSpace(raw), "/") +} + +func normalizeAnalyticsPathList(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + normalized := normalizeAnalyticsSubpath(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + +func (handler *Handler) writeGitAnalyticsError(ctx fiber.Ctx, projectID string, ref string, gitSubpath string, err error) error { + statusCode := http.StatusBadGateway + errorType := "integration_error" + if strings.Contains(strings.ToLower(err.Error()), "git tree path") { + statusCode = http.StatusNotFound + errorType = "not_found" + } + response := httputil.NewError(apierror.Type(errorType), err.Error(), statusCode, map[string]any{ + "project_id": projectID, + "ref": ref, + "git_subpath": gitSubpath, + }, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) +} diff --git a/internal/server/http/shared/handler.go b/internal/server/http/shared/handler.go index 9ffe04a..90cca05 100644 --- a/internal/server/http/shared/handler.go +++ b/internal/server/http/shared/handler.go @@ -38,6 +38,7 @@ type Handler struct { GitService *git.GitService ProjectSetup *git.SetupService ProjectSync *git.ReconcileService + SyfonManager *gintegrationsyfon.Manager ThumbnailStore thumbnail.Manager PresentationStore presentation.Manager } @@ -45,8 +46,9 @@ type Handler struct { func NewHandler(deps Dependencies) *Handler { var projectSetup *git.SetupService var projectSync *git.ReconcileService + var storageManager *gintegrationsyfon.Manager if deps.GitService != nil { - storageManager := gintegrationsyfon.NewManager(strings.TrimSpace(os.Getenv("SYFON_DATA_API_BASE_URL")), http.DefaultClient) + storageManager = gintegrationsyfon.NewManager(strings.TrimSpace(os.Getenv("SYFON_DATA_API_BASE_URL")), http.DefaultClient) projectSetup = git.NewSetupService(deps.DB, deps.GitService, storageManager, servermw.NewFenceUserAccessHandler(nil)) projectSync = git.NewReconcileService( deps.DB, @@ -64,6 +66,7 @@ func NewHandler(deps Dependencies) *Handler { GitService: deps.GitService, ProjectSetup: projectSetup, ProjectSync: projectSync, + SyfonManager: storageManager, ThumbnailStore: deps.ThumbnailStore, PresentationStore: deps.PresentationStore, } diff --git a/internal/server/server.go b/internal/server/server.go index a7c847b..8375806 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -106,8 +106,8 @@ func (server *Server) Init() (*Server, error) { func (server *Server) MakeRouter() *fiber.App { app := fiber.New(fiber.Config{ ReadBufferSize: 32 * 1024, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, }) app.Use(func(ctx fiber.Ctx) error { From af56e952fc0c275ac505a3d4be4cb8998c168478 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 2 Jul 2026 13:40:52 -0700 Subject: [PATCH 13/36] update analytics --- internal/git/storage_analytics.go | 261 ++++++++++- internal/git/storage_analytics_pipeline.go | 353 +++++++++++++- internal/git/storage_analytics_test.go | 440 ++++++++++++++++-- internal/git/storage_analytics_timing.go | 93 ++++ internal/git/types.go | 24 +- internal/integrations/syfon/adapter.go | 277 ++++++++--- internal/integrations/syfon/adapter_test.go | 56 +++ internal/server/http/git/storage_analytics.go | 122 ++++- 8 files changed, 1468 insertions(+), 158 deletions(-) create mode 100644 internal/git/storage_analytics_timing.go diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index ed5d4c2..81ff3bf 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -3,6 +3,7 @@ package git import ( "context" "fmt" + "log" "path" "sort" "strings" @@ -17,6 +18,8 @@ import ( const cleanupInactiveDays = 30 const projectJoinCacheTTL = 45 * time.Second +const chainInputCacheTTL = 45 * time.Second +const storageChainValidationDebugSampleLimit = 20 type storageAnalyticsBackend interface { ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) @@ -26,8 +29,10 @@ type storageAnalyticsBackend interface { ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]gintegrationsyfon.ProjectRecord, error) ListProjectFileUsage(ctx context.Context, authorizationHeader string, organization string, project string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) - ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectBucketObject, error) + ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) + ListProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) + BulkListStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error DeleteProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, objectURLs []string) ([]gintegrationsyfon.ProjectBucketDeleteResult, error) BulkUpdateAccessMethods(ctx context.Context, authorizationHeader string, updates map[string][]gintegrationsyfon.ProjectAccessMethod) error @@ -37,6 +42,8 @@ type StorageAnalyticsService struct { storage storageAnalyticsBackend projectJoinMu sync.RWMutex projectJoinCache map[string]cachedProjectJoinState + chainInputMu sync.RWMutex + chainInputCache map[string]cachedChainInputState } func NewStorageAnalyticsService(storage storageAnalyticsBackend) *StorageAnalyticsService { @@ -46,6 +53,7 @@ func NewStorageAnalyticsService(storage storageAnalyticsBackend) *StorageAnalyti return &StorageAnalyticsService{ storage: storage, projectJoinCache: map[string]cachedProjectJoinState{}, + chainInputCache: map[string]cachedChainInputState{}, } } @@ -112,6 +120,15 @@ type cachedProjectJoinState struct { usageByObjectID map[string]gintegrationsyfon.FileUsage } +type cachedChainInputState struct { + expiresAt time.Time + projectRecords []gintegrationsyfon.ProjectRecord + projectScopes []domain.StorageBucketScope + bucketSummary *gintegrationsyfon.ProjectBucketSummary + bucketObjects []gintegrationsyfon.ProjectBucketObject + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject +} + func BuildGitRepoInventory(ref string, gitSubpath string, repo *gogit.Repository, hash plumbing.Hash) ([]RepoInventoryFile, error) { index, err := buildRepoAnalyticsIndex(ref, repo, hash) if err != nil { @@ -215,29 +232,66 @@ func (service *StorageAnalyticsService) BuildStorageCleanupAudit(ctx context.Con } func (service *StorageAnalyticsService) BuildStorageChainAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitStorageChainAuditResponse, error) { - inventory, err := service.loadStorageChainInventory(ctx, ref, gitSubpath, mirrorPath, repo, hash) - if err != nil { - return nil, err + return service.BuildStorageChainAuditWithOptions(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, StorageChainAuditOptions{}) +} + +func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, error) { + probeMode, ok := NormalizeStorageChainProbeMode(options.ProbeMode) + if !ok { + return nil, fmt.Errorf("invalid storage chain probe mode %q", options.ProbeMode) + } + bucketMode, ok := NormalizeStorageChainBucketInventoryMode(options.BucketInventoryMode) + if !ok { + return nil, fmt.Errorf("invalid storage chain bucket inventory mode %q", options.BucketInventoryMode) } - recordSet, err := service.loadProjectAuditRecordSet(ctx, authorizationHeader, organization, project) + bucketPathPrefix := normalizeRepoSubpath(options.BucketPathPrefix) + start := time.Now() + options.Timings.StageStart("chain_setup_total") + inputs, err := service.loadStorageChainInputs(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, bucketMode, bucketPathPrefix, options.Timings) + options.Timings.Record("chain_setup_total", time.Since(start)) if err != nil { return nil, err } - storageView, err := service.loadStorageChainView(ctx, authorizationHeader, organization, project, recordSet) + storageViewStart := time.Now() + options.Timings.StageStart("storage_view") + storageView, err := service.buildStorageChainView(ctx, authorizationHeader, inputs.recordSet, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, probeMode, options.Timings) + options.Timings.Record("storage_view", time.Since(storageViewStart)) if err != nil { return nil, err } - model := buildStorageChainAuditModel(gitSubpath, inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL) + modelStart := time.Now() + options.Timings.StageStart("model_build") + includeBucketOrigin := bucketMode != StorageChainBucketModeValidate && storageView.bucketInventoryAvailable + model := buildStorageChainAuditModel(gitSubpath, inputs.inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL, includeBucketOrigin) + options.Timings.Record("model_build", time.Since(modelStart)) model.Summary.BucketInventoryAvailable = storageView.bucketInventoryAvailable model.Summary.BucketInventoryError = storageView.bucketInventoryError + if inputs.bucketSummary != nil { + exists := inputs.bucketSummary.Exists + model.Summary.BucketPathExists = &exists + model.Summary.BucketPathObjectURL = strings.TrimSpace(inputs.bucketSummary.ObjectURL) + model.Summary.BucketSummaryMode = strings.TrimSpace(inputs.bucketSummary.Mode) + } + findings := limitStorageChainFindings(model.Findings, options.FindingLimit) + model.Summary.ReturnedFindings = len(findings) + model.Summary.FindingLimit = options.FindingLimit + model.Summary.FindingsTruncated = len(findings) < len(model.Findings) return &GitStorageChainAuditResponse{ - Findings: append([]GitStorageChainFinding(nil), model.Findings...), - Groups: summarizeChainIssueGroups(model.Findings), - Summary: model.Summary, - PathPrefix: model.PathPrefix, + Findings: findings, + Groups: summarizeChainIssueGroups(model.Findings), + Summary: model.Summary, + PathPrefix: model.PathPrefix, + BucketPathPrefix: bucketPathPrefix, }, nil } +func limitStorageChainFindings(findings []GitStorageChainFinding, limit int) []GitStorageChainFinding { + if limit <= 0 || limit >= len(findings) { + return append([]GitStorageChainFinding(nil), findings...) + } + return append([]GitStorageChainFinding(nil), findings[:limit]...) +} + func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, selectedRepoPaths []string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, checkStorage bool, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool, dryRun bool) (*GitStorageCleanupApplyResponse, error) { _, model, err := service.BuildStorageCleanupAudit(ctx, authorizationHeader, organization, project, ref, gitSubpath, selectedRepoPaths, mirrorPath, repo, hash, checkStorage) if err != nil { @@ -535,44 +589,105 @@ func applyScopedStorageMappings(recordsByChecksum map[string][]projectRecordStat } func (service *StorageAnalyticsService) attachStorageProbes(ctx context.Context, authorizationHeader string, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState) (map[string][]projectRecordState, map[string][]projectRecordState, error) { + return service.attachStorageValidationResults(ctx, authorizationHeader, recordsByChecksum, allProjectRecords, false) +} + +func (service *StorageAnalyticsService) attachStorageListValidations(ctx context.Context, authorizationHeader string, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState) (map[string][]projectRecordState, map[string][]projectRecordState, error) { + return service.attachStorageValidationResults(ctx, authorizationHeader, recordsByChecksum, allProjectRecords, true) +} + +func (service *StorageAnalyticsService) attachStorageValidationResults(ctx context.Context, authorizationHeader string, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, useListValidation bool) (map[string][]projectRecordState, map[string][]projectRecordState, error) { + started := time.Now() items := make([]gintegrationsyfon.BulkStorageProbeItem, 0) itemKeys := map[string]string{} recordProbeKeysByObjectID := map[string][]string{} + accessURLCount := 0 for _, group := range allProjectRecords { for _, record := range group { for _, accessURL := range probeAccessURLsForRecord(record) { + accessURLCount++ normalizedURL := strings.TrimSpace(accessURL) if normalizedURL == "" { continue } key := storageProbeRequestKey(normalizedURL, record.Size, record.Checksum) + expectedName := "" + if useListValidation { + expectedName = expectedStorageObjectNameForListValidation(normalizedURL, record.Name) + key = storageListValidationRequestKey(normalizedURL, record.Size, expectedName) + } recordProbeKeysByObjectID[record.ObjectID] = append(recordProbeKeysByObjectID[record.ObjectID], key) if _, ok := itemKeys[key]; ok { continue } itemKeys[key] = key expectedSize := record.Size - items = append(items, gintegrationsyfon.BulkStorageProbeItem{ + item := gintegrationsyfon.BulkStorageProbeItem{ ID: key, ObjectURL: normalizedURL, ExpectedSizeBytes: &expectedSize, ExpectedSHA256: strings.TrimSpace(record.Checksum), - }) + } + if useListValidation { + item.ExpectedSHA256 = "" + item.ExpectedName = expectedName + } + items = append(items, item) + if useListValidation && len(items) <= storageChainValidationDebugSampleLimit { + log.Printf( + "INFO: storage_chain_bulk_list_item object_id=%s name=%q checksum=%s submitted_url=%s raw_access_urls=%q canonical_access_urls=%q expected_size=%d expected_name=%q", + strings.TrimSpace(record.ObjectID), + strings.TrimSpace(record.Name), + strings.TrimSpace(record.Checksum), + normalizedURL, + strings.Join(rawAccessURLsForRecord(record), ","), + strings.Join(accessURLsForStorage(record), ","), + record.Size, + item.ExpectedName, + ) + } } } } + mode := "head" + if useListValidation { + mode = "list" + } + log.Printf( + "INFO: storage_chain_validation_request_built mode=%s record_count=%d access_url_count=%d unique_request_count=%d duplicate_request_count=%d build_ms=%d", + mode, + countRecordStates(allProjectRecords), + accessURLCount, + len(items), + accessURLCount-len(items), + time.Since(started).Milliseconds(), + ) resultsByKey := map[string]gintegrationsyfon.BulkStorageProbeResult{} if len(items) > 0 { - results, err := service.storage.BulkProbeStorageObjects(ctx, authorizationHeader, items) + var ( + results []gintegrationsyfon.BulkStorageProbeResult + err error + ) + if useListValidation { + results, err = service.storage.BulkListStorageObjects(ctx, authorizationHeader, items) + } else { + results, err = service.storage.BulkProbeStorageObjects(ctx, authorizationHeader, items) + } if err != nil { + if useListValidation { + log.Printf("INFO: storage_chain_validation_request_done mode=%s request_count=%d duration_ms=%d error=%q", mode, len(items), time.Since(started).Milliseconds(), err.Error()) + return nil, nil, fmt.Errorf("list-validate syfon storage objects: %w", err) + } + log.Printf("INFO: storage_chain_validation_request_done mode=%s request_count=%d duration_ms=%d error=%q", mode, len(items), time.Since(started).Milliseconds(), err.Error()) return nil, nil, fmt.Errorf("probe syfon storage objects: %w", err) } for _, result := range results { resultsByKey[strings.TrimSpace(result.ID)] = result } } + log.Printf("INFO: storage_chain_validation_request_done mode=%s request_count=%d result_count=%d duration_ms=%d", mode, len(items), len(resultsByKey), time.Since(started).Milliseconds()) attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { out := make(map[string][]projectRecordState, len(input)) @@ -598,6 +713,14 @@ func (service *StorageAnalyticsService) attachStorageProbes(ctx context.Context, return attach(recordsByChecksum), attach(allProjectRecords), nil } +func countRecordStates(input map[string][]projectRecordState) int { + count := 0 + for _, group := range input { + count += len(group) + } + return count +} + func summarizeSubtree(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, usageByObjectID map[string]gintegrationsyfon.FileUsage, directChildCount int) storageAggregate { agg := storageAggregate{ path: normalizeRepoSubpath(gitSubpath), @@ -1071,17 +1194,20 @@ func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[st if recordHasBrokenAccess(record) { return storageFindingBrokenAccessURL } - if kind := classifyRawAccessURLFindings(record); kind != storageFindingNone { - return kind - } bucketMatches := matchedBucketObjectURLs(record, bucketObjectsByURL) + if hasExactPathBucketMismatch(record, bucketMatches, bucketObjectsByURL) { + return storageFindingBrokenBucketMap + } if inventoryHasValidationMismatch(record, bucketMatches, bucketObjectsByURL) { return storageFindingValidationMismatch } + if len(bucketMatches) > 0 { + return storageFindingNone + } + if kind := classifyRawAccessURLFindings(record); kind != storageFindingNone { + return kind + } if len(record.AccessProbes) == 0 { - if len(bucketMatches) > 0 { - return storageFindingNone - } if len(bucketObjectsByURL) > 0 && hasCanonicalBucketURL(record) { return storageFindingProbeError } @@ -1125,6 +1251,45 @@ func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[st return storageFindingNone } +func hasExactPathBucketMismatch(record projectRecordState, bucketMatches []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + if len(bucketObjectsByURL) == 0 || len(bucketMatches) > 0 { + return false + } + expectedBucketURLs := accessURLsForStorage(record) + if len(expectedBucketURLs) == 0 { + return false + } + expectedBasenames := make(map[string]struct{}, len(expectedBucketURLs)) + for _, accessURL := range expectedBucketURLs { + _, key, ok := parseStorageURL(accessURL) + if !ok { + continue + } + name := strings.TrimSpace(path.Base(key)) + if name == "" { + continue + } + expectedBasenames[name] = struct{}{} + } + if len(expectedBasenames) == 0 { + return false + } + for _, item := range bucketObjectsByURL { + itemName := strings.TrimSpace(path.Base(strings.TrimSpace(item.Key))) + if itemName == "" { + continue + } + if _, ok := expectedBasenames[itemName]; !ok { + continue + } + if record.Size > 0 && item.SizeBytes > 0 && item.SizeBytes != record.Size { + continue + } + return true + } + return false +} + func inventoryHasValidationMismatch(record projectRecordState, bucketObjectURLs []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { if len(bucketObjectURLs) == 0 { return false @@ -1169,6 +1334,7 @@ func accessProbesForRecord(record projectRecordState) []GitStorageCleanupAccessP LastModified: probe.LastModified, ValidationStatus: probe.ValidationStatus, SizeMatch: probe.SizeMatch, + NameMatch: probe.NameMatch, SHA256Match: probe.SHA256Match, ValidationMismatches: append([]string(nil), probe.ValidationMismatches...), }) @@ -1278,9 +1444,16 @@ func buildChainBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) Git } func buildChainRecordFindings(kind string, record projectRecordState, gitPaths []string, bucketObjectURLs []string, action string) []GitStorageChainFinding { + return buildChainRecordFindingsWithOptions(kind, record, gitPaths, bucketObjectURLs, action, false) +} + +func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, gitPaths []string, bucketObjectURLs []string, action string, preferSyfonRecordPath bool) []GitStorageChainFinding { paths := uniqueStrings(gitPaths) if len(paths) == 0 { displayPath := orphanDisplayPath(strings.TrimSpace(record.Checksum), append(bucketObjectURLs, record.AccessURLs...)) + if preferSyfonRecordPath { + displayPath = syfonRecordDisplayPath(record) + } if displayPath == "" { displayPath = strings.TrimSpace(record.Checksum) } @@ -1339,7 +1512,7 @@ func selectChainProbe(record projectRecordState, bucketObjectURLs []string) chai for _, probe := range probes { objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.URL) if probe.Status != "" || probe.ErrorKind != "" || objectURL != "" || probe.URL != "" { - return chainProbeSelection{bucketObjectURL: objectURL, probe: probe} + return chainProbeSelection{probe: probe} } } return chainProbeSelection{} @@ -1912,6 +2085,39 @@ func storageProbeRequestKey(objectURL string, size int64, checksum string) strin return strings.TrimSpace(objectURL) + "|" + fmt.Sprintf("%d", size) + "|" + strings.TrimSpace(checksum) } +func storageListValidationRequestKey(objectURL string, size int64, expectedName string) string { + return strings.TrimSpace(objectURL) + "|" + fmt.Sprintf("%d", size) + "|" + strings.TrimSpace(expectedName) +} + +func expectedStorageObjectNameForListValidation(objectURL string, recordName string) string { + expectedName := path.Base(strings.Trim(strings.TrimSpace(recordName), "/")) + if expectedName == "." || expectedName == "/" || expectedName == "" { + return "" + } + _, key, ok := parseStorageURL(objectURL) + if !ok { + return expectedName + } + keyBase := path.Base(strings.Trim(key, "/")) + if isContentAddressedStorageBasename(keyBase) { + return "" + } + return expectedName +} + +func isContentAddressedStorageBasename(value string) bool { + trimmed := strings.ToLower(strings.TrimSpace(value)) + if len(trimmed) < 32 { + return false + } + for _, r := range trimmed { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + return true +} + func sortStorageAggregates(items []storageAggregate, sortBy string, sortOrder string) { desc := !strings.EqualFold(strings.TrimSpace(sortOrder), "asc") switch strings.ToLower(strings.TrimSpace(sortBy)) { @@ -1947,6 +2153,19 @@ func orphanDisplayPath(checksum string, sourcePaths []string) string { return "sha256/" + strings.TrimSpace(checksum) } +func syfonRecordDisplayPath(record projectRecordState) string { + if name := strings.TrimSpace(record.Name); name != "" { + return name + } + if objectID := strings.TrimSpace(record.ObjectID); objectID != "" { + return "syfon/" + objectID + } + if checksum := strings.TrimSpace(record.Checksum); checksum != "" { + return "sha256/" + checksum + } + return "" +} + func recordSourcePaths(matches []projectRecordState) []string { seen := make(map[string]struct{}) out := make([]string, 0) diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 209cc20..437d05e 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -3,8 +3,11 @@ package git import ( "context" "fmt" + "log" + "path" "sort" "strings" + "time" "github.com/calypr/gecko/internal/git/domain" gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" @@ -50,6 +53,119 @@ func (service *StorageAnalyticsService) loadStorageChainInventory(ctx context.Co return filterRepoInventoryFiles(index, gitSubpath) } +type storageChainInputs struct { + inventory []RepoInventoryFile + recordSet *storageAuditRecordSet + scopes []domain.StorageBucketScope + bucketSummary *gintegrationsyfon.ProjectBucketSummary + bucketObjects []gintegrationsyfon.ProjectBucketObject + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + bucketInventoryErr error +} + +func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, bucketMode string, bucketPathPrefix string, timings *StorageChainAuditTimings) (*storageChainInputs, error) { + type inventoryResult struct { + inventory []RepoInventoryFile + err error + } + type recordResult struct { + recordSet *storageAuditRecordSet + err error + } + type scopeResult struct { + scopes []domain.StorageBucketScope + err error + } + type bucketResult struct { + bucketSummary *gintegrationsyfon.ProjectBucketSummary + bucketObjects []gintegrationsyfon.ProjectBucketObject + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + err error + } + + inventoryCh := make(chan inventoryResult, 1) + recordCh := make(chan recordResult, 1) + scopeCh := make(chan scopeResult, 1) + bucketCh := make(chan bucketResult, 1) + + go func() { + start := time.Now() + timings.StageStart("repo_index") + inventory, err := service.loadStorageChainInventory(ctx, ref, gitSubpath, mirrorPath, repo, hash) + timings.Record("repo_index", time.Since(start)) + logStorageChainInputResult("repo_index", len(inventory), err) + inventoryCh <- inventoryResult{inventory: inventory, err: err} + }() + go func() { + start := time.Now() + timings.StageStart("syfon_project_records") + recordSet, err := service.loadCachedProjectAuditRecordSet(ctx, authorizationHeader, organization, project) + timings.Record("syfon_project_records", time.Since(start)) + recordCount := 0 + if recordSet != nil { + recordCount = countRecordStates(recordSet.allProjectRecords) + } + logStorageChainInputResult("syfon_project_records", recordCount, err) + recordCh <- recordResult{recordSet: recordSet, err: err} + }() + go func() { + start := time.Now() + timings.StageStart("syfon_project_scopes") + scopes, err := service.loadCachedProjectChainScopeMappings(ctx, authorizationHeader, organization, project) + timings.Record("syfon_project_scopes", time.Since(start)) + logStorageChainInputResult("syfon_project_scopes", len(scopes), err) + scopeCh <- scopeResult{scopes: scopes, err: err} + }() + if bucketMode == StorageChainBucketModeValidate { + bucketCh <- bucketResult{ + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, + } + timings.Record("syfon_bucket_inventory_skipped", 0) + logStorageChainInputResult("syfon_bucket_inventory_skipped validate_mode", 0, nil) + } else { + go func() { + start := time.Now() + timings.StageStart("syfon_bucket_inventory") + bucketObjects, bucketObjectsByURL, err := service.loadCachedProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) + timings.Record("syfon_bucket_inventory", time.Since(start)) + logStorageChainInputResult("syfon_bucket_items", len(bucketObjects), err) + bucketCh <- bucketResult{bucketObjects: bucketObjects, bucketObjectsByURL: bucketObjectsByURL, err: err} + }() + } + + inventory := <-inventoryCh + recordSet := <-recordCh + scopes := <-scopeCh + bucketObjects := <-bucketCh + if inventory.err != nil { + return nil, inventory.err + } + if recordSet.err != nil { + return nil, recordSet.err + } + if scopes.err != nil { + return nil, scopes.err + } + return &storageChainInputs{ + inventory: inventory.inventory, + recordSet: recordSet.recordSet, + scopes: scopes.scopes, + bucketSummary: bucketObjects.bucketSummary, + bucketObjects: bucketObjects.bucketObjects, + bucketObjectsByURL: bucketObjects.bucketObjectsByURL, + bucketInventoryErr: bucketObjects.err, + }, nil +} + +func logStorageChainInputResult(stage string, count int, err error) { + if err != nil { + log.Printf("INFO: storage_chain_input_done stage=%s count=%d error=%q", strings.TrimSpace(stage), count, err.Error()) + return + } + log.Printf("INFO: storage_chain_input_done stage=%s count=%d", strings.TrimSpace(stage), count) +} + type storageFindingKind string const ( @@ -211,6 +327,28 @@ func (service *StorageAnalyticsService) loadProjectAuditRecordSet(ctx context.Co if err != nil { return nil, fmt.Errorf("list syfon project audit records: %w", err) } + return buildProjectAuditRecordSet(projectRecords), nil +} + +func (service *StorageAnalyticsService) loadCachedProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string) (*storageAuditRecordSet, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + service.chainInputMu.RLock() + cached, ok := service.chainInputCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && time.Now().Before(cached.expiresAt) && cached.projectRecords != nil { + return buildProjectAuditRecordSet(cached.projectRecords), nil + } + projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("list syfon project audit records: %w", err) + } + service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { + state.projectRecords = append([]gintegrationsyfon.ProjectRecord(nil), projectRecords...) + }) + return buildProjectAuditRecordSet(projectRecords), nil +} + +func buildProjectAuditRecordSet(projectRecords []gintegrationsyfon.ProjectRecord) *storageAuditRecordSet { recordsByChecksum := make(map[string][]projectRecordState) for _, record := range projectRecords { normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) @@ -226,21 +364,117 @@ func (service *StorageAnalyticsService) loadProjectAuditRecordSet(ctx context.Co return &storageAuditRecordSet{ recordsByChecksum: recordsByChecksum, allProjectRecords: allProjectRecords, - }, nil + } } -func (service *StorageAnalyticsService) loadProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { - bucketObjects, err := service.storage.ListProjectBucketObjects(ctx, authorizationHeader, organization, project) +func (service *StorageAnalyticsService) loadProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { + bucketObjects, err := service.storage.ListProjectBucketObjects(ctx, authorizationHeader, organization, project, bucketPathPrefix) if err != nil { return nil, nil, fmt.Errorf("list syfon project bucket objects: %w", err) } + objects, lookup := buildBucketObjectLookup(bucketObjects) + return objects, lookup, nil +} + +func (service *StorageAnalyticsService) loadCachedProjectChainScopeMappings(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + service.chainInputMu.RLock() + cached, ok := service.chainInputCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && time.Now().Before(cached.expiresAt) && cached.projectScopes != nil { + return append([]domain.StorageBucketScope(nil), cached.projectScopes...), nil + } + scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { + state.projectScopes = append([]domain.StorageBucketScope(nil), scopes...) + }) + return scopes, nil +} + +func (service *StorageAnalyticsService) loadCachedProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + "::bucket-items::" + normalizeRepoSubpath(bucketPathPrefix) + service.chainInputMu.RLock() + cached, ok := service.chainInputCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && time.Now().Before(cached.expiresAt) && cached.bucketObjects != nil && cached.bucketObjectsByURL != nil { + objects, lookup := cloneBucketInventory(cached.bucketObjects, cached.bucketObjectsByURL) + return objects, lookup, nil + } + bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) + if err != nil { + return nil, nil, err + } + service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { + state.bucketObjects, state.bucketObjectsByURL = cloneBucketInventory(bucketObjects, bucketObjectsByURL) + }) + objects, lookup := cloneBucketInventory(bucketObjects, bucketObjectsByURL) + return objects, lookup, nil +} + +func (service *StorageAnalyticsService) loadCachedProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + "::bucket-summary::" + strings.TrimSpace(mode) + service.chainInputMu.RLock() + cached, ok := service.chainInputCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && time.Now().Before(cached.expiresAt) && cached.bucketSummary != nil { + summary := *cached.bucketSummary + return &summary, nil + } + summary, err := service.storage.ListProjectBucketSummary(ctx, authorizationHeader, organization, project, mode) + if err != nil { + return nil, err + } + service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { + if summary == nil { + state.bucketSummary = nil + return + } + copy := *summary + state.bucketSummary = © + }) + if summary == nil { + return nil, nil + } + copy := *summary + return ©, nil +} + +func (service *StorageAnalyticsService) projectChainInputCacheKey(organization string, project string) string { + return strings.TrimSpace(organization) + "/" + strings.TrimSpace(project) +} + +func (service *StorageAnalyticsService) updateChainInputCache(cacheKey string, update func(*cachedChainInputState)) { + service.chainInputMu.Lock() + defer service.chainInputMu.Unlock() + state := service.chainInputCache[cacheKey] + if time.Now().After(state.expiresAt) { + state = cachedChainInputState{} + } + state.expiresAt = time.Now().Add(chainInputCacheTTL) + update(&state) + service.chainInputCache[cacheKey] = state +} + +func cloneBucketInventory(bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { + objects := append([]gintegrationsyfon.ProjectBucketObject(nil), bucketObjects...) + lookup := make(map[string]gintegrationsyfon.ProjectBucketObject, len(bucketObjectsByURL)) + for objectURL, item := range bucketObjectsByURL { + lookup[objectURL] = item + } + return objects, lookup +} + +func buildBucketObjectLookup(bucketObjects []gintegrationsyfon.ProjectBucketObject) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { bucketObjectsByURL := make(map[string]gintegrationsyfon.ProjectBucketObject, len(bucketObjects)) for _, item := range bucketObjects { if objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL); objectURL != "" { bucketObjectsByURL[objectURL] = item } } - return bucketObjects, bucketObjectsByURL, nil + return append([]gintegrationsyfon.ProjectBucketObject(nil), bucketObjects...), bucketObjectsByURL } func synthesizeBucketInventoryFromProbes(allProjectRecords map[string][]projectRecordState) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { @@ -310,6 +544,17 @@ func (service *StorageAnalyticsService) attachProjectStorageProbes(ctx context.C }, nil } +func (service *StorageAnalyticsService) attachProjectStorageListValidations(ctx context.Context, authorizationHeader string, recordSet *storageAuditRecordSet) (*storageAuditRecordSet, error) { + recordsByChecksum, allProjectRecords, err := service.attachStorageListValidations(ctx, authorizationHeader, recordSet.recordsByChecksum, recordSet.allProjectRecords) + if err != nil { + return nil, err + } + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: allProjectRecords, + }, nil +} + func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet) (*storageAuditStorageView, error) { scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) if err != nil { @@ -324,7 +569,7 @@ func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, bucketInventoryAvailable: true, } - bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project) + bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project, "") if err != nil { if !shouldDegradeBucketInventory(err) { return nil, err @@ -356,6 +601,58 @@ func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context return view, nil } +func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, probeMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { + recordSet = applyScopeCanonicalization(recordSet, scopes) + view := &storageAuditStorageView{ + scopes: append([]domain.StorageBucketScope(nil), scopes...), + recordsByChecksum: recordSet.recordsByChecksum, + allProjectRecords: recordSet.allProjectRecords, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, + bucketInventoryAvailable: true, + } + if bucketMode == StorageChainBucketModeValidate { + if bucketInventoryErr != nil { + view.bucketInventoryAvailable = false + view.bucketInventoryError = strings.TrimSpace(bucketInventoryErr.Error()) + } + probeStart := time.Now() + probedRecordSet, probeErr := service.attachProjectStorageListValidations(ctx, authorizationHeader, recordSet) + timings.Record("bulk_list_validation", time.Since(probeStart)) + if probeErr != nil { + return nil, probeErr + } + view.recordsByChecksum = probedRecordSet.recordsByChecksum + view.allProjectRecords = probedRecordSet.allProjectRecords + view.bucketObjects, view.bucketObjectsByURL = synthesizeBucketInventoryFromProbes(probedRecordSet.allProjectRecords) + return view, nil + } + if bucketInventoryErr != nil { + return nil, bucketInventoryErr + } + view.bucketObjects, view.bucketObjectsByURL = cloneBucketInventory(bucketObjects, bucketObjectsByURL) + if probeMode == StorageChainProbeModeInventoryOnly { + return view, nil + } + + probeSelectStart := time.Now() + probeCandidates := selectTargetedProbeRecordSet(recordSet, view.bucketObjectsByURL) + timings.Record("targeted_probe_selection", time.Since(probeSelectStart)) + if probeCandidates == nil { + return view, nil + } + probeStart := time.Now() + probedSubset, probeErr := service.attachProjectStorageProbes(ctx, authorizationHeader, probeCandidates) + timings.Record("bulk_probe", time.Since(probeStart)) + if probeErr != nil { + return nil, probeErr + } + merged := mergeRecordSetProbes(recordSet, probedSubset) + view.recordsByChecksum = merged.recordsByChecksum + view.allProjectRecords = merged.allProjectRecords + return view, nil +} + func (service *StorageAnalyticsService) loadStorageAuditStorageView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, includeBucketInventory bool, includeProbes bool) (*storageAuditStorageView, error) { scopes, err := service.loadProjectScopeMappings(ctx, authorizationHeader, organization, project) if err != nil { @@ -377,7 +674,7 @@ func (service *StorageAnalyticsService) loadStorageAuditStorageView(ctx context. bucketInventoryAvailable: includeBucketInventory, } if includeBucketInventory { - bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project) + bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project, "") if err != nil { if !includeProbes || !shouldDegradeBucketInventory(err) { return nil, err @@ -537,14 +834,16 @@ func buildStorageChainIndex(inventory []RepoInventoryFile, allProjectRecords map } } -func buildStorageChainAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) *chainAuditModel { +func buildStorageChainAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, includeBucketOrigin bool) *chainAuditModel { index := buildStorageChainIndex(inventory, allProjectRecords, bucketObjectsByURL) acc := chainAuditAccumulator{ findings: make([]GitStorageChainFinding, 0), summary: newChainSummary(len(bucketObjectsByURL), len(index.allRecords), len(inventory)), } - buildBucketOriginChainFindings(index, &acc) - buildSyfonOriginChainFindings(index, &acc) + if includeBucketOrigin { + buildBucketOriginChainFindings(index, &acc) + } + buildSyfonOriginChainFindings(index, &acc, !includeBucketOrigin) buildGitOriginChainFindings(index, recordsByChecksum, allProjectRecords, &acc) return finalizeChainFindings(gitSubpath, acc) } @@ -561,11 +860,37 @@ func buildBucketOriginChainFindings(index storageChainIndex, acc *chainAuditAccu acc.addCount("bucket_syfon_git_complete", 1) continue } + if bucketObjectHasEquivalentSyfonRecord(item, index.allRecords) { + continue + } acc.add("bucket_only_object", buildChainBucketOnlyFinding(item)) } } -func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccumulator) { +func bucketObjectHasEquivalentSyfonRecord(item gintegrationsyfon.ProjectBucketObject, records []projectRecordState) bool { + itemName := strings.TrimSpace(path.Base(strings.TrimSpace(item.Key))) + if itemName == "" { + return false + } + for _, record := range records { + for _, accessURL := range accessURLsForStorage(record) { + _, key, ok := parseStorageURL(accessURL) + if !ok { + continue + } + if strings.TrimSpace(path.Base(key)) != itemName { + continue + } + if record.Size > 0 && item.SizeBytes > 0 && record.Size != item.SizeBytes { + continue + } + return true + } + } + return false +} + +func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccumulator, countCompleteFromSyfon bool) { for _, record := range index.allRecords { gitPaths := uniqueStrings(index.repoPathsByChecksum[normalizeAnalyticsChecksum(record.Checksum)]) bucketMatches := matchedBucketObjectURLs(record, index.bucketObjectsByURL) @@ -589,14 +914,18 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum acc.addCount("git_syfon_metadata_mismatch", len(gitPaths)) continue } - acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) + acc.add("bucket_syfon_no_git", buildChainRecordFindingsWithOptions("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.", countCompleteFromSyfon)...) case storageFindingBrokenAccessURL, storageFindingProbeError: findings := buildChainRecordFindings("probe_error", record, gitPaths, bucketMatches, "Bucket verification failed before Gecko could classify this record cleanly.") acc.findings = append(acc.findings, findings...) acc.addCount("probe_error", chainPathCount(gitPaths)) case storageFindingNone: + if countCompleteFromSyfon && len(gitPaths) > 0 && len(bucketMatches) > 0 { + acc.addCount("bucket_syfon_git_complete", len(gitPaths)) + continue + } if len(gitPaths) == 0 && len(bucketMatches) > 0 { - acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) + acc.add("bucket_syfon_no_git", buildChainRecordFindingsWithOptions("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.", countCompleteFromSyfon)...) } } } diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index c4c9d00..606dfe6 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -18,26 +18,32 @@ import ( ) type fakeStorageAnalyticsBackend struct { - projectRecords []gintegrationsyfon.ProjectRecord - bulkRecords map[string][]gintegrationsyfon.ProjectRecord - buckets map[string]domain.StorageBucket - bucketScopes map[string][]domain.StorageBucketScope - projectScopes []domain.StorageBucketScope - usageByObject map[string]gintegrationsyfon.FileUsage - probeResults map[string]gintegrationsyfon.BulkStorageProbeResult - bucketObjects []gintegrationsyfon.ProjectBucketObject - listProjectBucketObjectsErr error - listBucketsCalls int - listBucketScopesCalls int - listProjectAuditRecordsCalls int - listProjectScopesCalls int - listProjectFileUsageCalls int - listProjectBucketObjectsCalls int - probeCalls int - probeItems []gintegrationsyfon.BulkStorageProbeItem - deletedIDs []string - updatedAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod - deletedBucketObjects []string + projectRecords []gintegrationsyfon.ProjectRecord + bulkRecords map[string][]gintegrationsyfon.ProjectRecord + buckets map[string]domain.StorageBucket + bucketScopes map[string][]domain.StorageBucketScope + projectScopes []domain.StorageBucketScope + usageByObject map[string]gintegrationsyfon.FileUsage + probeResults map[string]gintegrationsyfon.BulkStorageProbeResult + listProbeResults map[string]gintegrationsyfon.BulkStorageProbeResult + bucketObjects []gintegrationsyfon.ProjectBucketObject + listProjectBucketObjectsErr error + listBucketsCalls int + listBucketScopesCalls int + listProjectAuditRecordsCalls int + listProjectScopesCalls int + listProjectFileUsageCalls int + listProjectBucketObjectsCalls int + listProjectBucketObjectsPathPrefix string + listProjectBucketSummaryCalls int + listProjectBucketSummaryMode string + probeCalls int + probeItems []gintegrationsyfon.BulkStorageProbeItem + listProbeCalls int + listProbeItems []gintegrationsyfon.BulkStorageProbeItem + deletedIDs []string + updatedAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod + deletedBucketObjects []string } func (fake *fakeStorageAnalyticsBackend) ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) { @@ -109,14 +115,34 @@ func (fake *fakeStorageAnalyticsBackend) ListProjectFileUsage(ctx context.Contex return out, nil } -func (fake *fakeStorageAnalyticsBackend) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectBucketObject, error) { +func (fake *fakeStorageAnalyticsBackend) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) { fake.listProjectBucketObjectsCalls++ + fake.listProjectBucketObjectsPathPrefix = pathPrefix if fake.listProjectBucketObjectsErr != nil { return nil, fake.listProjectBucketObjectsErr } return append([]gintegrationsyfon.ProjectBucketObject(nil), fake.bucketObjects...), nil } +func (fake *fakeStorageAnalyticsBackend) ListProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) { + fake.listProjectBucketSummaryCalls++ + fake.listProjectBucketSummaryMode = mode + var totalBytes int64 + for _, item := range fake.bucketObjects { + totalBytes += item.SizeBytes + } + return &gintegrationsyfon.ProjectBucketSummary{ + Provider: "s3", + Bucket: "bucket", + Prefix: organization + "/" + project, + ObjectURL: "s3://bucket/" + organization + "/" + project, + Exists: len(fake.bucketObjects) > 0, + ObjectCount: len(fake.bucketObjects), + TotalBytes: totalBytes, + Mode: mode, + }, nil +} + func (fake *fakeStorageAnalyticsBackend) BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) { fake.probeCalls++ fake.probeItems = append(fake.probeItems, items...) @@ -140,6 +166,30 @@ func (fake *fakeStorageAnalyticsBackend) BulkProbeStorageObjects(ctx context.Con return out, nil } +func (fake *fakeStorageAnalyticsBackend) BulkListStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) { + fake.listProbeCalls++ + fake.listProbeItems = append(fake.listProbeItems, items...) + out := make([]gintegrationsyfon.BulkStorageProbeResult, 0, len(items)) + for _, item := range items { + if result, ok := fake.listProbeResults[item.ID]; ok { + out = append(out, result) + continue + } + exists := true + out = append(out, gintegrationsyfon.BulkStorageProbeResult{ + ID: item.ID, + ObjectURL: item.ObjectURL, + Exists: exists, + Status: "present", + ValidationStatus: "matched", + SizeBytes: item.ExpectedSizeBytes, + SizeMatch: ptrBool(true), + NameMatch: ptrBool(true), + }) + } + return out, nil +} + func (fake *fakeStorageAnalyticsBackend) BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error { fake.deletedIDs = append(fake.deletedIDs, objectIDs...) return nil @@ -743,6 +793,213 @@ func TestBuildStorageChainAuditUsesProjectAuditSourcesAndTargetsProbes(t *testin } } +func TestBuildStorageChainAuditInventoryOnlySkipsTargetedProbes(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/present"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present", Bucket: "bucket", Key: "present", Path: "present", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly}, + ) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if backend.probeCalls != 0 || len(backend.probeItems) != 0 { + t.Fatalf("expected inventory-only audit to skip targeted probes, got calls=%d items=%+v", backend.probeCalls, backend.probeItems) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected inventory-derived complete chain, got summary %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/mismatch.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Name: "present.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/present.txt"}}, + {ObjectID: "obj-mismatch", Name: "expected.txt", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, AccessURLs: []string{"s3://bucket/mismatch.txt"}}, + {ObjectID: "obj-syfon-only", Name: "syfon-only.txt", Checksum: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Organization: "org", Project: "proj", Size: 300, AccessURLs: []string{"s3://bucket/syfon-only.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present.txt", Bucket: "bucket", Key: "present.txt", Path: "present.txt", SizeBytes: 100}, + }, + listProbeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageListValidationRequestKey("s3://bucket/mismatch.txt", 200, "expected.txt"): { + ID: storageListValidationRequestKey("s3://bucket/mismatch.txt", 200, "expected.txt"), + ObjectURL: "s3://bucket/mismatch.txt", + Provider: "s3", + Bucket: "bucket", + Key: "mismatch.txt", + Path: "mismatch.txt", + Exists: true, + Status: "present", + ValidationStatus: "mismatched", + SizeBytes: int64Ptr(200), + SizeMatch: ptrBool(true), + NameMatch: ptrBool(false), + ValidationMismatches: []string{"name"}, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build validate chain audit: %v", err) + } + if backend.listProjectBucketObjectsCalls != 0 { + t.Fatalf("expected validate mode to skip recursive bucket inventory, got %d calls", backend.listProjectBucketObjectsCalls) + } + if backend.listProjectBucketSummaryCalls != 0 { + t.Fatalf("expected validate mode to skip bucket summary preflight, got %d calls", backend.listProjectBucketSummaryCalls) + } + if backend.probeCalls != 0 { + t.Fatalf("expected validate mode to skip HEAD probes, got %d calls", backend.probeCalls) + } + if backend.listProbeCalls != 1 || len(backend.listProbeItems) != 3 { + t.Fatalf("expected three bulk LIST validation items, got calls=%d items=%+v", backend.listProbeCalls, backend.listProbeItems) + } + if backend.listProbeItems[0].ExpectedName == "" && backend.listProbeItems[1].ExpectedName == "" { + t.Fatalf("expected LIST validation items to include expected Syfon names, got %+v", backend.listProbeItems) + } + if chain.Summary.BucketPathExists != nil || chain.Summary.BucketSummaryMode != "" { + t.Fatalf("expected validate mode not to claim bucket prefix summary, got %+v", chain.Summary) + } + if chain.Summary.BucketObjectCount != 3 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { + t.Fatalf("expected validate mode to count validated objects without bucket-only inventory, got summary %+v", chain.Summary) + } + if chain.Summary.CountsByKind["bucket_syfon_git_complete"] != 1 { + t.Fatalf("expected validate mode to count clean Syfon/Git/bucket join, got summary %+v", chain.Summary) + } + if chain.Summary.CountsByKind["bucket_syfon_no_git"] != 1 { + t.Fatalf("expected validate mode to report Syfon-backed no-Git record, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["git_syfon_metadata_mismatch"]; got != 1 { + t.Fatalf("expected one LIST-derived metadata mismatch, got summary %+v", chain.Summary) + } + assertNoChainFinding(t, chain.Findings, "bucket_only_object") + assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/mismatch.txt") + assertHasChainFinding(t, chain.Findings, "bucket_syfon_no_git", "syfon-only.txt") +} + +func TestExpectedStorageObjectNameForListValidationSkipsContentAddressedKeys(t *testing.T) { + if got := expectedStorageObjectNameForListValidation("s3://cbds/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/ec4b068cb42b52449dd44052c3bfb2a459b00336a9cd42cd29c22ca1d1b26cb0", "CONFIG/cbds-BForePC.json"); got != "" { + t.Fatalf("expected content-addressed storage key to skip name validation, got %q", got) + } + if got := expectedStorageObjectNameForListValidation("s3://bucket/path/cbds-BForePC.json", "CONFIG/cbds-BForePC.json"); got != "cbds-BForePC.json" { + t.Fatalf("expected filename-backed storage key to validate basename, got %q", got) + } +} + +func TestBuildStorageChainAuditCachesProjectChainInputs(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/present"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present", Bucket: "bucket", Key: "present", Path: "present", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + options := StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly} + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build first chain audit: %v", err) + } + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build second chain audit: %v", err) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected cached project records, got %d calls", backend.listProjectAuditRecordsCalls) + } + if backend.listProjectScopesCalls != 1 { + t.Fatalf("expected cached project scopes, got %d calls", backend.listProjectScopesCalls) + } + if backend.listProjectBucketObjectsCalls != 1 { + t.Fatalf("expected cached project bucket inventory, got %d calls", backend.listProjectBucketObjectsCalls) + } +} + +func TestBuildStorageChainAuditForwardsBucketPathPrefixForExplicitInventory(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "README.md": "fixture", + }) + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/root/CONFIG/a.json", Bucket: "bucket", Key: "root/CONFIG/a.json", Path: "CONFIG/a.json", SizeBytes: 10}, + {ObjectURL: "s3://bucket/root/CONFIG/nested/b.json", Bucket: "bucket", Key: "root/CONFIG/nested/b.json", Path: "CONFIG/nested/b.json", SizeBytes: 15}, + }, + } + service := NewStorageAnalyticsService(backend) + response, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeItems, + BucketPathPrefix: "/CONFIG/", + ProbeMode: StorageChainProbeModeInventoryOnly, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if backend.listProjectBucketObjectsCalls != 1 { + t.Fatalf("expected one recursive bucket inventory call, got %d", backend.listProjectBucketObjectsCalls) + } + if backend.listProjectBucketObjectsPathPrefix != "CONFIG" { + t.Fatalf("expected bucket path prefix to be forwarded, got %q", backend.listProjectBucketObjectsPathPrefix) + } + if response.BucketPathPrefix != "CONFIG" { + t.Fatalf("expected response bucket path prefix, got %q", response.BucketPathPrefix) + } + if response.Summary.BucketObjectCount != 2 { + t.Fatalf("expected recursive LIST rows to drive bucket count, got %+v", response.Summary) + } +} + func TestBuildStorageChainAuditSurfacesMetadataMismatchAndEvidence(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -887,16 +1144,17 @@ func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) if err != nil { t.Fatalf("build chain audit: %v", err) } - if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { - t.Fatalf("expected stale raw scoped URL to surface as broken bucket mapping, got %+v", chain.Summary) + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected mapped bucket match to suppress stale raw URL misclassification, got %+v", chain.Summary) } - if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 0 { - t.Fatalf("expected stale raw scoped URL to block clean-chain count, got %+v", chain.Summary) + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected mapped bucket match to preserve clean-chain count, got %+v", chain.Summary) } findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) - finding := assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "data/slide.ome.tiff") - if finding.Evidence == nil || !contains(finding.Evidence.AccessURLs, "s3://bforepc-prod/OHSU/slide.ome.tiff") { - t.Fatalf("expected raw stale access URL in evidence, got %+v", finding) + for _, finding := range findings { + if finding.NormalizedPath == "data/slide.ome.tiff" { + t.Fatalf("did not expect stale raw URL to produce a chain finding once mapped object exists: %+v", finding) + } } if len(backend.probeItems) != 2 { t.Fatalf("expected raw and canonical probes, got %+v", backend.probeItems) @@ -906,6 +1164,86 @@ func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) } } +func TestBuildStorageChainAuditFlagsExactPathMismatchWhenHashExistsElsewhereInBucket(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "CONFIG/cbds-BForePC.json": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 23739), + }) + now := time.Date(2026, 7, 2, 12, 0, 0, 0, time.UTC) + objectHash := "ec4b068cb42b52449dd44052c3bfb2a459b00336a9cd42cd29c22ca1d1b26cb0" + rawURL := "s3://cbds/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/" + objectHash + canonicalURL := "s3://bforepc/bforepc-prod/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/" + objectHash + relocatedURL := "s3://bforepc/bforepc-prod/2532ab27-4961-57da-8bef-e9774093bf56/" + objectHash + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Name: "CONFIG/cbds-BForePC.json", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "HTAN_INT", + Project: "BForePC", + Size: 23739, + UpdatedAt: &now, + AccessURLs: []string{rawURL}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + { + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: relocatedURL, + Bucket: "bforepc", + Key: "bforepc-prod/2532ab27-4961-57da-8bef-e9774093bf56/" + objectHash, + Path: objectHash, + SizeBytes: 23739, + }, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey(rawURL, 23739, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey(rawURL, 23739, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: rawURL, + Bucket: "cbds", + Key: "0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/" + objectHash, + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey(canonicalURL, 23739, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey(canonicalURL, 23739, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: canonicalURL, + Bucket: "bforepc", + Key: "bforepc-prod/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/" + objectHash, + Status: "not_found", + Exists: false, + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "CONFIG", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { + t.Fatalf("expected exact mapped-key miss to surface as broken bucket mapping, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_only_object"]; got != 0 { + t.Fatalf("expected relocated hash not to be double-counted as bucket-only, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 0 { + t.Fatalf("expected exact mapped-key miss to block clean-chain count, got %+v", chain.Summary) + } + findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) + assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "CONFIG/cbds-BForePC.json") +} + func TestBuildStorageChainAuditNormalizesChecksumJoinKeys(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -975,7 +1313,7 @@ func TestBuildStorageChainAuditNormalizesChecksumJoinKeys(t *testing.T) { } } -func TestBuildStorageChainAuditDegradesWhenProjectBucketListDenied(t *testing.T) { +func TestBuildStorageChainAuditFailsWhenProjectBucketListDenied(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), }) @@ -1019,29 +1357,18 @@ func TestBuildStorageChainAuditDegradesWhenProjectBucketListDenied(t *testing.T) } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) - if err != nil { - t.Fatalf("build chain audit: %v", err) + _, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err == nil { + t.Fatal("expected bucket inventory failure to hard-error chain audit") } - if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { - t.Fatalf("expected one connected chain when bucket inventory degrades to probes, got %+v", chain.Summary) - } - if chain.Summary.BucketInventoryAvailable { - t.Fatalf("expected bucket inventory to be marked unavailable, got %+v", chain.Summary) - } - if !strings.Contains(chain.Summary.BucketInventoryError, "mapped bucket target may be missing or inaccessible") { - t.Fatalf("expected bucket inventory error detail, got %+v", chain.Summary) - } - for _, finding := range loadAllChainFindings(t, service, "org", "proj", chain) { - if finding.Kind == "git_only_no_syfon" { - t.Fatalf("did not expect git-only miss after degraded inventory fallback: %+v", finding) - } + if !strings.Contains(err.Error(), "mapped bucket target may be missing or inaccessible") { + t.Fatalf("expected bucket inventory error detail, got %v", err) } if backend.listProjectBucketObjectsCalls != 1 { t.Fatalf("expected one bucket inventory attempt, got %d", backend.listProjectBucketObjectsCalls) } - if backend.probeCalls != 1 { - t.Fatalf("expected probe fallback to run once, got %d", backend.probeCalls) + if backend.probeCalls != 0 { + t.Fatalf("expected no probe fallback after bucket inventory failure, got %d", backend.probeCalls) } } @@ -1272,6 +1599,16 @@ func ptrTime(value time.Time) *time.Time { return ©Value } +func ptrBool(value bool) *bool { + copyValue := value + return ©Value +} + +func int64Ptr(value int64) *int64 { + copyValue := value + return ©Value +} + func contains(values []string, target string) bool { for _, value := range values { if value == target { @@ -1332,3 +1669,12 @@ func assertHasChainFinding(t *testing.T, findings []GitStorageChainFinding, kind t.Fatalf("missing chain finding kind=%s path=%s in %+v", kind, path, findings) return GitStorageChainFinding{} } + +func assertNoChainFinding(t *testing.T, findings []GitStorageChainFinding, kind string) { + t.Helper() + for _, finding := range findings { + if finding.Kind == kind { + t.Fatalf("unexpected chain finding kind=%s in %+v", kind, findings) + } + } +} diff --git a/internal/git/storage_analytics_timing.go b/internal/git/storage_analytics_timing.go new file mode 100644 index 0000000..529c22f --- /dev/null +++ b/internal/git/storage_analytics_timing.go @@ -0,0 +1,93 @@ +package git + +import ( + "strings" + "sync" + "time" +) + +const ( + StorageChainProbeModeFull = "full" + StorageChainProbeModeInventoryOnly = "inventory_only" + + StorageChainBucketModeItems = "items" + StorageChainBucketModeValidate = "validate" +) + +type StorageChainAuditOptions struct { + ProbeMode string + BucketInventoryMode string + BucketPathPrefix string + FindingLimit int + Timings *StorageChainAuditTimings +} + +type StorageChainAuditStageTiming struct { + Stage string + Duration time.Duration +} + +type StorageChainAuditTimings struct { + mu sync.Mutex + stages []StorageChainAuditStageTiming + Logf func(format string, args ...any) + DebugPrefix string +} + +func (timings *StorageChainAuditTimings) StageStart(stage string) { + if timings == nil || timings.Logf == nil { + return + } + timings.Logf("storage_chain_audit_stage_start %s stage=%s", strings.TrimSpace(timings.DebugPrefix), strings.TrimSpace(stage)) +} + +func (timings *StorageChainAuditTimings) Record(stage string, duration time.Duration) { + if timings == nil { + return + } + normalizedStage := strings.TrimSpace(stage) + timings.mu.Lock() + timings.stages = append(timings.stages, StorageChainAuditStageTiming{ + Stage: normalizedStage, + Duration: duration, + }) + logf := timings.Logf + prefix := strings.TrimSpace(timings.DebugPrefix) + timings.mu.Unlock() + if logf != nil { + logf("storage_chain_audit_stage_done %s stage=%s duration_ms=%d", prefix, normalizedStage, duration.Milliseconds()) + } +} + +func (timings *StorageChainAuditTimings) Snapshot() []StorageChainAuditStageTiming { + if timings == nil { + return nil + } + timings.mu.Lock() + defer timings.mu.Unlock() + out := make([]StorageChainAuditStageTiming, len(timings.stages)) + copy(out, timings.stages) + return out +} + +func NormalizeStorageChainProbeMode(raw string) (string, bool) { + switch strings.TrimSpace(raw) { + case "", StorageChainProbeModeFull: + return StorageChainProbeModeFull, true + case StorageChainProbeModeInventoryOnly: + return StorageChainProbeModeInventoryOnly, true + default: + return "", false + } +} + +func NormalizeStorageChainBucketInventoryMode(raw string) (string, bool) { + switch strings.TrimSpace(raw) { + case "", StorageChainBucketModeItems: + return StorageChainBucketModeItems, true + case StorageChainBucketModeValidate: + return StorageChainBucketModeValidate, true + default: + return "", false + } +} diff --git a/internal/git/types.go b/internal/git/types.go index e5e31fd..4cc5103 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -391,8 +391,12 @@ type GitStorageCleanupAuditRequest struct { } type GitStorageChainAuditRequest struct { - GitSubpath string `json:"git_subpath,omitempty"` - Ref string `json:"ref,omitempty"` + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + ProbeMode string `json:"probe_mode,omitempty"` + BucketInventoryMode string `json:"bucket_inventory_mode,omitempty"` + BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` + FindingLimit int `json:"finding_limit,omitempty"` } type GitStorageChainFinding struct { @@ -417,9 +421,15 @@ type GitStorageChainFinding struct { type GitStorageChainAuditSummary struct { CountsByKind map[string]int `json:"counts_by_kind"` TotalFindings int `json:"total_findings"` + ReturnedFindings int `json:"returned_findings"` + FindingsTruncated bool `json:"findings_truncated"` + FindingLimit int `json:"finding_limit,omitempty"` BucketObjectCount int `json:"bucket_object_count"` SyfonRecordCount int `json:"syfon_record_count"` GitTrackedFileCount int `json:"git_tracked_file_count"` + BucketPathExists *bool `json:"bucket_path_exists,omitempty"` + BucketPathObjectURL string `json:"bucket_path_object_url,omitempty"` + BucketSummaryMode string `json:"bucket_summary_mode,omitempty"` BucketInventoryAvailable bool `json:"bucket_inventory_available"` BucketInventoryError string `json:"bucket_inventory_error,omitempty"` } @@ -434,10 +444,11 @@ type GitStorageChainIssueGroup struct { } type GitStorageChainAuditResponse struct { - Findings []GitStorageChainFinding `json:"findings"` - Groups []GitStorageChainIssueGroup `json:"groups,omitempty"` - Summary GitStorageChainAuditSummary `json:"summary"` - PathPrefix string `json:"path_prefix"` + Findings []GitStorageChainFinding `json:"findings"` + Groups []GitStorageChainIssueGroup `json:"groups,omitempty"` + Summary GitStorageChainAuditSummary `json:"summary"` + PathPrefix string `json:"path_prefix"` + BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` } type GitStorageCleanupAccessProbe struct { @@ -456,6 +467,7 @@ type GitStorageCleanupAccessProbe struct { LastModified string `json:"last_modified,omitempty"` ValidationStatus string `json:"validation_status,omitempty"` SizeMatch *bool `json:"size_match,omitempty"` + NameMatch *bool `json:"name_match,omitempty"` SHA256Match *bool `json:"sha256_match,omitempty"` ValidationMismatches []string `json:"validation_mismatches,omitempty"` } diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index d89a172..6c77346 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -5,12 +5,14 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" "net/url" "path" "sort" "strconv" "strings" + "sync" "time" "github.com/calypr/gecko/internal/git/domain" @@ -23,6 +25,7 @@ import ( const refreshAuthzHeader = "X-Syfon-Refresh-Authz" const bulkStorageProbeBatchSize = 200 +const bulkStorageProbeConcurrency = 4 type Manager struct { baseURL string @@ -31,6 +34,7 @@ type Manager struct { type ProjectRecord struct { ObjectID string + Name string Checksum string Organization string Project string @@ -53,6 +57,7 @@ type BulkStorageProbeItem struct { ObjectURL string ExpectedSizeBytes *int64 ExpectedSHA256 string + ExpectedName string } type BulkStorageProbeResult struct { @@ -72,10 +77,23 @@ type BulkStorageProbeResult struct { LastModified string ValidationStatus string SizeMatch *bool + NameMatch *bool SHA256Match *bool ValidationMismatches []string } +type ProjectBucketSummary struct { + ObjectURL string + Provider string + Bucket string + Prefix string + Exists bool + ObjectCount int + TotalBytes int64 + ComputedAt string + Mode string +} + type ProjectBucketObject struct { ObjectURL string Provider string @@ -293,6 +311,7 @@ func (manager *Manager) ListProjectAuditRecords(ctx context.Context, authorizati var response struct { Items []struct { ObjectID string `json:"object_id"` + Name string `json:"name"` Checksum string `json:"checksum"` Organization string `json:"organization"` Project string `json:"project"` @@ -334,6 +353,7 @@ func (manager *Manager) ListProjectAuditRecords(ctx context.Context, authorizati } out = append(out, ProjectRecord{ ObjectID: strings.TrimSpace(item.ObjectID), + Name: strings.TrimSpace(item.Name), Checksum: checksum, Organization: strings.TrimSpace(item.Organization), Project: strings.TrimSpace(item.Project), @@ -588,99 +608,218 @@ func (manager *Manager) BulkUpdateAccessMethods(ctx context.Context, authorizati } func (manager *Manager) BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []BulkStorageProbeItem) ([]BulkStorageProbeResult, error) { + return manager.bulkStorageObjectRequest(ctx, authorizationHeader, "/data/inspect/bulk", items, false) +} + +func (manager *Manager) bulkStorageObjectRequest(ctx context.Context, authorizationHeader string, requestPath string, items []BulkStorageProbeItem, includeExpectedName bool) ([]BulkStorageProbeResult, error) { if len(items) == 0 { return []BulkStorageProbeResult{}, nil } - out := make([]BulkStorageProbeResult, 0, len(items)) + started := time.Now() + batchCount := (len(items) + bulkStorageProbeBatchSize - 1) / bulkStorageProbeBatchSize + log.Printf("INFO: syfon_bulk_storage_request_start path=%s items=%d batch_size=%d batches=%d concurrency=%d include_expected_name=%t", requestPath, len(items), bulkStorageProbeBatchSize, batchCount, bulkStorageProbeConcurrency, includeExpectedName) + resultsByID := make(map[string]BulkStorageProbeResult, len(items)) + var mu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, bulkStorageProbeConcurrency) + var firstErr error for start := 0; start < len(items); start += bulkStorageProbeBatchSize { end := start + bulkStorageProbeBatchSize if end > len(items) { end = len(items) } - requestBody := struct { - Items []struct { - ID string `json:"id,omitempty"` - ObjectURL string `json:"object_url,omitempty"` - ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` - ExpectedSHA256 string `json:"expected_sha256,omitempty"` - } `json:"items"` - }{Items: make([]struct { + batchStart := start + batch := append([]BulkStorageProbeItem(nil), items[start:end]...) + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + batchStarted := time.Now() + results, err := manager.probeStorageObjectBatch(ctx, authorizationHeader, requestPath, batch, includeExpectedName) + batchMs := time.Since(batchStarted).Milliseconds() + mu.Lock() + defer mu.Unlock() + if err != nil { + log.Printf("INFO: syfon_bulk_storage_request_batch path=%s batch_start=%d batch_items=%d duration_ms=%d error=%q", requestPath, batchStart, len(batch), batchMs, err.Error()) + if firstErr == nil { + firstErr = fmt.Errorf("bulk syfon storage object request %s batch starting at %d: %w", requestPath, batchStart, err) + } + return + } + for _, result := range results { + resultsByID[strings.TrimSpace(result.ID)] = result + } + log.Printf("INFO: syfon_bulk_storage_request_batch path=%s batch_start=%d batch_items=%d result_items=%d duration_ms=%d", requestPath, batchStart, len(batch), len(results), batchMs) + }() + } + wg.Wait() + if firstErr != nil { + log.Printf("INFO: syfon_bulk_storage_request_done path=%s items=%d results=%d batches=%d duration_ms=%d error=%q", requestPath, len(items), len(resultsByID), batchCount, time.Since(started).Milliseconds(), firstErr.Error()) + return nil, firstErr + } + out := make([]BulkStorageProbeResult, 0, len(resultsByID)) + for _, item := range items { + if result, ok := resultsByID[strings.TrimSpace(item.ID)]; ok { + out = append(out, result) + } + } + log.Printf("INFO: syfon_bulk_storage_request_done path=%s items=%d results=%d batches=%d duration_ms=%d", requestPath, len(items), len(out), batchCount, time.Since(started).Milliseconds()) + return out, nil +} + +func (manager *Manager) probeStorageObjectBatch(ctx context.Context, authorizationHeader string, requestPath string, items []BulkStorageProbeItem, includeExpectedName bool) ([]BulkStorageProbeResult, error) { + requestBody := struct { + Items []struct { ID string `json:"id,omitempty"` ObjectURL string `json:"object_url,omitempty"` ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` ExpectedSHA256 string `json:"expected_sha256,omitempty"` - }, 0, end-start)} - for _, item := range items[start:end] { - requestBody.Items = append(requestBody.Items, struct { - ID string `json:"id,omitempty"` - ObjectURL string `json:"object_url,omitempty"` - ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` - ExpectedSHA256 string `json:"expected_sha256,omitempty"` - }{ - ID: strings.TrimSpace(item.ID), - ObjectURL: strings.TrimSpace(item.ObjectURL), - ExpectedSizeBytes: item.ExpectedSizeBytes, - ExpectedSHA256: strings.TrimSpace(item.ExpectedSHA256), - }) + ExpectedName string `json:"expected_name,omitempty"` + } `json:"items"` + }{Items: make([]struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` + ExpectedName string `json:"expected_name,omitempty"` + }, 0, len(items))} + for _, item := range items { + row := struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` + ExpectedName string `json:"expected_name,omitempty"` + }{ + ID: strings.TrimSpace(item.ID), + ObjectURL: strings.TrimSpace(item.ObjectURL), + ExpectedSizeBytes: item.ExpectedSizeBytes, + ExpectedSHA256: strings.TrimSpace(item.ExpectedSHA256), } - var response struct { - Items []struct { - ID string `json:"id"` - ObjectURL string `json:"object_url"` - Provider string `json:"provider"` - Bucket string `json:"bucket"` - Key string `json:"key"` - Path string `json:"path"` - Exists bool `json:"exists"` - Status string `json:"status"` - Error string `json:"error"` - ErrorKind string `json:"error_kind"` - SizeBytes *int64 `json:"size_bytes"` - MetaSHA256 string `json:"meta_sha256"` - ETag string `json:"etag"` - LastModified string `json:"last_modified"` - ValidationStatus string `json:"validation_status"` - SizeMatch *bool `json:"size_match"` - SHA256Match *bool `json:"sha256_match"` - ValidationMismatches []string `json:"validation_mismatches"` - } `json:"items"` - } - if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/bulk", nil, requestBody, &response); err != nil { - return nil, fmt.Errorf("bulk probe syfon storage objects: %w", err) - } - for _, item := range response.Items { - out = append(out, BulkStorageProbeResult{ - ID: strings.TrimSpace(item.ID), - ObjectURL: strings.TrimSpace(item.ObjectURL), - Provider: strings.TrimSpace(item.Provider), - Bucket: strings.TrimSpace(item.Bucket), - Key: strings.TrimSpace(item.Key), - Path: strings.TrimSpace(item.Path), - Exists: item.Exists, - Status: strings.TrimSpace(item.Status), - Error: strings.TrimSpace(item.Error), - ErrorKind: strings.TrimSpace(item.ErrorKind), - SizeBytes: item.SizeBytes, - MetaSHA256: strings.TrimSpace(item.MetaSHA256), - ETag: strings.TrimSpace(item.ETag), - LastModified: strings.TrimSpace(item.LastModified), - ValidationStatus: strings.TrimSpace(item.ValidationStatus), - SizeMatch: item.SizeMatch, - SHA256Match: item.SHA256Match, - ValidationMismatches: append([]string(nil), item.ValidationMismatches...), - }) + if includeExpectedName { + row.ExpectedName = strings.TrimSpace(item.ExpectedName) } + requestBody.Items = append(requestBody.Items, row) + } + var response struct { + Items []struct { + ID string `json:"id"` + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Path string `json:"path"` + Exists bool `json:"exists"` + Status string `json:"status"` + Error string `json:"error"` + ErrorKind string `json:"error_kind"` + SizeBytes *int64 `json:"size_bytes"` + MetaSHA256 string `json:"meta_sha256"` + ETag string `json:"etag"` + LastModified string `json:"last_modified"` + ValidationStatus string `json:"validation_status"` + SizeMatch *bool `json:"size_match"` + NameMatch *bool `json:"name_match"` + SHA256Match *bool `json:"sha256_match"` + ValidationMismatches []string `json:"validation_mismatches"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, requestPath, nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("bulk syfon storage object request %s: %w", requestPath, err) + } + out := make([]BulkStorageProbeResult, 0, len(response.Items)) + for _, item := range response.Items { + out = append(out, BulkStorageProbeResult{ + ID: strings.TrimSpace(item.ID), + ObjectURL: strings.TrimSpace(item.ObjectURL), + Provider: strings.TrimSpace(item.Provider), + Bucket: strings.TrimSpace(item.Bucket), + Key: strings.TrimSpace(item.Key), + Path: strings.TrimSpace(item.Path), + Exists: item.Exists, + Status: strings.TrimSpace(item.Status), + Error: strings.TrimSpace(item.Error), + ErrorKind: strings.TrimSpace(item.ErrorKind), + SizeBytes: item.SizeBytes, + MetaSHA256: strings.TrimSpace(item.MetaSHA256), + ETag: strings.TrimSpace(item.ETag), + LastModified: strings.TrimSpace(item.LastModified), + ValidationStatus: strings.TrimSpace(item.ValidationStatus), + SizeMatch: item.SizeMatch, + NameMatch: item.NameMatch, + SHA256Match: item.SHA256Match, + ValidationMismatches: append([]string(nil), item.ValidationMismatches...), + }) } return out, nil } -func (manager *Manager) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string) ([]ProjectBucketObject, error) { +func (manager *Manager) BulkListStorageObjects(ctx context.Context, authorizationHeader string, items []BulkStorageProbeItem) ([]BulkStorageProbeResult, error) { + return manager.bulkStorageObjectRequest(ctx, authorizationHeader, "/data/inspect/bulk-list", items, true) +} + +func (manager *Manager) ListProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*ProjectBucketSummary, error) { + started := time.Now() + trimmedOrg := strings.TrimSpace(organization) + trimmedProject := strings.TrimSpace(project) + trimmedMode := strings.TrimSpace(mode) + log.Printf("INFO: syfon_project_bucket_summary_request_start organization=%s project=%s mode=%s", trimmedOrg, trimmedProject, trimmedMode) + requestBody := struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + Mode string `json:"mode,omitempty"` + }{ + Organization: trimmedOrg, + Project: trimmedProject, + Mode: trimmedMode, + } + var response struct { + Summary *struct { + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + Exists bool `json:"exists"` + ObjectCount int `json:"object_count"` + TotalBytes int64 `json:"total_bytes"` + ComputedAt string `json:"computed_at"` + Mode string `json:"mode"` + } `json:"summary"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-bucket", nil, requestBody, &response); err != nil { + log.Printf("INFO: syfon_project_bucket_summary_request_done organization=%s project=%s mode=%s duration_ms=%d error=%q", trimmedOrg, trimmedProject, trimmedMode, time.Since(started).Milliseconds(), err.Error()) + return nil, fmt.Errorf("inspect syfon project bucket summary: %w", err) + } + if response.Summary == nil { + log.Printf("INFO: syfon_project_bucket_summary_request_done organization=%s project=%s mode=%s duration_ms=%d error=%q", trimmedOrg, trimmedProject, trimmedMode, time.Since(started).Milliseconds(), "response summary is missing") + return nil, fmt.Errorf("inspect syfon project bucket summary: response summary is missing") + } + log.Printf("INFO: syfon_project_bucket_summary_request_done organization=%s project=%s mode=%s exists=%t object_count=%d total_bytes=%d duration_ms=%d", trimmedOrg, trimmedProject, trimmedMode, response.Summary.Exists, response.Summary.ObjectCount, response.Summary.TotalBytes, time.Since(started).Milliseconds()) + return &ProjectBucketSummary{ + ObjectURL: strings.TrimSpace(response.Summary.ObjectURL), + Provider: strings.TrimSpace(response.Summary.Provider), + Bucket: strings.TrimSpace(response.Summary.Bucket), + Prefix: strings.TrimSpace(response.Summary.Prefix), + Exists: response.Summary.Exists, + ObjectCount: response.Summary.ObjectCount, + TotalBytes: response.Summary.TotalBytes, + ComputedAt: strings.TrimSpace(response.Summary.ComputedAt), + Mode: strings.TrimSpace(response.Summary.Mode), + }, nil +} + +func (manager *Manager) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]ProjectBucketObject, error) { requestBody := struct { Organization string `json:"organization,omitempty"` Project string `json:"project,omitempty"` + Mode string `json:"mode,omitempty"` + PathPrefix string `json:"path_prefix,omitempty"` }{ Organization: strings.TrimSpace(organization), Project: strings.TrimSpace(project), + Mode: "items", + PathPrefix: strings.Trim(strings.TrimSpace(pathPrefix), "/"), } var response struct { Items []struct { diff --git a/internal/integrations/syfon/adapter_test.go b/internal/integrations/syfon/adapter_test.go index c2c4394..4aa41cf 100644 --- a/internal/integrations/syfon/adapter_test.go +++ b/internal/integrations/syfon/adapter_test.go @@ -226,6 +226,62 @@ func TestBulkProbeStorageObjectsBatchesRequests(t *testing.T) { } } +func TestBulkListStorageObjectsSendsExpectedName(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/bulk-list" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Items []struct { + ID string `json:"id"` + ObjectURL string `json:"object_url"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes"` + ExpectedName string `json:"expected_name"` + } `json:"items"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if len(req.Items) != 1 || req.Items[0].ExpectedName != "file.bin" { + t.Fatalf("expected request to include expected_name, got %+v", req.Items) + } + sizeMatch := true + nameMatch := true + body, err := json.Marshal(map[string]any{"items": []map[string]any{{ + "id": req.Items[0].ID, + "object_url": req.Items[0].ObjectURL, + "exists": true, + "status": "present", + "validation_status": "matched", + "size_match": sizeMatch, + "name_match": nameMatch, + }}}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + size := int64(17) + manager := NewManager("http://syfon.example", client) + results, err := manager.BulkListStorageObjects(context.Background(), "Bearer token", []BulkStorageProbeItem{{ + ID: "item-1", + ObjectURL: "s3://bucket/file.bin", + ExpectedSizeBytes: &size, + ExpectedName: "file.bin", + }}) + if err != nil { + t.Fatalf("BulkListStorageObjects returned error: %v", err) + } + if len(results) != 1 || results[0].NameMatch == nil || !*results[0].NameMatch { + t.Fatalf("expected parsed name_match result, got %+v", results) + } +} + type roundTripFunc func(*http.Request) (*http.Response, error) func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index ba103c4..a3b6ca4 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -16,6 +16,8 @@ import ( "github.com/gofiber/fiber/v3" ) +const defaultStorageChainFindingLimit = 500 + func (handler *Handler) handleGitProjectStorageSummaryGET(ctx fiber.Ctx) error { projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) if errResponse != nil { @@ -109,18 +111,65 @@ func (handler *Handler) handleGitProjectStorageCleanupAuditPOST(ctx fiber.Ctx) e } func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) error { - projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + routeOrg := strings.TrimSpace(ctx.Params("orgTitle")) + routeProject := strings.TrimSpace(ctx.Params("projectTitle")) + handler.logger.Info("storage_chain_audit_request_received org=%s project=%s path=%s query=%s", routeOrg, routeProject, ctx.Path(), string(ctx.Request().URI().QueryString())) + timings := &gitcore.StorageChainAuditTimings{ + Logf: handler.logger.Info, + DebugPrefix: fmt.Sprintf("org=%s project=%s", routeOrg, routeProject), + } + resolveStart := time.Now() + timings.StageStart("resolve_git_context") + projectCtx, errResponse := handler.resolveGitAnalyticsContextWithTimings(ctx, timings) if errResponse != nil { return errResponse.Write(ctx) } + timings.Record("resolve_git_context", time.Since(resolveStart)) + timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q", projectCtx.projectID, projectCtx.refName, "") requestBody := gitcore.GitStorageChainAuditRequest{} + parseStart := time.Now() + timings.StageStart("parse_request_body") if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { errResponse.WriteLog(handler.logger) return errResponse.Write(ctx) } + timings.Record("parse_request_body", time.Since(parseStart)) + probeMode, ok := gitcore.NormalizeStorageChainProbeMode(requestBody.ProbeMode) + if !ok { + response := httputil.NewError("invalid_request", "probe_mode must be either full or inventory_only", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + bucketMode := gitcore.StorageChainBucketModeItems + if strings.TrimSpace(requestBody.BucketInventoryMode) != "" { + var valid bool + bucketMode, valid = gitcore.NormalizeStorageChainBucketInventoryMode(requestBody.BucketInventoryMode) + if !valid { + response := httputil.NewError("invalid_request", "bucket_inventory_mode must be either validate or items", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + } + findingLimit := requestBody.FindingLimit + if findingLimit == 0 { + findingLimit = defaultStorageChainFindingLimit + } + if findingLimit < -1 { + response := httputil.NewError("invalid_request", "finding_limit must be -1 for all findings or a positive integer", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + bucketPathPrefix := normalizeAnalyticsSubpath(requestBody.BucketPathPrefix) + if bucketPathPrefix != "" && bucketMode != gitcore.StorageChainBucketModeItems { + response := httputil.NewError("invalid_request", "bucket_path_prefix requires bucket_inventory_mode=items", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } projectCtx.applyRequestRef(requestBody.Ref) gitSubpath := normalizeAnalyticsSubpath(requestBody.GitSubpath) - response, err := handler.storageAnalytics.BuildStorageChainAudit( + timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q probe_mode=%s bucket_inventory_mode=%s bucket_path_prefix=%q finding_limit=%d", projectCtx.projectID, projectCtx.refName, gitSubpath, probeMode, bucketMode, bucketPathPrefix, findingLimit) + handler.logger.Info("storage_chain_audit_request_start %s", timings.DebugPrefix) + response, err := handler.storageAnalytics.BuildStorageChainAuditWithOptions( ctx.Context(), projectCtx.authorizationHeader, projectCtx.organization, @@ -130,11 +179,35 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err projectCtx.mirrorPath, projectCtx.repo, projectCtx.hash, + gitcore.StorageChainAuditOptions{ + ProbeMode: probeMode, + BucketInventoryMode: bucketMode, + BucketPathPrefix: bucketPathPrefix, + FindingLimit: findingLimit, + Timings: timings, + }, ) if err != nil { + handler.logger.Info("storage_chain_audit_request_error %s error=%q timings=%s", timings.DebugPrefix, err.Error(), formatStorageChainTimingSnapshot(timings)) return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) } - return httputil.JSON(response, http.StatusOK).Write(ctx) + writeStart := time.Now() + timings.StageStart("json_response") + writeErr := httputil.JSON(response, http.StatusOK).Write(ctx) + timings.Record("json_response", time.Since(writeStart)) + handler.logStorageChainAuditTimings(projectCtx, gitSubpath, probeMode, bucketMode, response, timings) + return writeErr +} + +func formatStorageChainTimingSnapshot(timings *gitcore.StorageChainAuditTimings) string { + parts := make([]string, 0) + for _, stage := range timings.Snapshot() { + if stage.Stage == "" { + continue + } + parts = append(parts, fmt.Sprintf("%s_ms=%d", stage.Stage, stage.Duration.Milliseconds())) + } + return strings.Join(parts, ",") } func (handler *Handler) handleGitProjectStorageCleanupApplyPOST(ctx fiber.Ctx) error { @@ -216,6 +289,10 @@ func (handler *Handler) parseCleanupAnalyticsRequest(ctx fiber.Ctx) (*gitAnalyti } func (handler *Handler) resolveGitAnalyticsContext(ctx fiber.Ctx) (*gitAnalyticsContext, *httputil.ErrorResponse) { + return handler.resolveGitAnalyticsContextWithTimings(ctx, nil) +} + +func (handler *Handler) resolveGitAnalyticsContextWithTimings(ctx fiber.Ctx, timings *gitcore.StorageChainAuditTimings) (*gitAnalyticsContext, *httputil.ErrorResponse) { if handler.storageAnalytics == nil { response := httputil.NewError("internal_error", "storage analytics service is not configured", http.StatusInternalServerError, nil, nil) response.WriteLog(handler.logger) @@ -238,9 +315,12 @@ func (handler *Handler) resolveGitAnalyticsContext(ctx fiber.Ctx) (*gitAnalytics } authorizationHeader := strings.TrimSpace(ctx.Get("Authorization")) if authorizationHeader != "" { + start := time.Now() + timings.StageStart("mirror_warmup") refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + timings.Record("mirror_warmup", time.Since(start)) if err != nil { handler.logger.Warning("failed to warm git mirror for %s analytics: %v", projectID, err) } @@ -320,3 +400,39 @@ func (handler *Handler) writeGitAnalyticsError(ctx fiber.Ctx, projectID string, response.WriteLog(handler.logger) return response.Write(ctx) } + +func (handler *Handler) logStorageChainAuditTimings(projectCtx *gitAnalyticsContext, gitSubpath string, probeMode string, bucketMode string, response *gitcore.GitStorageChainAuditResponse, timings *gitcore.StorageChainAuditTimings) { + if projectCtx == nil || response == nil { + return + } + parts := make([]string, 0) + for _, stage := range timings.Snapshot() { + if stage.Stage == "" { + continue + } + parts = append(parts, fmt.Sprintf("%s_ms=%d", stage.Stage, stage.Duration.Milliseconds())) + } + bucketPathExists := "unknown" + if response.Summary.BucketPathExists != nil { + bucketPathExists = strconv.FormatBool(*response.Summary.BucketPathExists) + } + handler.logger.Info( + "storage_chain_audit project_id=%s ref=%s git_subpath=%q probe_mode=%s bucket_inventory_mode=%s bucket_path_exists=%s bucket_summary_mode=%s bucket_inventory_available=%t findings=%d returned_findings=%d findings_truncated=%t finding_limit=%d bucket_objects=%d syfon_records=%d git_files=%d %s", + projectCtx.projectID, + projectCtx.refName, + gitSubpath, + probeMode, + bucketMode, + bucketPathExists, + response.Summary.BucketSummaryMode, + response.Summary.BucketInventoryAvailable, + response.Summary.TotalFindings, + response.Summary.ReturnedFindings, + response.Summary.FindingsTruncated, + response.Summary.FindingLimit, + response.Summary.BucketObjectCount, + response.Summary.SyfonRecordCount, + response.Summary.GitTrackedFileCount, + strings.Join(parts, " "), + ) +} From 97bdffb5101442447cca2310a6999804066865a0 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 2 Jul 2026 15:29:32 -0700 Subject: [PATCH 14/36] fix LIST operation in gecko audit --- internal/git/storage_analytics.go | 12 +++ internal/git/storage_analytics_pipeline.go | 76 ++++++++++++++----- internal/git/storage_analytics_test.go | 2 +- internal/server/http/git/storage_analytics.go | 2 +- 4 files changed, 70 insertions(+), 22 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 81ff3bf..0295eed 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -1201,6 +1201,9 @@ func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[st if inventoryHasValidationMismatch(record, bucketMatches, bucketObjectsByURL) { return storageFindingValidationMismatch } + if recordHasValidationMismatchProbe(record) { + return storageFindingValidationMismatch + } if len(bucketMatches) > 0 { return storageFindingNone } @@ -1290,6 +1293,15 @@ func hasExactPathBucketMismatch(record projectRecordState, bucketMatches []strin return false } +func recordHasValidationMismatchProbe(record projectRecordState) bool { + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.ValidationStatus) == "mismatched" { + return true + } + } + return false +} + func inventoryHasValidationMismatch(record projectRecordState, bucketObjectURLs []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { if len(bucketObjectURLs) == 0 { return false diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 437d05e..f40a6c2 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -38,11 +38,18 @@ type storageAuditStorageView struct { } type storageChainIndex struct { - inventory []RepoInventoryFile - allRecords []projectRecordState - bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject - repoPathsByChecksum map[string][]string - recordsByBucketURL map[string][]projectRecordState + inventory []RepoInventoryFile + allRecords []projectRecordState + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + repoPathsByChecksum map[string][]string + recordsByBucketURL map[string][]projectRecordState + equivalentRecordKeys equivalentRecordKeyIndex +} + +type equivalentRecordKeyIndex struct { + byName map[string]struct{} + byNameSize map[string]map[int64]struct{} + unknownSizeByName map[string]struct{} } func (service *StorageAnalyticsService) loadStorageChainInventory(ctx context.Context, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) ([]RepoInventoryFile, error) { @@ -826,11 +833,12 @@ func buildStorageChainIndex(inventory []RepoInventoryFile, allProjectRecords map } } return storageChainIndex{ - inventory: inventory, - allRecords: allRecords, - bucketObjectsByURL: bucketObjectsByURL, - repoPathsByChecksum: repoPathsByChecksum, - recordsByBucketURL: recordsByBucketURL, + inventory: inventory, + allRecords: allRecords, + bucketObjectsByURL: bucketObjectsByURL, + repoPathsByChecksum: repoPathsByChecksum, + recordsByBucketURL: recordsByBucketURL, + equivalentRecordKeys: buildEquivalentRecordKeyIndex(allRecords), } } @@ -860,17 +868,18 @@ func buildBucketOriginChainFindings(index storageChainIndex, acc *chainAuditAccu acc.addCount("bucket_syfon_git_complete", 1) continue } - if bucketObjectHasEquivalentSyfonRecord(item, index.allRecords) { + if bucketObjectHasEquivalentSyfonRecord(item, index.equivalentRecordKeys) { continue } acc.add("bucket_only_object", buildChainBucketOnlyFinding(item)) } } -func bucketObjectHasEquivalentSyfonRecord(item gintegrationsyfon.ProjectBucketObject, records []projectRecordState) bool { - itemName := strings.TrimSpace(path.Base(strings.TrimSpace(item.Key))) - if itemName == "" { - return false +func buildEquivalentRecordKeyIndex(records []projectRecordState) equivalentRecordKeyIndex { + index := equivalentRecordKeyIndex{ + byName: make(map[string]struct{}), + byNameSize: make(map[string]map[int64]struct{}), + unknownSizeByName: make(map[string]struct{}), } for _, record := range records { for _, accessURL := range accessURLsForStorage(record) { @@ -878,15 +887,42 @@ func bucketObjectHasEquivalentSyfonRecord(item gintegrationsyfon.ProjectBucketOb if !ok { continue } - if strings.TrimSpace(path.Base(key)) != itemName { + name := strings.TrimSpace(path.Base(key)) + if name == "" { continue } - if record.Size > 0 && item.SizeBytes > 0 && record.Size != item.SizeBytes { + index.byName[name] = struct{}{} + if record.Size <= 0 { + index.unknownSizeByName[name] = struct{}{} continue } - return true + if index.byNameSize[name] == nil { + index.byNameSize[name] = make(map[int64]struct{}) + } + index.byNameSize[name][record.Size] = struct{}{} } } + return index +} + +func bucketObjectHasEquivalentSyfonRecord(item gintegrationsyfon.ProjectBucketObject, index equivalentRecordKeyIndex) bool { + itemName := strings.TrimSpace(path.Base(strings.TrimSpace(item.Key))) + if itemName == "" { + return false + } + if _, ok := index.byName[itemName]; !ok { + return false + } + if item.SizeBytes <= 0 { + return true + } + if _, ok := index.unknownSizeByName[itemName]; ok { + return true + } + if sizes := index.byNameSize[itemName]; sizes != nil { + _, ok := sizes[item.SizeBytes] + return ok + } return false } @@ -914,7 +950,7 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum acc.addCount("git_syfon_metadata_mismatch", len(gitPaths)) continue } - acc.add("bucket_syfon_no_git", buildChainRecordFindingsWithOptions("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.", countCompleteFromSyfon)...) + acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) case storageFindingBrokenAccessURL, storageFindingProbeError: findings := buildChainRecordFindings("probe_error", record, gitPaths, bucketMatches, "Bucket verification failed before Gecko could classify this record cleanly.") acc.findings = append(acc.findings, findings...) @@ -925,7 +961,7 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum continue } if len(gitPaths) == 0 && len(bucketMatches) > 0 { - acc.add("bucket_syfon_no_git", buildChainRecordFindingsWithOptions("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.", countCompleteFromSyfon)...) + acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) } } } diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 606dfe6..8fc1210 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -917,7 +917,7 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { } assertNoChainFinding(t, chain.Findings, "bucket_only_object") assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/mismatch.txt") - assertHasChainFinding(t, chain.Findings, "bucket_syfon_no_git", "syfon-only.txt") + assertHasChainFinding(t, chain.Findings, "bucket_syfon_no_git", "s3://bucket/syfon-only.txt") } func TestExpectedStorageObjectNameForListValidationSkipsContentAddressedKeys(t *testing.T) { diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index a3b6ca4..046fbb9 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -140,7 +140,7 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err response.WriteLog(handler.logger) return response.Write(ctx) } - bucketMode := gitcore.StorageChainBucketModeItems + bucketMode := gitcore.StorageChainBucketModeValidate if strings.TrimSpace(requestBody.BucketInventoryMode) != "" { var valid bool bucketMode, valid = gitcore.NormalizeStorageChainBucketInventoryMode(requestBody.BucketInventoryMode) From 9ebf621bba1e4c127a5ac5d462a274ae3f8229cd Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 2 Jul 2026 16:18:47 -0700 Subject: [PATCH 15/36] upgrade audit analytics --- internal/git/storage_analytics.go | 76 ++++++++----- internal/git/storage_analytics_pipeline.go | 35 ++++-- internal/git/storage_analytics_test.go | 103 +++++++++++++++++- internal/git/storage_analytics_timing.go | 28 +++++ internal/git/types.go | 28 +++-- internal/integrations/syfon/adapter.go | 62 ++++++++++- internal/integrations/syfon/adapter_test.go | 92 ++++++++++++++++ internal/server/http/git/storage_analytics.go | 21 +++- 8 files changed, 389 insertions(+), 56 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 0295eed..e6bbcd2 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -25,7 +25,7 @@ type storageAnalyticsBackend interface { ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) ListProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) - ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) + ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectRecord, error) ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]gintegrationsyfon.ProjectRecord, error) ListProjectFileUsage(ctx context.Context, authorizationHeader string, organization string, project string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) @@ -244,6 +244,18 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co if !ok { return nil, fmt.Errorf("invalid storage chain bucket inventory mode %q", options.BucketInventoryMode) } + validationMode := strings.TrimSpace(options.ValidationMode) + if validationMode == "" { + if strings.TrimSpace(options.ProbeMode) == "" && bucketMode == StorageChainBucketModeValidate { + validationMode = StorageChainValidationModeList + } else { + validationMode = DefaultStorageChainValidationMode(probeMode, bucketMode) + } + } + validationMode, ok = NormalizeStorageChainValidationMode(validationMode) + if !ok { + return nil, fmt.Errorf("invalid storage chain validation mode %q", options.ValidationMode) + } bucketPathPrefix := normalizeRepoSubpath(options.BucketPathPrefix) start := time.Now() options.Timings.StageStart("chain_setup_total") @@ -254,7 +266,7 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co } storageViewStart := time.Now() options.Timings.StageStart("storage_view") - storageView, err := service.buildStorageChainView(ctx, authorizationHeader, inputs.recordSet, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, probeMode, options.Timings) + storageView, err := service.buildStorageChainView(ctx, authorizationHeader, inputs.recordSet, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.Timings) options.Timings.Record("storage_view", time.Since(storageViewStart)) if err != nil { return nil, err @@ -266,6 +278,7 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co options.Timings.Record("model_build", time.Since(modelStart)) model.Summary.BucketInventoryAvailable = storageView.bucketInventoryAvailable model.Summary.BucketInventoryError = storageView.bucketInventoryError + model.Summary.ValidationMode = validationMode if inputs.bucketSummary != nil { exists := inputs.bucketSummary.Exists model.Summary.BucketPathExists = &exists @@ -651,8 +664,10 @@ func (service *StorageAnalyticsService) attachStorageValidationResults(ctx conte } } mode := "head" + operation := StorageChainValidationModeMetadata if useListValidation { mode = "list" + operation = StorageChainValidationModeList } log.Printf( "INFO: storage_chain_validation_request_built mode=%s record_count=%d access_url_count=%d unique_request_count=%d duplicate_request_count=%d build_ms=%d", @@ -684,6 +699,7 @@ func (service *StorageAnalyticsService) attachStorageValidationResults(ctx conte return nil, nil, fmt.Errorf("probe syfon storage objects: %w", err) } for _, result := range results { + result.Operation = operation resultsByKey[strings.TrimSpace(result.ID)] = result } } @@ -699,6 +715,7 @@ func (service *StorageAnalyticsService) attachStorageValidationResults(ctx conte probes := make([]gintegrationsyfon.BulkStorageProbeResult, 0, len(keys)) for _, key := range keys { if result, ok := resultsByKey[key]; ok { + result.Operation = operation probes = append(probes, result) } } @@ -1332,6 +1349,7 @@ func accessProbesForRecord(record projectRecordState) []GitStorageCleanupAccessP exists := probe.Exists probes = append(probes, GitStorageCleanupAccessProbe{ URL: probe.ObjectURL, + Operation: probe.Operation, Provider: probe.Provider, Bucket: probe.Bucket, Key: probe.Key, @@ -1421,11 +1439,12 @@ func buildBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStora SizeBytes: item.SizeBytes, LastUpdated: strings.TrimSpace(item.LastModified), Evidence: &GitAuditEvidence{ - AccessURLs: []string{objectURL}, - Buckets: uniqueStrings([]string{item.Bucket}), - Keys: uniqueStrings([]string{item.Key}), - ProbeStatuses: []string{"enumerated"}, - BucketEvaluation: "enumerated", + AccessURLs: []string{objectURL}, + Buckets: uniqueStrings([]string{item.Bucket}), + Keys: uniqueStrings([]string{item.Key}), + StorageOperations: []string{StorageChainValidationModeInventory}, + ProbeStatuses: []string{"enumerated"}, + BucketEvaluation: "enumerated", }, } } @@ -1445,12 +1464,13 @@ func buildChainBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) Git SizeBytes: item.SizeBytes, RecommendedAction: "Bucket object exists, but no Syfon record matched it.", Evidence: &GitAuditEvidence{ - AccessURLs: []string{objectURL}, - BucketObjectURLs: []string{objectURL}, - Buckets: uniqueStrings([]string{item.Bucket}), - Keys: uniqueStrings([]string{item.Key}), - ProbeStatuses: []string{"enumerated"}, - BucketEvaluation: "enumerated", + AccessURLs: []string{objectURL}, + BucketObjectURLs: []string{objectURL}, + Buckets: uniqueStrings([]string{item.Bucket}), + Keys: uniqueStrings([]string{item.Key}), + StorageOperations: []string{StorageChainValidationModeInventory}, + ProbeStatuses: []string{"enumerated"}, + BucketEvaluation: "enumerated", }, } } @@ -1973,18 +1993,19 @@ func recordObjectIDs(matches []projectRecordState) []string { func buildFindingEvidence(checksum string, sourcePaths []string, matches []projectRecordState, bucketEvaluation string) *GitAuditEvidence { evidence := &GitAuditEvidence{ - Checksum: strings.TrimSpace(checksum), - SourcePaths: uniqueStrings(sourcePaths), - ObjectIDs: []string{}, - AccessURLs: []string{}, - BucketObjectURLs: []string{}, - Buckets: []string{}, - Keys: []string{}, - ProbeStatuses: []string{}, - ValidationStates: []string{}, - ErrorKinds: []string{}, - Errors: []string{}, - BucketEvaluation: strings.TrimSpace(bucketEvaluation), + Checksum: strings.TrimSpace(checksum), + SourcePaths: uniqueStrings(sourcePaths), + ObjectIDs: []string{}, + AccessURLs: []string{}, + BucketObjectURLs: []string{}, + Buckets: []string{}, + Keys: []string{}, + StorageOperations: []string{}, + ProbeStatuses: []string{}, + ValidationStates: []string{}, + ErrorKinds: []string{}, + Errors: []string{}, + BucketEvaluation: strings.TrimSpace(bucketEvaluation), } for _, match := range matches { if objectID := strings.TrimSpace(match.ObjectID); objectID != "" { @@ -2004,6 +2025,9 @@ func buildFindingEvidence(checksum string, sourcePaths []string, matches []proje if key := strings.TrimSpace(probe.Key); key != "" { evidence.Keys = append(evidence.Keys, key) } + if operation := strings.TrimSpace(probe.Operation); operation != "" { + evidence.StorageOperations = append(evidence.StorageOperations, operation) + } if status := strings.TrimSpace(probe.Status); status != "" { evidence.ProbeStatuses = append(evidence.ProbeStatuses, status) } @@ -2023,6 +2047,7 @@ func buildFindingEvidence(checksum string, sourcePaths []string, matches []proje evidence.BucketObjectURLs = uniqueStrings(evidence.BucketObjectURLs) evidence.Buckets = uniqueStrings(evidence.Buckets) evidence.Keys = uniqueStrings(evidence.Keys) + evidence.StorageOperations = uniqueStrings(evidence.StorageOperations) evidence.ProbeStatuses = uniqueStrings(evidence.ProbeStatuses) evidence.ValidationStates = uniqueStrings(evidence.ValidationStates) evidence.ErrorKinds = uniqueStrings(evidence.ErrorKinds) @@ -2034,6 +2059,7 @@ func buildFindingEvidence(checksum string, sourcePaths []string, matches []proje len(evidence.BucketObjectURLs) == 0 && len(evidence.Buckets) == 0 && len(evidence.Keys) == 0 && + len(evidence.StorageOperations) == 0 && len(evidence.ProbeStatuses) == 0 && len(evidence.ValidationStates) == 0 && len(evidence.ErrorKinds) == 0 && diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index f40a6c2..e52ebf8 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -106,7 +106,7 @@ func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Conte go func() { start := time.Now() timings.StageStart("syfon_project_records") - recordSet, err := service.loadCachedProjectAuditRecordSet(ctx, authorizationHeader, organization, project) + recordSet, err := service.loadCachedProjectAuditRecordSet(ctx, authorizationHeader, organization, project, gitSubpath) timings.Record("syfon_project_records", time.Since(start)) recordCount := 0 if recordSet != nil { @@ -329,23 +329,23 @@ func applyScopeCanonicalization(recordSet *storageAuditRecordSet, scopes []domai } } -func (service *StorageAnalyticsService) loadProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string) (*storageAuditRecordSet, error) { - projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project) +func (service *StorageAnalyticsService) loadProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) (*storageAuditRecordSet, error) { + projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project, pathPrefix) if err != nil { return nil, fmt.Errorf("list syfon project audit records: %w", err) } return buildProjectAuditRecordSet(projectRecords), nil } -func (service *StorageAnalyticsService) loadCachedProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string) (*storageAuditRecordSet, error) { - cacheKey := service.projectChainInputCacheKey(organization, project) +func (service *StorageAnalyticsService) loadCachedProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) (*storageAuditRecordSet, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + "::audit-records::" + normalizeRepoSubpath(pathPrefix) service.chainInputMu.RLock() cached, ok := service.chainInputCache[cacheKey] service.chainInputMu.RUnlock() if ok && time.Now().Before(cached.expiresAt) && cached.projectRecords != nil { return buildProjectAuditRecordSet(cached.projectRecords), nil } - projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project) + projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project, pathPrefix) if err != nil { return nil, fmt.Errorf("list syfon project audit records: %w", err) } @@ -608,7 +608,7 @@ func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context return view, nil } -func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, probeMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { +func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { recordSet = applyScopeCanonicalization(recordSet, scopes) view := &storageAuditStorageView{ scopes: append([]domain.StorageBucketScope(nil), scopes...), @@ -623,9 +623,24 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex view.bucketInventoryAvailable = false view.bucketInventoryError = strings.TrimSpace(bucketInventoryErr.Error()) } + if validationMode == StorageChainValidationModeInventory { + return view, nil + } probeStart := time.Now() - probedRecordSet, probeErr := service.attachProjectStorageListValidations(ctx, authorizationHeader, recordSet) - timings.Record("bulk_list_validation", time.Since(probeStart)) + var ( + probedRecordSet *storageAuditRecordSet + probeErr error + stage string + ) + switch validationMode { + case StorageChainValidationModeMetadata: + probedRecordSet, probeErr = service.attachProjectStorageProbes(ctx, authorizationHeader, recordSet) + stage = "bulk_metadata_validation" + default: + probedRecordSet, probeErr = service.attachProjectStorageListValidations(ctx, authorizationHeader, recordSet) + stage = "bulk_list_validation" + } + timings.Record(stage, time.Since(probeStart)) if probeErr != nil { return nil, probeErr } @@ -638,7 +653,7 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex return nil, bucketInventoryErr } view.bucketObjects, view.bucketObjectsByURL = cloneBucketInventory(bucketObjects, bucketObjectsByURL) - if probeMode == StorageChainProbeModeInventoryOnly { + if validationMode == StorageChainValidationModeInventory || validationMode == StorageChainValidationModeList { return view, nil } diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 8fc1210..9c0e3a0 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -31,6 +31,7 @@ type fakeStorageAnalyticsBackend struct { listBucketsCalls int listBucketScopesCalls int listProjectAuditRecordsCalls int + listProjectAuditRecordsPathPrefix string listProjectScopesCalls int listProjectFileUsageCalls int listProjectBucketObjectsCalls int @@ -64,8 +65,9 @@ func (fake *fakeStorageAnalyticsBackend) ListProjectRecords(ctx context.Context, return append([]gintegrationsyfon.ProjectRecord(nil), fake.projectRecords...), nil } -func (fake *fakeStorageAnalyticsBackend) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) { +func (fake *fakeStorageAnalyticsBackend) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectRecord, error) { fake.listProjectAuditRecordsCalls++ + fake.listProjectAuditRecordsPathPrefix = pathPrefix return append([]gintegrationsyfon.ProjectRecord(nil), fake.projectRecords...), nil } @@ -779,6 +781,9 @@ func TestBuildStorageChainAuditUsesProjectAuditSourcesAndTargetsProbes(t *testin if backend.listProjectAuditRecordsCalls != 1 { t.Fatalf("expected one project audit record call, got %d", backend.listProjectAuditRecordsCalls) } + if backend.listProjectAuditRecordsPathPrefix != "data" { + t.Fatalf("expected audit path prefix to scope project records, got %q", backend.listProjectAuditRecordsPathPrefix) + } if backend.listProjectScopesCalls != 1 { t.Fatalf("expected one project scope call, got %d", backend.listProjectScopesCalls) } @@ -903,6 +908,9 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { if chain.Summary.BucketPathExists != nil || chain.Summary.BucketSummaryMode != "" { t.Fatalf("expected validate mode not to claim bucket prefix summary, got %+v", chain.Summary) } + if chain.Summary.ValidationMode != StorageChainValidationModeList { + t.Fatalf("expected LIST validation mode, got %+v", chain.Summary) + } if chain.Summary.BucketObjectCount != 3 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { t.Fatalf("expected validate mode to count validated objects without bucket-only inventory, got summary %+v", chain.Summary) } @@ -920,6 +928,68 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { assertHasChainFinding(t, chain.Findings, "bucket_syfon_no_git", "s3://bucket/syfon-only.txt") } +func TestBuildStorageChainAuditMetadataModeUsesMetadataValidation(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a.txt", + Bucket: "bucket", + Key: "a.txt", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"sha256_mismatch"}, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + ValidationMode: StorageChainValidationModeMetadata, + }, + ) + if err != nil { + t.Fatalf("build metadata chain audit: %v", err) + } + if backend.listProjectBucketObjectsCalls != 0 { + t.Fatalf("expected metadata validation to skip recursive bucket inventory, got %d calls", backend.listProjectBucketObjectsCalls) + } + if backend.listProbeCalls != 0 { + t.Fatalf("expected metadata validation to skip LIST validation, got %d calls", backend.listProbeCalls) + } + if backend.probeCalls != 1 || len(backend.probeItems) != 1 { + t.Fatalf("expected one metadata probe call, got calls=%d items=%+v", backend.probeCalls, backend.probeItems) + } + if chain.Summary.ValidationMode != StorageChainValidationModeMetadata { + t.Fatalf("expected metadata validation mode, got %+v", chain.Summary) + } + finding := assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/a.txt") + if finding.Evidence == nil || !contains(finding.Evidence.StorageOperations, StorageChainValidationModeMetadata) { + t.Fatalf("expected metadata operation evidence, got %+v", finding.Evidence) + } +} + func TestExpectedStorageObjectNameForListValidationSkipsContentAddressedKeys(t *testing.T) { if got := expectedStorageObjectNameForListValidation("s3://cbds/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/ec4b068cb42b52449dd44052c3bfb2a459b00336a9cd42cd29c22ca1d1b26cb0", "CONFIG/cbds-BForePC.json"); got != "" { t.Fatalf("expected content-addressed storage key to skip name validation, got %q", got) @@ -956,6 +1026,9 @@ func TestBuildStorageChainAuditCachesProjectChainInputs(t *testing.T) { if backend.listProjectAuditRecordsCalls != 1 { t.Fatalf("expected cached project records, got %d calls", backend.listProjectAuditRecordsCalls) } + if backend.listProjectAuditRecordsPathPrefix != "data" { + t.Fatalf("expected cached project records to use data subpath, got %q", backend.listProjectAuditRecordsPathPrefix) + } if backend.listProjectScopesCalls != 1 { t.Fatalf("expected cached project scopes, got %d calls", backend.listProjectScopesCalls) } @@ -964,6 +1037,34 @@ func TestBuildStorageChainAuditCachesProjectChainInputs(t *testing.T) { } } +func TestBuildStorageChainAuditCachesProjectRecordsPerPathPrefix(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "CONFIG/a.json": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "DATA/b.json": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/root/CONFIG/a.json"}}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/root/DATA/b.json"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket/root"}, + }, + } + service := NewStorageAnalyticsService(backend) + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "CONFIG", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build first chain audit: %v", err) + } + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "DATA", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build second chain audit: %v", err) + } + if backend.listProjectAuditRecordsCalls != 2 { + t.Fatalf("expected path-scoped project record cache to distinguish prefixes, got %d calls", backend.listProjectAuditRecordsCalls) + } +} + func TestBuildStorageChainAuditForwardsBucketPathPrefixForExplicitInventory(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "README.md": "fixture", diff --git a/internal/git/storage_analytics_timing.go b/internal/git/storage_analytics_timing.go index 529c22f..955ee97 100644 --- a/internal/git/storage_analytics_timing.go +++ b/internal/git/storage_analytics_timing.go @@ -12,10 +12,15 @@ const ( StorageChainBucketModeItems = "items" StorageChainBucketModeValidate = "validate" + + StorageChainValidationModeList = "list" + StorageChainValidationModeMetadata = "metadata" + StorageChainValidationModeInventory = "inventory" ) type StorageChainAuditOptions struct { ProbeMode string + ValidationMode string BucketInventoryMode string BucketPathPrefix string FindingLimit int @@ -81,6 +86,29 @@ func NormalizeStorageChainProbeMode(raw string) (string, bool) { } } +func NormalizeStorageChainValidationMode(raw string) (string, bool) { + switch strings.TrimSpace(raw) { + case "", StorageChainValidationModeList: + return StorageChainValidationModeList, true + case StorageChainValidationModeMetadata, StorageChainProbeModeFull: + return StorageChainValidationModeMetadata, true + case StorageChainValidationModeInventory, StorageChainProbeModeInventoryOnly: + return StorageChainValidationModeInventory, true + default: + return "", false + } +} + +func DefaultStorageChainValidationMode(probeMode string, bucketMode string) string { + switch strings.TrimSpace(probeMode) { + case StorageChainProbeModeInventoryOnly: + return StorageChainValidationModeInventory + case StorageChainProbeModeFull: + return StorageChainValidationModeMetadata + } + return StorageChainValidationModeList +} + func NormalizeStorageChainBucketInventoryMode(raw string) (string, bool) { switch strings.TrimSpace(raw) { case "", StorageChainBucketModeItems: diff --git a/internal/git/types.go b/internal/git/types.go index 4cc5103..4fdd832 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -339,18 +339,19 @@ type GitProjectDiffAuditRequest struct { } type GitAuditEvidence struct { - Checksum string `json:"checksum,omitempty"` - SourcePaths []string `json:"source_paths,omitempty"` - ObjectIDs []string `json:"object_ids,omitempty"` - AccessURLs []string `json:"access_urls,omitempty"` - BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` - Buckets []string `json:"buckets,omitempty"` - Keys []string `json:"keys,omitempty"` - ProbeStatuses []string `json:"probe_statuses,omitempty"` - ValidationStates []string `json:"validation_states,omitempty"` - ErrorKinds []string `json:"error_kinds,omitempty"` - Errors []string `json:"errors,omitempty"` - BucketEvaluation string `json:"bucket_evaluation,omitempty"` + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` + Buckets []string `json:"buckets,omitempty"` + Keys []string `json:"keys,omitempty"` + StorageOperations []string `json:"storage_operations,omitempty"` + ProbeStatuses []string `json:"probe_statuses,omitempty"` + ValidationStates []string `json:"validation_states,omitempty"` + ErrorKinds []string `json:"error_kinds,omitempty"` + Errors []string `json:"errors,omitempty"` + BucketEvaluation string `json:"bucket_evaluation,omitempty"` } type GitProjectDiffFinding struct { @@ -394,6 +395,7 @@ type GitStorageChainAuditRequest struct { GitSubpath string `json:"git_subpath,omitempty"` Ref string `json:"ref,omitempty"` ProbeMode string `json:"probe_mode,omitempty"` + ValidationMode string `json:"validation_mode,omitempty"` BucketInventoryMode string `json:"bucket_inventory_mode,omitempty"` BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` FindingLimit int `json:"finding_limit,omitempty"` @@ -424,6 +426,7 @@ type GitStorageChainAuditSummary struct { ReturnedFindings int `json:"returned_findings"` FindingsTruncated bool `json:"findings_truncated"` FindingLimit int `json:"finding_limit,omitempty"` + ValidationMode string `json:"validation_mode,omitempty"` BucketObjectCount int `json:"bucket_object_count"` SyfonRecordCount int `json:"syfon_record_count"` GitTrackedFileCount int `json:"git_tracked_file_count"` @@ -453,6 +456,7 @@ type GitStorageChainAuditResponse struct { type GitStorageCleanupAccessProbe struct { URL string `json:"url"` + Operation string `json:"operation,omitempty"` Provider string `json:"provider,omitempty"` Bucket string `json:"bucket,omitempty"` Key string `json:"key,omitempty"` diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index 6c77346..4afb940 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -62,6 +62,7 @@ type BulkStorageProbeItem struct { type BulkStorageProbeResult struct { ID string + Operation string ObjectURL string Provider string Bucket string @@ -300,13 +301,15 @@ func (manager *Manager) ListProjectRecords(ctx context.Context, authorizationHea return out, nil } -func (manager *Manager) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]ProjectRecord, error) { +func (manager *Manager) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]ProjectRecord, error) { requestBody := struct { Organization string `json:"organization,omitempty"` Project string `json:"project,omitempty"` + PathPrefix string `json:"path_prefix,omitempty"` }{ Organization: strings.TrimSpace(organization), Project: strings.TrimSpace(project), + PathPrefix: strings.Trim(strings.TrimSpace(pathPrefix), "/"), } var response struct { Items []struct { @@ -616,8 +619,20 @@ func (manager *Manager) bulkStorageObjectRequest(ctx context.Context, authorizat return []BulkStorageProbeResult{}, nil } started := time.Now() + requestItems := append([]BulkStorageProbeItem(nil), items...) + duplicateCount := 0 + resultKeyByOriginalID := make(map[string]string, len(requestItems)) + for _, item := range requestItems { + resultKeyByOriginalID[strings.TrimSpace(item.ID)] = strings.TrimSpace(item.ID) + } + if includeExpectedName { + var deduped map[string]string + items, deduped = dedupeBulkStorageProbeItems(items, includeExpectedName) + duplicateCount = len(requestItems) - len(items) + resultKeyByOriginalID = deduped + } batchCount := (len(items) + bulkStorageProbeBatchSize - 1) / bulkStorageProbeBatchSize - log.Printf("INFO: syfon_bulk_storage_request_start path=%s items=%d batch_size=%d batches=%d concurrency=%d include_expected_name=%t", requestPath, len(items), bulkStorageProbeBatchSize, batchCount, bulkStorageProbeConcurrency, includeExpectedName) + log.Printf("INFO: syfon_bulk_storage_request_start path=%s items=%d unique_items=%d duplicate_items=%d batch_size=%d batches=%d concurrency=%d include_expected_name=%t", requestPath, len(requestItems), len(items), duplicateCount, bulkStorageProbeBatchSize, batchCount, bulkStorageProbeConcurrency, includeExpectedName) resultsByID := make(map[string]BulkStorageProbeResult, len(items)) var mu sync.Mutex var wg sync.WaitGroup @@ -658,16 +673,51 @@ func (manager *Manager) bulkStorageObjectRequest(ctx context.Context, authorizat log.Printf("INFO: syfon_bulk_storage_request_done path=%s items=%d results=%d batches=%d duration_ms=%d error=%q", requestPath, len(items), len(resultsByID), batchCount, time.Since(started).Milliseconds(), firstErr.Error()) return nil, firstErr } - out := make([]BulkStorageProbeResult, 0, len(resultsByID)) - for _, item := range items { - if result, ok := resultsByID[strings.TrimSpace(item.ID)]; ok { + out := make([]BulkStorageProbeResult, 0, len(requestItems)) + for _, item := range requestItems { + resultKey := resultKeyByOriginalID[strings.TrimSpace(item.ID)] + if result, ok := resultsByID[resultKey]; ok { + result.ID = strings.TrimSpace(item.ID) out = append(out, result) } } - log.Printf("INFO: syfon_bulk_storage_request_done path=%s items=%d results=%d batches=%d duration_ms=%d", requestPath, len(items), len(out), batchCount, time.Since(started).Milliseconds()) + log.Printf("INFO: syfon_bulk_storage_request_done path=%s items=%d unique_items=%d results=%d batches=%d duration_ms=%d", requestPath, len(requestItems), len(items), len(out), batchCount, time.Since(started).Milliseconds()) return out, nil } +func dedupeBulkStorageProbeItems(items []BulkStorageProbeItem, includeExpectedName bool) ([]BulkStorageProbeItem, map[string]string) { + seen := make(map[string]BulkStorageProbeItem, len(items)) + keyByOriginalID := make(map[string]string, len(items)) + out := make([]BulkStorageProbeItem, 0, len(items)) + for _, item := range items { + key := bulkStorageProbeDedupKey(item, includeExpectedName) + originalID := strings.TrimSpace(item.ID) + if existing, ok := seen[key]; ok { + keyByOriginalID[originalID] = strings.TrimSpace(existing.ID) + continue + } + seen[key] = item + keyByOriginalID[originalID] = originalID + out = append(out, item) + } + return out, keyByOriginalID +} + +func bulkStorageProbeDedupKey(item BulkStorageProbeItem, includeExpectedName bool) string { + parts := []string{strings.TrimSpace(item.ObjectURL)} + if item.ExpectedSizeBytes == nil { + parts = append(parts, "") + } else { + parts = append(parts, strconv.FormatInt(*item.ExpectedSizeBytes, 10)) + } + if includeExpectedName { + parts = append(parts, strings.TrimSpace(item.ExpectedName)) + } else { + parts = append(parts, strings.TrimSpace(item.ExpectedSHA256)) + } + return strings.Join(parts, "\x00") +} + func (manager *Manager) probeStorageObjectBatch(ctx context.Context, authorizationHeader string, requestPath string, items []BulkStorageProbeItem, includeExpectedName bool) ([]BulkStorageProbeResult, error) { requestBody := struct { Items []struct { diff --git a/internal/integrations/syfon/adapter_test.go b/internal/integrations/syfon/adapter_test.go index 4aa41cf..928f943 100644 --- a/internal/integrations/syfon/adapter_test.go +++ b/internal/integrations/syfon/adapter_test.go @@ -282,6 +282,98 @@ func TestBulkListStorageObjectsSendsExpectedName(t *testing.T) { } } +func TestBulkListStorageObjectsDeduplicatesIdenticalRequests(t *testing.T) { + requests := 0 + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/bulk-list" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + requests++ + var req struct { + Items []struct { + ID string `json:"id"` + ObjectURL string `json:"object_url"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes"` + ExpectedName string `json:"expected_name"` + } `json:"items"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if len(req.Items) != 1 { + t.Fatalf("expected identical LIST items to be deduplicated, got %+v", req.Items) + } + sizeMatch := true + nameMatch := true + body, err := json.Marshal(map[string]any{"items": []map[string]any{{ + "id": req.Items[0].ID, + "object_url": req.Items[0].ObjectURL, + "exists": true, + "status": "present", + "validation_status": "matched", + "size_match": sizeMatch, + "name_match": nameMatch, + }}}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + size := int64(17) + manager := NewManager("http://syfon.example", client) + results, err := manager.BulkListStorageObjects(context.Background(), "Bearer token", []BulkStorageProbeItem{ + {ID: "item-1", ObjectURL: "s3://bucket/file.bin", ExpectedSizeBytes: &size, ExpectedName: "file.bin"}, + {ID: "item-2", ObjectURL: "s3://bucket/file.bin", ExpectedSizeBytes: &size, ExpectedName: "file.bin"}, + }) + if err != nil { + t.Fatalf("BulkListStorageObjects returned error: %v", err) + } + if requests != 1 { + t.Fatalf("expected one bulk-list request, got %d", requests) + } + if len(results) != 2 || results[0].ObjectURL != "s3://bucket/file.bin" || results[1].ObjectURL != "s3://bucket/file.bin" { + t.Fatalf("expected deduplicated response to fan out to all original items, got %+v", results) + } +} + +func TestListProjectAuditRecordsIncludesPathPrefix(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/project-records" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Organization string `json:"organization"` + Project string `json:"project"` + PathPrefix string `json:"path_prefix"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if req.Organization != "org" || req.Project != "proj" || req.PathPrefix != "CONFIG" { + t.Fatalf("unexpected request payload: %+v", req) + } + body, err := json.Marshal(map[string]any{"items": []map[string]any{}}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + if _, err := manager.ListProjectAuditRecords(context.Background(), "Bearer token", "org", "proj", "/CONFIG/"); err != nil { + t.Fatalf("ListProjectAuditRecords returned error: %v", err) + } +} + type roundTripFunc func(*http.Request) (*http.Response, error) func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index 046fbb9..a13a302 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -140,6 +140,18 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err response.WriteLog(handler.logger) return response.Write(ctx) } + validationMode := gitcore.StorageChainValidationModeList + if strings.TrimSpace(requestBody.ValidationMode) != "" { + var valid bool + validationMode, valid = gitcore.NormalizeStorageChainValidationMode(requestBody.ValidationMode) + if !valid { + response := httputil.NewError("invalid_request", "validation_mode must be either list, metadata, or inventory", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + } else if strings.TrimSpace(requestBody.ProbeMode) != "" { + validationMode = gitcore.DefaultStorageChainValidationMode(probeMode, "") + } bucketMode := gitcore.StorageChainBucketModeValidate if strings.TrimSpace(requestBody.BucketInventoryMode) != "" { var valid bool @@ -150,6 +162,9 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err return response.Write(ctx) } } + if validationMode == gitcore.StorageChainValidationModeInventory && strings.TrimSpace(requestBody.BucketInventoryMode) == "" { + bucketMode = gitcore.StorageChainBucketModeItems + } findingLimit := requestBody.FindingLimit if findingLimit == 0 { findingLimit = defaultStorageChainFindingLimit @@ -167,7 +182,7 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err } projectCtx.applyRequestRef(requestBody.Ref) gitSubpath := normalizeAnalyticsSubpath(requestBody.GitSubpath) - timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q probe_mode=%s bucket_inventory_mode=%s bucket_path_prefix=%q finding_limit=%d", projectCtx.projectID, projectCtx.refName, gitSubpath, probeMode, bucketMode, bucketPathPrefix, findingLimit) + timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_prefix=%q finding_limit=%d", projectCtx.projectID, projectCtx.refName, gitSubpath, validationMode, probeMode, bucketMode, bucketPathPrefix, findingLimit) handler.logger.Info("storage_chain_audit_request_start %s", timings.DebugPrefix) response, err := handler.storageAnalytics.BuildStorageChainAuditWithOptions( ctx.Context(), @@ -181,6 +196,7 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err projectCtx.hash, gitcore.StorageChainAuditOptions{ ProbeMode: probeMode, + ValidationMode: validationMode, BucketInventoryMode: bucketMode, BucketPathPrefix: bucketPathPrefix, FindingLimit: findingLimit, @@ -417,10 +433,11 @@ func (handler *Handler) logStorageChainAuditTimings(projectCtx *gitAnalyticsCont bucketPathExists = strconv.FormatBool(*response.Summary.BucketPathExists) } handler.logger.Info( - "storage_chain_audit project_id=%s ref=%s git_subpath=%q probe_mode=%s bucket_inventory_mode=%s bucket_path_exists=%s bucket_summary_mode=%s bucket_inventory_available=%t findings=%d returned_findings=%d findings_truncated=%t finding_limit=%d bucket_objects=%d syfon_records=%d git_files=%d %s", + "storage_chain_audit project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_exists=%s bucket_summary_mode=%s bucket_inventory_available=%t findings=%d returned_findings=%d findings_truncated=%t finding_limit=%d bucket_objects=%d syfon_records=%d git_files=%d %s", projectCtx.projectID, projectCtx.refName, gitSubpath, + response.Summary.ValidationMode, probeMode, bucketMode, bucketPathExists, From 12ca2d53b21f771ac174a18bf21255b92761ca88 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 2 Jul 2026 17:25:19 -0700 Subject: [PATCH 16/36] make backend support all error methods --- internal/git/storage_analytics.go | 252 ++++++++++++++-- internal/git/storage_analytics_pipeline.go | 9 +- internal/git/storage_analytics_test.go | 270 +++++++++++++++++- internal/git/storage_analytics_timing.go | 1 + internal/git/storage_index.go | 38 ++- internal/git/types.go | 32 ++- internal/server/http/git/storage_analytics.go | 10 +- 7 files changed, 572 insertions(+), 40 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index e6bbcd2..0f543ce 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -21,6 +21,18 @@ const projectJoinCacheTTL = 45 * time.Second const chainInputCacheTTL = 45 * time.Second const storageChainValidationDebugSampleLimit = 20 +const ( + storageActionabilityAutoRepair = "auto_repair" + storageActionabilityManualChoice = "manual_choice" + storageActionabilityInspectOnly = "inspect_only" + + storageActionRemoveBrokenAccessURLs = "remove_broken_access_urls" + storageActionDeleteSyfonRecord = "delete_syfon_record" + storageActionDeleteBucketObject = "delete_bucket_object" + storageActionDeleteBoth = "delete_both" + storageActionInspectEvidence = "inspect_evidence" +) + type storageAnalyticsBackend interface { ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) @@ -42,6 +54,7 @@ type StorageAnalyticsService struct { storage storageAnalyticsBackend projectJoinMu sync.RWMutex projectJoinCache map[string]cachedProjectJoinState + projectJoinWork map[string]*inflightProjectJoinState chainInputMu sync.RWMutex chainInputCache map[string]cachedChainInputState } @@ -53,6 +66,7 @@ func NewStorageAnalyticsService(storage storageAnalyticsBackend) *StorageAnalyti return &StorageAnalyticsService{ storage: storage, projectJoinCache: map[string]cachedProjectJoinState{}, + projectJoinWork: map[string]*inflightProjectJoinState{}, chainInputCache: map[string]cachedChainInputState{}, } } @@ -120,6 +134,13 @@ type cachedProjectJoinState struct { usageByObjectID map[string]gintegrationsyfon.FileUsage } +type inflightProjectJoinState struct { + done chan struct{} + recordsByChecksum map[string][]projectRecordState + usageByObjectID map[string]gintegrationsyfon.FileUsage + err error +} + type cachedChainInputState struct { expiresAt time.Time projectRecords []gintegrationsyfon.ProjectRecord @@ -285,19 +306,48 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co model.Summary.BucketPathObjectURL = strings.TrimSpace(inputs.bucketSummary.ObjectURL) model.Summary.BucketSummaryMode = strings.TrimSpace(inputs.bucketSummary.Mode) } - findings := limitStorageChainFindings(model.Findings, options.FindingLimit) - model.Summary.ReturnedFindings = len(findings) - model.Summary.FindingLimit = options.FindingLimit - model.Summary.FindingsTruncated = len(findings) < len(model.Findings) + filteredFindings := filterStorageChainFindingsByKind(model.Findings, options.FindingKind) + summary := filterStorageChainSummary(model.Summary, filteredFindings) + findings := limitStorageChainFindings(filteredFindings, options.FindingLimit) + summary.ReturnedFindings = len(findings) + summary.FindingLimit = options.FindingLimit + summary.FindingsTruncated = len(findings) < len(filteredFindings) return &GitStorageChainAuditResponse{ Findings: findings, - Groups: summarizeChainIssueGroups(model.Findings), - Summary: model.Summary, + Groups: summarizeChainIssueGroups(filteredFindings), + Summary: summary, PathPrefix: model.PathPrefix, BucketPathPrefix: bucketPathPrefix, }, nil } +func filterStorageChainFindingsByKind(findings []GitStorageChainFinding, kind string) []GitStorageChainFinding { + kind = strings.TrimSpace(kind) + if kind == "" { + return append([]GitStorageChainFinding(nil), findings...) + } + filtered := make([]GitStorageChainFinding, 0, len(findings)) + for _, finding := range findings { + if finding.Kind == kind { + filtered = append(filtered, finding) + } + } + return filtered +} + +func filterStorageChainSummary(summary GitStorageChainAuditSummary, findings []GitStorageChainFinding) GitStorageChainAuditSummary { + filteredCounts := make(map[string]int, len(summary.CountsByKind)) + for kind := range summary.CountsByKind { + filteredCounts[kind] = 0 + } + for _, finding := range findings { + filteredCounts[finding.Kind]++ + } + summary.CountsByKind = filteredCounts + summary.TotalFindings = len(findings) + return summary +} + func limitStorageChainFindings(findings []GitStorageChainFinding, limit int) []GitStorageChainFinding { if limit <= 0 || limit >= len(findings) { return append([]GitStorageChainFinding(nil), findings...) @@ -305,11 +355,12 @@ func limitStorageChainFindings(findings []GitStorageChainFinding, limit int) []G return append([]GitStorageChainFinding(nil), findings[:limit]...) } -func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, selectedRepoPaths []string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, checkStorage bool, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool, dryRun bool) (*GitStorageCleanupApplyResponse, error) { +func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, selectedRepoPaths []string, selectedActions []GitStorageCleanupApplyAction, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, checkStorage bool, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool, dryRun bool) (*GitStorageCleanupApplyResponse, error) { _, model, err := service.BuildStorageCleanupAudit(ctx, authorizationHeader, organization, project, ref, gitSubpath, selectedRepoPaths, mirrorPath, repo, hash, checkStorage) if err != nil { return nil, err } + actionSelection := indexCleanupActions(selectedActions) toDelete := make([]string, 0) toDeleteWithStorage := make([]string, 0) toDeleteBucketObjects := make([]string, 0) @@ -355,6 +406,18 @@ func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, } else { skippedPaths = append(skippedPaths, finding.Public.NormalizedPath) } + case "storage_validation_mismatch": + switch cleanupActionForFinding(actionSelection, finding.Public) { + case storageActionDeleteSyfonRecord: + toDelete = append(toDelete, finding.DeleteObjectIDs...) + case storageActionDeleteBucketObject: + toDeleteBucketObjects = append(toDeleteBucketObjects, finding.DeleteBucketObjects...) + case storageActionDeleteBoth: + toDelete = append(toDelete, finding.DeleteObjectIDs...) + toDeleteBucketObjects = append(toDeleteBucketObjects, finding.DeleteBucketObjects...) + default: + manualPaths = append(manualPaths, finding.Public.NormalizedPath) + } default: manualPaths = append(manualPaths, finding.Public.NormalizedPath) } @@ -445,6 +508,43 @@ func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, }, nil } +func indexCleanupActions(actions []GitStorageCleanupApplyAction) map[string]string { + if len(actions) == 0 { + return nil + } + index := make(map[string]string, len(actions)) + for _, action := range actions { + if strings.TrimSpace(action.Action) == "" { + continue + } + path := normalizeRepoSubpath(action.NormalizedPath) + if path == "" { + continue + } + key := cleanupActionKey(strings.TrimSpace(action.Kind), path) + index[key] = strings.TrimSpace(action.Action) + } + return index +} + +func cleanupActionKey(kind string, path string) string { + return strings.TrimSpace(kind) + "::" + normalizeRepoSubpath(path) +} + +func cleanupActionForFinding(index map[string]string, finding GitStorageCleanupFinding) string { + if len(index) == 0 { + return "" + } + path := normalizeRepoSubpath(finding.NormalizedPath) + if path == "" { + return "" + } + if action := strings.TrimSpace(index[cleanupActionKey(finding.Kind, path)]); action != "" { + return action + } + return strings.TrimSpace(index[cleanupActionKey("", path)]) +} + func (service *StorageAnalyticsService) loadJoinState(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, []RepoInventoryFile, map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) if err != nil { @@ -463,23 +563,48 @@ func (service *StorageAnalyticsService) loadJoinState(ctx context.Context, autho func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context, authorizationHeader string, organization string, project string, hash plumbing.Hash, inventory []RepoInventoryFile) (map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { cacheKey := service.projectJoinCacheKey(organization, project, hash.String()) - service.projectJoinMu.RLock() - cached, ok := service.projectJoinCache[cacheKey] - service.projectJoinMu.RUnlock() - if ok && time.Now().Before(cached.expiresAt) { + service.projectJoinMu.Lock() + if cached, ok := service.projectJoinCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) { + service.projectJoinMu.Unlock() return cached.recordsByChecksum, cached.usageByObjectID, nil } + if inflight, ok := service.projectJoinWork[cacheKey]; ok { + service.projectJoinMu.Unlock() + <-inflight.done + return inflight.recordsByChecksum, inflight.usageByObjectID, inflight.err + } + inflight := &inflightProjectJoinState{done: make(chan struct{})} + service.projectJoinWork[cacheKey] = inflight + service.projectJoinMu.Unlock() + defer func() { + service.projectJoinMu.Lock() + delete(service.projectJoinWork, cacheKey) + close(inflight.done) + service.projectJoinMu.Unlock() + }() + checksums := make([]string, 0, len(inventory)) + seenChecksums := make(map[string]struct{}, len(inventory)) for _, item := range inventory { - checksums = append(checksums, item.Checksum) + checksum := strings.TrimSpace(item.Checksum) + if checksum == "" { + continue + } + if _, ok := seenChecksums[checksum]; ok { + continue + } + seenChecksums[checksum] = struct{}{} + checksums = append(checksums, checksum) } recordsByChecksumRaw, err := service.storage.BulkGetProjectRecordsByChecksum(ctx, authorizationHeader, organization, project, checksums) if err != nil { - return nil, nil, fmt.Errorf("lookup syfon project records by checksum: %w", err) + inflight.err = fmt.Errorf("lookup syfon project records by checksum: %w", err) + return nil, nil, inflight.err } usageByObjectID, err := service.storage.ListProjectFileUsage(ctx, authorizationHeader, organization, project, cleanupInactiveDays) if err != nil { - return nil, nil, fmt.Errorf("list syfon project file usage: %w", err) + inflight.err = fmt.Errorf("list syfon project file usage: %w", err) + return nil, nil, inflight.err } recordsByChecksum := make(map[string][]projectRecordState, len(recordsByChecksumRaw)) for _, records := range recordsByChecksumRaw { @@ -502,6 +627,8 @@ func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context usageByObjectID: usageByObjectID, } service.projectJoinMu.Unlock() + inflight.recordsByChecksum = recordsByChecksum + inflight.usageByObjectID = usageByObjectID return recordsByChecksum, usageByObjectID, nil } @@ -970,7 +1097,7 @@ func buildCleanupAuditModel(gitSubpath string, inventory []RepoInventoryFile, re findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) countsByKind[string(storageFindingKind)]++ case storageFindingBrokenBucketMap: - public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Fix or remove the Syfon access URL because no bucket mapping is configured for it") + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Syfon access URL did not resolve through a configured bucket mapping.") repairDeleteIDs, repairUpdates := brokenBucketMappingRepairPlan(matches) findings = append(findings, cleanupFindingModel{ Public: public, @@ -984,8 +1111,13 @@ func buildCleanupAuditModel(gitSubpath string, inventory []RepoInventoryFile, re findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) countsByKind[string(storageFindingKind)]++ case storageFindingValidationMismatch: - public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Storage metadata does not match the Syfon record") - findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Mapped bucket object exists, but object evidence disagrees with the Syfon/Git record.") + findings = append(findings, cleanupFindingModel{ + Public: public, + DeleteObjectIDs: recordObjectIDs(matches), + DeleteBucketObjects: cleanupMatchedBucketObjectURLs(matches, bucketObjectsByURL), + Manual: true, + }) countsByKind[string(storageFindingKind)]++ case storageFindingProbeError: public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Manual review required for storage probe errors") @@ -1132,6 +1264,7 @@ func buildCleanupFinding(kind string, normalizedPath string, matches []projectRe if cleanupScope == "access_url" { bucketEvaluation = "probed" } + actionability, availableActions, defaultAction, supportsDryRun := storageCleanupActionSupport(kind) return GitStorageCleanupFinding{ Kind: kind, NormalizedPath: normalizedPath, @@ -1145,6 +1278,10 @@ func buildCleanupFinding(kind string, normalizedPath string, matches []projectRe LastUpdated: formatOptionalTime(latestUpdate), DownloadCount: totalDownloads, LastDownload: formatOptionalTime(latestDownload), + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, Evidence: buildFindingEvidence(checksum, nil, matches, bucketEvaluation), } } @@ -1156,6 +1293,46 @@ func repoOrphanAction(kind string) string { return "Delete stale Syfon record" } +func storageCleanupActionSupport(kind string) (string, []string, string, bool) { + switch strings.TrimSpace(kind) { + case "broken_bucket_mapping": + return storageActionabilityAutoRepair, []string{ + storageActionRemoveBrokenAccessURLs, + storageActionDeleteSyfonRecord, + storageActionInspectEvidence, + }, storageActionRemoveBrokenAccessURLs, true + case "storage_validation_mismatch": + return storageActionabilityManualChoice, []string{ + storageActionDeleteSyfonRecord, + storageActionDeleteBucketObject, + storageActionDeleteBoth, + storageActionInspectEvidence, + }, "", true + default: + return storageActionabilityInspectOnly, []string{storageActionInspectEvidence}, storageActionInspectEvidence, false + } +} + +func storageChainActionSupport(kind string) (string, []string, string, bool) { + switch strings.TrimSpace(kind) { + case "syfon_broken_bucket_mapping": + return storageActionabilityAutoRepair, []string{ + storageActionRemoveBrokenAccessURLs, + storageActionDeleteSyfonRecord, + storageActionInspectEvidence, + }, storageActionRemoveBrokenAccessURLs, true + case "git_syfon_metadata_mismatch": + return storageActionabilityManualChoice, []string{ + storageActionDeleteSyfonRecord, + storageActionDeleteBucketObject, + storageActionDeleteBoth, + storageActionInspectEvidence, + }, "", true + default: + return storageActionabilityInspectOnly, []string{storageActionInspectEvidence}, storageActionInspectEvidence, false + } +} + func compareRecordState(left projectRecordState, right projectRecordState) int { if left.Usage.DownloadCount != right.Usage.DownloadCount { if left.Usage.DownloadCount > right.Usage.DownloadCount { @@ -1415,11 +1592,11 @@ func accessErrorForRecord(record projectRecordState) string { case storageFindingBrokenAccessURL: return "no usable access URL present" case storageFindingBrokenBucketMap: - return "no Syfon bucket mapping is configured for this access URL" + return "Syfon access URL did not resolve through a configured bucket mapping" case storageFindingObjectMissing: return "storage object not found" case storageFindingValidationMismatch: - return "storage metadata does not match the Syfon record" + return "mapped bucket object exists, but object evidence disagrees with the Syfon/Git record" case storageFindingProbeError: return "storage probe failed" } @@ -1428,6 +1605,7 @@ func accessErrorForRecord(record projectRecordState) string { func buildBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStorageCleanupFinding { objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL) + actionability, availableActions, defaultAction, supportsDryRun := storageCleanupActionSupport("bucket_only_object") return GitStorageCleanupFinding{ Kind: "bucket_only_object", NormalizedPath: objectURL, @@ -1438,6 +1616,10 @@ func buildBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStora CleanupScope: "bucket_object", SizeBytes: item.SizeBytes, LastUpdated: strings.TrimSpace(item.LastModified), + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, Evidence: &GitAuditEvidence{ AccessURLs: []string{objectURL}, Buckets: uniqueStrings([]string{item.Bucket}), @@ -1451,6 +1633,7 @@ func buildBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStora func buildChainBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStorageChainFinding { objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL) + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupport("bucket_only_object") return GitStorageChainFinding{ Kind: "bucket_only_object", NormalizedPath: objectURL, @@ -1463,6 +1646,10 @@ func buildChainBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) Git RecordCount: 0, SizeBytes: item.SizeBytes, RecommendedAction: "Bucket object exists, but no Syfon record matched it.", + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, Evidence: &GitAuditEvidence{ AccessURLs: []string{objectURL}, BucketObjectURLs: []string{objectURL}, @@ -1497,6 +1684,7 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, if evidence != nil { evidence.BucketObjectURLs = uniqueStrings(append(evidence.BucketObjectURLs, bucketObjectURLs...)) } + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupport(kind) primaryProbe := selectChainProbe(record, bucketObjectURLs) findings := make([]GitStorageChainFinding, 0, len(paths)) for _, path := range paths { @@ -1516,6 +1704,10 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, RecordCount: 1, SizeBytes: record.Size, RecommendedAction: action, + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, Evidence: evidence, }) } @@ -1556,11 +1748,11 @@ func chainFindingError(kind string, record projectRecordState, probe GitStorageC } switch kind { case "syfon_broken_bucket_mapping": - return "no configured Syfon bucket mapping matched this access URL" + return "Syfon access URL did not resolve through a configured bucket mapping" case "syfon_missing_bucket_object", "syfon_git_no_bucket": return "mapped bucket object was not found" case "git_syfon_metadata_mismatch": - return "bucket metadata did not match the Syfon record" + return "mapped bucket object exists, but object evidence disagrees with the Syfon/Git record" case "probe_error": return accessErrorForRecord(record) default: @@ -1590,6 +1782,24 @@ func matchedBucketObjectURLs(record projectRecordState, bucketObjectsByURL map[s return uniqueStrings(matches) } +func cleanupMatchedBucketObjectURLs(matches []projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []string { + objectURLs := make([]string, 0) + for _, match := range matches { + objectURLs = append(objectURLs, matchedBucketObjectURLs(match, bucketObjectsByURL)...) + for _, probe := range accessProbesForRecord(match) { + if strings.TrimSpace(probe.Status) != "present" { + continue + } + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.URL) + if strings.TrimSpace(objectURL) == "" { + continue + } + objectURLs = append(objectURLs, objectURL) + } + } + return uniqueStrings(objectURLs) +} + func hasCanonicalBucketURL(record projectRecordState) bool { return len(recordBucketURLs(record)) > 0 } diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index e52ebf8..898cc5c 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -947,7 +947,7 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum bucketMatches := matchedBucketObjectURLs(record, index.bucketObjectsByURL) switch classifyStorageFinding(record, index.bucketObjectsByURL) { case storageFindingBrokenBucketMap: - findings := buildChainRecordFindings("syfon_broken_bucket_mapping", record, gitPaths, bucketMatches, "Syfon record exists, but its access URL does not resolve to a configured bucket mapping.") + findings := buildChainRecordFindings("syfon_broken_bucket_mapping", record, gitPaths, bucketMatches, "Syfon access URL did not resolve through a configured bucket mapping.") acc.findings = append(acc.findings, findings...) acc.addCount("syfon_broken_bucket_mapping", chainPathCount(gitPaths)) case storageFindingObjectMissing: @@ -960,7 +960,7 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum acc.add("syfon_missing_bucket_object", buildChainRecordFindings("syfon_missing_bucket_object", record, nil, bucketMatches, "Syfon record points to a mapped bucket location, but the object does not exist.")...) case storageFindingValidationMismatch: if len(gitPaths) > 0 { - findings := buildChainRecordFindings("git_syfon_metadata_mismatch", record, gitPaths, bucketMatches, "Bucket object exists, but its metadata does not match what Syfon expects.") + findings := buildChainRecordFindings("git_syfon_metadata_mismatch", record, gitPaths, bucketMatches, "Mapped bucket object exists, but object evidence disagrees with the Syfon/Git record.") acc.findings = append(acc.findings, findings...) acc.addCount("git_syfon_metadata_mismatch", len(gitPaths)) continue @@ -988,6 +988,7 @@ func buildGitOriginChainFindings(index storageChainIndex, recordsByChecksum map[ if len(allProjectRecords[checksum]) > 0 || len(recordsByChecksum[checksum]) > 0 { continue } + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupport("git_only_no_syfon") acc.add("git_only_no_syfon", GitStorageChainFinding{ Kind: "git_only_no_syfon", NormalizedPath: item.RepoPath, @@ -997,6 +998,10 @@ func buildGitOriginChainFindings(index storageChainIndex, recordsByChecksum map[ RecordCount: 0, SizeBytes: item.Size, RecommendedAction: "Git checksum has no matching Syfon record. Bucket presence is not claimed by this finding.", + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, Evidence: &GitAuditEvidence{ Checksum: checksum, SourcePaths: []string{item.RepoPath}, diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 9c0e3a0..bc3f68e 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -24,6 +24,7 @@ type fakeStorageAnalyticsBackend struct { bucketScopes map[string][]domain.StorageBucketScope projectScopes []domain.StorageBucketScope usageByObject map[string]gintegrationsyfon.FileUsage + bulkChecksums []string probeResults map[string]gintegrationsyfon.BulkStorageProbeResult listProbeResults map[string]gintegrationsyfon.BulkStorageProbeResult bucketObjects []gintegrationsyfon.ProjectBucketObject @@ -38,6 +39,8 @@ type fakeStorageAnalyticsBackend struct { listProjectBucketObjectsPathPrefix string listProjectBucketSummaryCalls int listProjectBucketSummaryMode string + bulkGetProjectRecordsCalls int + bulkGetDelay time.Duration probeCalls int probeItems []gintegrationsyfon.BulkStorageProbeItem listProbeCalls int @@ -84,6 +87,11 @@ func (fake *fakeStorageAnalyticsBackend) ListProjectScopes(ctx context.Context, } func (fake *fakeStorageAnalyticsBackend) BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]gintegrationsyfon.ProjectRecord, error) { + fake.bulkGetProjectRecordsCalls++ + fake.bulkChecksums = append([]string(nil), checksums...) + if fake.bulkGetDelay > 0 { + time.Sleep(fake.bulkGetDelay) + } if fake.bulkRecords != nil { out := make(map[string][]gintegrationsyfon.ProjectRecord, len(fake.bulkRecords)) for checksum, records := range fake.bulkRecords { @@ -320,7 +328,7 @@ func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { assertHasCleanupFinding(t, cleanup.Findings, "broken_access_url_error", "data/c.txt") assertHasCleanupFinding(t, cleanup.Findings, "repo_orphan_stale_record", "s3://bucket/orphan") - applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true, true, true, false, false, true) + applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, nil, mirrorPath, repo, hash, true, true, true, false, false, true) if err != nil { t.Fatalf("apply cleanup dry run: %v", err) } @@ -389,7 +397,7 @@ func TestApplyStorageCleanupRepairsBrokenBucketMappings(t *testing.T) { } service := NewStorageAnalyticsService(backend) - applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", []string{"data/a.txt", "data/b.txt"}, mirrorPath, repo, hash, true, false, false, false, true, false) + applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", []string{"data/a.txt", "data/b.txt"}, nil, mirrorPath, repo, hash, true, false, false, false, true, false) if err != nil { t.Fatalf("apply cleanup broken bucket mapping repair: %v", err) } @@ -435,7 +443,7 @@ func TestApplyStorageCleanupDeletesBucketOnlyObjects(t *testing.T) { } service := NewStorageAnalyticsService(backend) - applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", []string{"s3://bucket/orphan"}, mirrorPath, repo, hash, true, false, false, true, false, false) + applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", []string{"s3://bucket/orphan"}, nil, mirrorPath, repo, hash, true, false, false, true, false, false) if err != nil { t.Fatalf("apply cleanup bucket only delete: %v", err) } @@ -509,6 +517,127 @@ func TestBuildStorageCleanupAuditReportsStorageProbeFailures(t *testing.T) { mismatchFinding.Evidence.BucketEvaluation != "probed" { t.Fatalf("expected storage evidence on mismatch finding, got %+v", mismatchFinding) } + if mismatchFinding.Actionability != "manual_choice" || + !contains(mismatchFinding.AvailableActions, "delete_syfon_record") || + !contains(mismatchFinding.AvailableActions, "delete_bucket_object") || + !contains(mismatchFinding.AvailableActions, "delete_both") || + mismatchFinding.DefaultAction != "" || + !mismatchFinding.SupportsDryRun { + t.Fatalf("expected mismatch action metadata, got %+v", mismatchFinding) + } +} + +func TestApplyStorageCleanupSupportsMetadataMismatchActions(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + now := time.Date(2026, 7, 2, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/b"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://bucket/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://bucket/b", + Bucket: "bucket", + Key: "b", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + deleteRecordOnly, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "data", + []string{"data/b.txt"}, + []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/b.txt", Action: "delete_syfon_record"}}, + mirrorPath, + repo, + hash, + true, + false, + false, + false, + false, + true, + ) + if err != nil { + t.Fatalf("apply cleanup delete syfon record dry run: %v", err) + } + if len(deleteRecordOnly.DeletedRecordIDs) != 1 || deleteRecordOnly.DeletedRecordIDs[0] != "obj-b" { + t.Fatalf("expected mismatch delete-record action to select Syfon record, got %+v", deleteRecordOnly) + } + if len(deleteRecordOnly.DeletedBucketObjectURLs) != 0 { + t.Fatalf("expected record-only action to avoid bucket delete, got %+v", deleteRecordOnly) + } + + deleteBucketOnly, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "data", + []string{"data/b.txt"}, + []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/b.txt", Action: "delete_bucket_object"}}, + mirrorPath, + repo, + hash, + true, + false, + false, + false, + false, + true, + ) + if err != nil { + t.Fatalf("apply cleanup delete bucket object dry run: %v", err) + } + if len(deleteBucketOnly.DeletedRecordIDs) != 0 { + t.Fatalf("expected bucket-only action to avoid Syfon delete, got %+v", deleteBucketOnly) + } + if len(deleteBucketOnly.DeletedBucketObjectURLs) != 1 || deleteBucketOnly.DeletedBucketObjectURLs[0] != "s3://bucket/b" { + t.Fatalf("expected mismatch delete-bucket action to select bucket object, got %+v", deleteBucketOnly) + } + + deleteBoth, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "data", + []string{"data/b.txt"}, + []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/b.txt", Action: "delete_both"}}, + mirrorPath, + repo, + hash, + true, + false, + false, + false, + false, + true, + ) + if err != nil { + t.Fatalf("apply cleanup delete both dry run: %v", err) + } + if len(deleteBoth.DeletedRecordIDs) != 1 || deleteBoth.DeletedRecordIDs[0] != "obj-b" { + t.Fatalf("expected dual delete action to include Syfon record, got %+v", deleteBoth) + } + if len(deleteBoth.DeletedBucketObjectURLs) != 1 || deleteBoth.DeletedBucketObjectURLs[0] != "s3://bucket/b" { + t.Fatalf("expected dual delete action to include bucket object, got %+v", deleteBoth) + } } func TestBuildStorageCleanupAuditFlagsRecordWhenAnyAccessProbeIsDead(t *testing.T) { @@ -600,7 +729,7 @@ func TestBuildStorageCleanupAuditReportsBrokenBucketMappingSeparately(t *testing assertHasCleanupFinding(t, cleanup.Findings, "broken_bucket_mapping", "data/a.txt") for _, finding := range cleanup.Findings { if finding.Kind == "broken_bucket_mapping" && finding.NormalizedPath == "data/a.txt" { - if len(finding.Records) != 1 || finding.Records[0].Error != "no Syfon bucket mapping is configured for this access URL" { + if len(finding.Records) != 1 || finding.Records[0].Error != "Syfon access URL did not resolve through a configured bucket mapping" { t.Fatalf("expected broken bucket mapping detail, got %+v", finding) } if finding.Evidence == nil || @@ -611,6 +740,12 @@ func TestBuildStorageCleanupAuditReportsBrokenBucketMappingSeparately(t *testing finding.Evidence.BucketEvaluation != "probed" { t.Fatalf("expected broken bucket mapping evidence, got %+v", finding) } + if finding.Actionability != "auto_repair" || + !contains(finding.AvailableActions, "remove_broken_access_urls") || + finding.DefaultAction != "remove_broken_access_urls" || + !finding.SupportsDryRun { + t.Fatalf("expected broken mapping action metadata, got %+v", finding) + } return } } @@ -1065,6 +1200,62 @@ func TestBuildStorageChainAuditCachesProjectRecordsPerPathPrefix(t *testing.T) { } } +func TestStorageSummaryAndChildrenShareInflightJoinWork(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 1}, + }, + bulkGetDelay: 75 * time.Millisecond, + } + service := NewStorageAnalyticsService(backend) + errCh := make(chan error, 2) + go func() { + _, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + errCh <- err + }() + go func() { + _, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1000, "bytes", "desc") + errCh <- err + }() + for i := 0; i < 2; i++ { + if err := <-errCh; err != nil { + t.Fatalf("expected concurrent storage analytics request to succeed, got %v", err) + } + } + if backend.bulkGetProjectRecordsCalls != 1 { + t.Fatalf("expected concurrent summary/children requests to share one checksum lookup, got %d", backend.bulkGetProjectRecordsCalls) + } + if backend.listProjectFileUsageCalls != 1 { + t.Fatalf("expected concurrent summary/children requests to share one usage lookup, got %d", backend.listProjectFileUsageCalls) + } +} + +func TestLoadProjectJoinCacheDeduplicatesChecksums(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/a"}}, + }, + } + service := NewStorageAnalyticsService(backend) + if _, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash); err != nil { + t.Fatalf("build storage summary: %v", err) + } + if len(backend.bulkChecksums) != 1 { + t.Fatalf("expected duplicate file checksums to be deduplicated before backend lookup, got %+v", backend.bulkChecksums) + } +} + func TestBuildStorageChainAuditForwardsBucketPathPrefixForExplicitInventory(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "README.md": "fixture", @@ -1142,6 +1333,77 @@ func TestBuildStorageChainAuditSurfacesMetadataMismatchAndEvidence(t *testing.T) finding.Evidence.ValidationStates[0] != "mismatched" { t.Fatalf("expected metadata mismatch evidence, got %+v", finding) } + if finding.Actionability != "manual_choice" || + !contains(finding.AvailableActions, "delete_syfon_record") || + !contains(finding.AvailableActions, "delete_bucket_object") || + !contains(finding.AvailableActions, "delete_both") || + finding.DefaultAction != "" || + !finding.SupportsDryRun { + t.Fatalf("expected metadata mismatch chain actions, got %+v", finding) + } +} + +func TestBuildStorageChainAuditFiltersFindingsByKind(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/bad-map.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + now := time.Date(2026, 7, 2, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-bad-map", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://legacy-bucket/bad-map.txt"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 999}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + storageProbeRequestKey("s3://legacy-bucket/bad-map.txt", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://legacy-bucket/bad-map.txt", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://legacy-bucket/bad-map.txt", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "legacy-bucket"`, + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ + FindingKind: "syfon_broken_bucket_mapping", + FindingLimit: 1, + }) + if err != nil { + t.Fatalf("build filtered chain audit: %v", err) + } + if chain.Summary.TotalFindings != 1 || chain.Summary.ReturnedFindings != 1 || chain.Summary.FindingsTruncated { + t.Fatalf("expected one filtered finding without truncation, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { + t.Fatalf("expected filtered broken mapping count, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["git_syfon_metadata_mismatch"]; got != 0 { + t.Fatalf("expected mismatch count to be excluded from filtered response, got %+v", chain.Summary) + } + if len(chain.Groups) != 1 || chain.Groups[0].Kind != "syfon_broken_bucket_mapping" { + t.Fatalf("expected one filtered group, got %+v", chain.Groups) + } + if len(chain.Findings) != 1 || chain.Findings[0].Kind != "syfon_broken_bucket_mapping" || chain.Findings[0].NormalizedPath != "data/bad-map.txt" { + t.Fatalf("expected filtered broken mapping detail row, got %+v", chain.Findings) + } } func TestBuildStorageChainAuditUsesScopedProjectRecordsForGitJoin(t *testing.T) { diff --git a/internal/git/storage_analytics_timing.go b/internal/git/storage_analytics_timing.go index 955ee97..5d63f52 100644 --- a/internal/git/storage_analytics_timing.go +++ b/internal/git/storage_analytics_timing.go @@ -23,6 +23,7 @@ type StorageChainAuditOptions struct { ValidationMode string BucketInventoryMode string BucketPathPrefix string + FindingKind string FindingLimit int Timings *StorageChainAuditTimings } diff --git a/internal/git/storage_index.go b/internal/git/storage_index.go index 795cbe4..48fc234 100644 --- a/internal/git/storage_index.go +++ b/internal/git/storage_index.go @@ -21,12 +21,20 @@ import ( const repoAnalyticsIndexSchemaVersion = 1 var repoAnalyticsIndexCache = &repoAnalyticsIndexMemoryCache{ - entries: map[string]*repoAnalyticsIndex{}, + entries: map[string]*repoAnalyticsIndex{}, + inflight: map[string]*inflightRepoAnalyticsIndex{}, } type repoAnalyticsIndexMemoryCache struct { - mu sync.RWMutex - entries map[string]*repoAnalyticsIndex + mu sync.RWMutex + entries map[string]*repoAnalyticsIndex + inflight map[string]*inflightRepoAnalyticsIndex +} + +type inflightRepoAnalyticsIndex struct { + done chan struct{} + index *repoAnalyticsIndex + err error } type repoAnalyticsIndex struct { @@ -81,23 +89,45 @@ func PersistRepoAnalyticsIndex(ctx context.Context, mirrorPath string, repo *gog } func loadOrBuildRepoAnalyticsIndex(_ context.Context, mirrorPath string, ref string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, error) { - if cached := repoAnalyticsIndexCache.get(mirrorPath, hash); cached != nil { + cacheKey := repoAnalyticsCacheKey(mirrorPath, hash) + repoAnalyticsIndexCache.mu.Lock() + if cached := repoAnalyticsIndexCache.entries[cacheKey]; cached != nil { + repoAnalyticsIndexCache.mu.Unlock() return cached, nil } + if inflight := repoAnalyticsIndexCache.inflight[cacheKey]; inflight != nil { + repoAnalyticsIndexCache.mu.Unlock() + <-inflight.done + return inflight.index, inflight.err + } + inflight := &inflightRepoAnalyticsIndex{done: make(chan struct{})} + repoAnalyticsIndexCache.inflight[cacheKey] = inflight + repoAnalyticsIndexCache.mu.Unlock() + defer func() { + repoAnalyticsIndexCache.mu.Lock() + delete(repoAnalyticsIndexCache.inflight, cacheKey) + close(inflight.done) + repoAnalyticsIndexCache.mu.Unlock() + }() + sidecar, err := readRepoAnalyticsIndexSidecar(mirrorPath) if err == nil && sidecar.SchemaVersion == repoAnalyticsIndexSchemaVersion && sidecar.CommitHash == hash.String() { index := repoAnalyticsIndexFromSidecar(sidecar) repoAnalyticsIndexCache.put(mirrorPath, hash, index) + inflight.index = index return index, nil } index, buildErr := buildRepoAnalyticsIndex(ref, repo, hash) if buildErr != nil { + inflight.err = buildErr return nil, buildErr } if writeErr := writeRepoAnalyticsIndexSidecar(mirrorPath, index.sidecar); writeErr != nil { + inflight.err = writeErr return nil, writeErr } repoAnalyticsIndexCache.put(mirrorPath, hash, index) + inflight.index = index return index, nil } diff --git a/internal/git/types.go b/internal/git/types.go index 4fdd832..cd8c8c5 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -398,6 +398,7 @@ type GitStorageChainAuditRequest struct { ValidationMode string `json:"validation_mode,omitempty"` BucketInventoryMode string `json:"bucket_inventory_mode,omitempty"` BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` + FindingKind string `json:"finding_kind,omitempty"` FindingLimit int `json:"finding_limit,omitempty"` } @@ -417,6 +418,10 @@ type GitStorageChainFinding struct { RecordCount int `json:"record_count"` SizeBytes int64 `json:"size_bytes,omitempty"` RecommendedAction string `json:"recommended_action"` + Actionability string `json:"actionability,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + SupportsDryRun bool `json:"supports_dry_run,omitempty"` Evidence *GitAuditEvidence `json:"evidence,omitempty"` } @@ -503,9 +508,19 @@ type GitStorageCleanupFinding struct { LastUpdated string `json:"last_updated,omitempty"` DownloadCount int64 `json:"download_count,omitempty"` LastDownload string `json:"last_download_time,omitempty"` + Actionability string `json:"actionability,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + SupportsDryRun bool `json:"supports_dry_run,omitempty"` Evidence *GitAuditEvidence `json:"evidence,omitempty"` } +type GitStorageCleanupApplyAction struct { + Action string `json:"action"` + Kind string `json:"kind,omitempty"` + NormalizedPath string `json:"normalized_path,omitempty"` +} + type GitStorageCleanupAuditSummary struct { CountsByKind map[string]int `json:"counts_by_kind"` TotalFindings int `json:"total_findings"` @@ -524,14 +539,15 @@ type GitStorageCleanupAuditResponse struct { } type GitStorageCleanupApplyRequest struct { - GitSubpath string `json:"git_subpath,omitempty"` - Ref string `json:"ref,omitempty"` - DeleteRepoOrphans bool `json:"delete_repo_orphans,omitempty"` - DeleteStaleDuplicates bool `json:"delete_stale_duplicates,omitempty"` - DeleteBucketOnlyObjects bool `json:"delete_bucket_only_objects,omitempty"` - RepairBrokenBucketMappings bool `json:"repair_broken_bucket_mappings,omitempty"` - DryRun bool `json:"dry_run,omitempty"` - SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + DeleteRepoOrphans bool `json:"delete_repo_orphans,omitempty"` + DeleteStaleDuplicates bool `json:"delete_stale_duplicates,omitempty"` + DeleteBucketOnlyObjects bool `json:"delete_bucket_only_objects,omitempty"` + RepairBrokenBucketMappings bool `json:"repair_broken_bucket_mappings,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` + Actions []GitStorageCleanupApplyAction `json:"actions,omitempty"` } type GitStorageCleanupPurgeResult struct { diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index a13a302..9405147 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -182,7 +182,8 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err } projectCtx.applyRequestRef(requestBody.Ref) gitSubpath := normalizeAnalyticsSubpath(requestBody.GitSubpath) - timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_prefix=%q finding_limit=%d", projectCtx.projectID, projectCtx.refName, gitSubpath, validationMode, probeMode, bucketMode, bucketPathPrefix, findingLimit) + findingKind := strings.TrimSpace(requestBody.FindingKind) + timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_prefix=%q finding_kind=%q finding_limit=%d", projectCtx.projectID, projectCtx.refName, gitSubpath, validationMode, probeMode, bucketMode, bucketPathPrefix, findingKind, findingLimit) handler.logger.Info("storage_chain_audit_request_start %s", timings.DebugPrefix) response, err := handler.storageAnalytics.BuildStorageChainAuditWithOptions( ctx.Context(), @@ -199,6 +200,7 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err ValidationMode: validationMode, BucketInventoryMode: bucketMode, BucketPathPrefix: bucketPathPrefix, + FindingKind: findingKind, FindingLimit: findingLimit, Timings: timings, }, @@ -239,6 +241,11 @@ func (handler *Handler) handleGitProjectStorageCleanupApplyPOST(ctx fiber.Ctx) e projectCtx.applyRequestRef(requestBody.Ref) requestBody.GitSubpath = normalizeAnalyticsSubpath(requestBody.GitSubpath) requestBody.SelectedRepoPaths = normalizeAnalyticsPathList(requestBody.SelectedRepoPaths) + for i := range requestBody.Actions { + requestBody.Actions[i].NormalizedPath = normalizeAnalyticsSubpath(requestBody.Actions[i].NormalizedPath) + requestBody.Actions[i].Kind = strings.TrimSpace(requestBody.Actions[i].Kind) + requestBody.Actions[i].Action = strings.TrimSpace(requestBody.Actions[i].Action) + } response, err := handler.storageAnalytics.ApplyStorageCleanup( ctx.Context(), projectCtx.authorizationHeader, @@ -247,6 +254,7 @@ func (handler *Handler) handleGitProjectStorageCleanupApplyPOST(ctx fiber.Ctx) e projectCtx.refName, requestBody.GitSubpath, requestBody.SelectedRepoPaths, + requestBody.Actions, projectCtx.mirrorPath, projectCtx.repo, projectCtx.hash, From fb1fce18dd5574c98feb8b23276f85f380fd377f Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 3 Jul 2026 13:43:14 -0700 Subject: [PATCH 17/36] fix gecko audit fix wiring to match audit outputs --- internal/git/storage_analytics.go | 677 ++++++++++++++---- internal/git/storage_analytics_test.go | 631 +++++++++++++++- internal/git/storage_children.go | 191 +++++ internal/git/types.go | 108 +-- internal/integrations/syfon/adapter.go | 23 +- internal/integrations/syfon/adapter_test.go | 54 ++ internal/server/http/git/storage_analytics.go | 124 +++- .../git/storage_analytics_options_test.go | 102 +++ 8 files changed, 1664 insertions(+), 246 deletions(-) create mode 100644 internal/git/storage_children.go create mode 100644 internal/server/http/git/storage_analytics_options_test.go diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 0f543ce..2690e04 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -33,6 +33,54 @@ const ( storageActionInspectEvidence = "inspect_evidence" ) +type storageRepairPolicy struct { + actionability string + actions []string + defaultAction string + supportsDryRun bool +} + +func storageRepairPolicyForKind(kind string) storageRepairPolicy { + switch strings.TrimSpace(kind) { + case "bucket_only_object": + return autoRepairPolicy(storageActionDeleteBucketObject, storageActionInspectEvidence) + case "bucket_syfon_no_git", "repo_orphan_live_object": + return autoRepairPolicy(storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionInspectEvidence) + case "repo_orphan_stale_record", "stale_duplicate_record", "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object": + return autoRepairPolicy(storageActionDeleteSyfonRecord, storageActionInspectEvidence) + case "broken_access_url_error", "broken_bucket_mapping", "syfon_broken_bucket_mapping": + return autoRepairPolicy(storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord, storageActionInspectEvidence) + case "storage_validation_mismatch", "git_syfon_metadata_mismatch": + return storageRepairPolicy{ + actionability: storageActionabilityManualChoice, + actions: []string{ + storageActionDeleteSyfonRecord, + storageActionDeleteBucketObject, + storageActionDeleteBoth, + storageActionInspectEvidence, + }, + supportsDryRun: true, + } + default: + return storageRepairPolicy{ + actionability: storageActionabilityInspectOnly, + actions: []string{storageActionInspectEvidence}, + defaultAction: storageActionInspectEvidence, + supportsDryRun: false, + } + } +} + +func autoRepairPolicy(defaultAction string, extraActions ...string) storageRepairPolicy { + actions := append([]string{defaultAction}, extraActions...) + return storageRepairPolicy{ + actionability: storageActionabilityAutoRepair, + actions: uniqueStrings(actions), + defaultAction: defaultAction, + supportsDryRun: true, + } +} + type storageAnalyticsBackend interface { ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) @@ -113,6 +161,17 @@ type cleanupFindingModel struct { Manual bool } +type storageCleanupApplyPlan struct { + DeleteObjectIDs []string + DeleteStorageObjectIDs []string + DeleteBucketObjects []string + UpdateAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod + UpdatedRecordIDs []string + RepoDeletePaths []string + ManualPaths []string + SkippedPaths []string +} + type cleanupAuditModel struct { Findings []cleanupFindingModel PublicFindings []GitStorageCleanupFinding @@ -181,8 +240,8 @@ func (service *StorageAnalyticsService) BuildStorageSummary(ctx context.Context, }, nil } -func (service *StorageAnalyticsService) BuildStorageChildren(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string) (*GitStorageChildrenResponse, error) { - index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) +func (service *StorageAnalyticsService) BuildStorageChildren(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string, cursor string) (*GitStorageChildrenResponse, error) { + index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) if err != nil { return nil, err } @@ -190,26 +249,22 @@ func (service *StorageAnalyticsService) BuildStorageChildren(ctx context.Context if err != nil { return nil, err } - aggregates := aggregateImmediateChildren(gitSubpath, inventory, recordsByChecksum, usageByObjectID, cloneDirectoryChildren(directory.Children)) + aggregates := cloneDirectoryChildren(directory.Children) sortStorageAggregates(aggregates, sortBy, sortOrder) - if limit > 0 && len(aggregates) > limit { - aggregates = aggregates[:limit] - } - items := make([]GitStorageChildResponseItem, 0, len(aggregates)) - for _, agg := range aggregates { - items = append(items, GitStorageChildResponseItem{ - Name: agg.name, - Path: agg.path, - Type: agg.rowType, - FileCount: agg.fileCount, - RecordCount: agg.recordCount, - TotalBytes: agg.totalBytes, - DownloadCount: agg.downloadCount, - LastDownloadTime: formatOptionalTime(agg.lastDownload), - LatestUpdateTime: formatOptionalTime(agg.latestUpdate), - }) + page, err := storageChildrenPageForRequest(aggregates, hash, gitSubpath, sortBy, sortOrder, limit, cursor) + if err != nil { + return nil, err + } + inventory := filterInventoryForStorageChildren(index.sidecar.Files, page.items) + enriched, err := service.enrichStorageChildrenPage(ctx, authorizationHeader, organization, project, gitSubpath, inventory, page.items) + if err != nil { + return nil, err } - return &GitStorageChildrenResponse{Items: items}, nil + return &GitStorageChildrenResponse{ + Items: storageChildrenItemsFromAggregates(enriched), + HasMore: page.hasMore, + NextCursor: page.nextCursor, + }, nil } func (service *StorageAnalyticsService) BuildProjectDiffAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitProjectDiffAuditResponse, error) { @@ -355,82 +410,280 @@ func limitStorageChainFindings(findings []GitStorageChainFinding, limit int) []G return append([]GitStorageChainFinding(nil), findings[:limit]...) } -func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, selectedRepoPaths []string, selectedActions []GitStorageCleanupApplyAction, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, checkStorage bool, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool, dryRun bool) (*GitStorageCleanupApplyResponse, error) { - _, model, err := service.BuildStorageCleanupAudit(ctx, authorizationHeader, organization, project, ref, gitSubpath, selectedRepoPaths, mirrorPath, repo, hash, checkStorage) - if err != nil { - return nil, err +func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, authorizationHeader string, organization string, project string, selectedRepoPaths []string, selectedActions []GitStorageCleanupApplyAction, selectedFindings []GitStorageCleanupApplyFinding, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool, dryRun bool) (*GitStorageCleanupApplyResponse, error) { + if len(selectedFindings) == 0 { + return nil, fmt.Errorf("cleanup apply requires findings from a prior audit; refusing to rebuild audit during apply") } actionSelection := indexCleanupActions(selectedActions) - toDelete := make([]string, 0) - toDeleteWithStorage := make([]string, 0) - toDeleteBucketObjects := make([]string, 0) - toUpdate := make(map[string][]gintegrationsyfon.ProjectAccessMethod) - repoDeletePaths := make([]string, 0) - deletedBucketObjectURLs := make([]string, 0) - updatedRecordIDs := make([]string, 0) - manualPaths := make([]string, 0) - skippedPaths := make([]string, 0) - for _, finding := range model.Findings { - switch finding.Public.Kind { - case "repo_orphan_live_object", "repo_orphan_stale_record": - if deleteRepoOrphans { - toDelete = append(toDelete, finding.DeleteObjectIDs...) - if finding.DeleteStorageData { - toDeleteWithStorage = append(toDeleteWithStorage, finding.DeleteObjectIDs...) - } - repoDeletePaths = append(repoDeletePaths, finding.Public.NormalizedPath) - } else { - skippedPaths = append(skippedPaths, finding.Public.NormalizedPath) - } - case "stale_duplicate_record": - if deleteStaleDuplicates { - toDelete = append(toDelete, finding.DeleteObjectIDs...) - } else { - skippedPaths = append(skippedPaths, finding.Public.NormalizedPath) - } - case "broken_bucket_mapping": - if repairBrokenBucketMappings { - toDelete = append(toDelete, finding.DeleteObjectIDs...) - for objectID, methods := range finding.UpdateAccessMethods { - if trimmed := strings.TrimSpace(objectID); trimmed != "" { - toUpdate[trimmed] = append([]gintegrationsyfon.ProjectAccessMethod(nil), methods...) - updatedRecordIDs = append(updatedRecordIDs, trimmed) - } - } - } else { - skippedPaths = append(skippedPaths, finding.Public.NormalizedPath) - } - case "bucket_only_object": - if deleteBucketOnlyObjects { - toDeleteBucketObjects = append(toDeleteBucketObjects, finding.DeleteBucketObjects...) - } else { - skippedPaths = append(skippedPaths, finding.Public.NormalizedPath) - } - case "storage_validation_mismatch": - switch cleanupActionForFinding(actionSelection, finding.Public) { - case storageActionDeleteSyfonRecord: - toDelete = append(toDelete, finding.DeleteObjectIDs...) - case storageActionDeleteBucketObject: - toDeleteBucketObjects = append(toDeleteBucketObjects, finding.DeleteBucketObjects...) - case storageActionDeleteBoth: - toDelete = append(toDelete, finding.DeleteObjectIDs...) - toDeleteBucketObjects = append(toDeleteBucketObjects, finding.DeleteBucketObjects...) - default: - manualPaths = append(manualPaths, finding.Public.NormalizedPath) + plan := storageCleanupApplyPlan{ + UpdateAccessMethods: make(map[string][]gintegrationsyfon.ProjectAccessMethod), + } + selected := indexCleanupSelection(selectedRepoPaths) + for _, finding := range selectedFindings { + if len(selected) > 0 && !storageApplyFindingSelected(selected, finding) { + continue + } + action, err := resolveStorageCleanupApplyAction(finding, actionSelection, deleteRepoOrphans, deleteStaleDuplicates, deleteBucketOnlyObjects, repairBrokenBucketMappings) + if err != nil { + return nil, err + } + if err := addStorageCleanupApplyFindingToPlan(&plan, finding, action); err != nil { + return nil, err + } + } + if len(selectedRepoPaths) > 0 && len(plan.DeleteObjectIDs) == 0 && len(plan.DeleteStorageObjectIDs) == 0 && len(plan.DeleteBucketObjects) == 0 && len(plan.UpdateAccessMethods) == 0 && len(plan.ManualPaths) == 0 && len(plan.SkippedPaths) == 0 { + return nil, fmt.Errorf("selected cleanup paths did not match provided cleanup findings") + } + return service.executeStorageCleanupApplyPlan(ctx, authorizationHeader, organization, project, plan, dryRun) +} + +func resolveStorageCleanupApplyAction(finding GitStorageCleanupApplyFinding, actionSelection map[string]string, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool) (string, error) { + kind := strings.TrimSpace(finding.Kind) + if !knownStorageRepairKind(kind) { + return "", fmt.Errorf("unsupported cleanup finding kind %q", kind) + } + action := cleanupActionForApplyFinding(actionSelection, finding) + if action == "" { + action = legacyDefaultStorageCleanupAction(kind, deleteRepoOrphans, deleteStaleDuplicates, deleteBucketOnlyObjects, repairBrokenBucketMappings) + } + if action == "" { + action = strings.TrimSpace(finding.DefaultAction) + } + if action == "" { + action = storageRepairPolicyForKind(kind).defaultAction + } + if action == storageActionInspectEvidence { + return action, nil + } + if !storageRepairActionAllowed(kind, action) { + return "", fmt.Errorf("cleanup action %q is not supported for finding kind %q", action, kind) + } + if len(finding.AvailableActions) > 0 && !stringSliceContains(finding.AvailableActions, action) { + return "", fmt.Errorf("cleanup action %q is not advertised for finding kind %q at %q", action, kind, finding.NormalizedPath) + } + return action, nil +} + +func knownStorageRepairKind(kind string) bool { + switch strings.TrimSpace(kind) { + case "bucket_only_object", "bucket_syfon_no_git", + "repo_orphan_live_object", "repo_orphan_stale_record", + "stale_duplicate_record", "live_duplicate_conflict", + "broken_access_url_error", "broken_bucket_mapping", "syfon_broken_bucket_mapping", + "storage_validation_mismatch", "git_syfon_metadata_mismatch", + "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object", + "git_only_no_syfon", "storage_probe_error", "probe_error": + return true + default: + return false + } +} + +func legacyDefaultStorageCleanupAction(kind string, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool) string { + switch strings.TrimSpace(kind) { + case "repo_orphan_live_object", "repo_orphan_stale_record": + if deleteRepoOrphans { + return storageRepairPolicyForKind(kind).defaultAction + } + case "bucket_syfon_no_git": + if deleteRepoOrphans { + return storageActionDeleteBoth + } + case "stale_duplicate_record": + if deleteStaleDuplicates { + return storageActionDeleteSyfonRecord + } + case "bucket_only_object": + if deleteBucketOnlyObjects { + return storageActionDeleteBucketObject + } + case "broken_access_url_error", "broken_bucket_mapping", "syfon_broken_bucket_mapping": + if repairBrokenBucketMappings { + return storageActionRemoveBrokenAccessURLs + } + } + return "" +} + +func storageRepairActionAllowed(kind string, action string) bool { + for _, allowed := range storageRepairPolicyForKind(kind).actions { + if action == allowed { + return true + } + } + return false +} + +func stringSliceContains(values []string, target string) bool { + for _, value := range values { + if strings.TrimSpace(value) == target { + return true + } + } + return false +} + +func addStorageCleanupApplyFindingToPlan(plan *storageCleanupApplyPlan, finding GitStorageCleanupApplyFinding, action string) error { + switch action { + case storageActionInspectEvidence: + plan.ManualPaths = append(plan.ManualPaths, finding.NormalizedPath) + case storageActionDeleteSyfonRecord: + objectIDs := storageApplyFindingObjectIDs(finding) + if len(objectIDs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing object_ids for %s", finding.Kind, finding.NormalizedPath, action) + } + plan.DeleteObjectIDs = append(plan.DeleteObjectIDs, objectIDs...) + plan.RepoDeletePaths = append(plan.RepoDeletePaths, finding.NormalizedPath) + case storageActionDeleteBucketObject: + bucketURLs := storageApplyFindingBucketObjectURLs(finding) + if len(bucketURLs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing bucket object URLs for %s", finding.Kind, finding.NormalizedPath, action) + } + plan.DeleteBucketObjects = append(plan.DeleteBucketObjects, bucketURLs...) + case storageActionDeleteBoth: + objectIDs := storageApplyFindingObjectIDs(finding) + bucketURLs := storageApplyFindingBucketObjectURLs(finding) + if len(objectIDs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing object_ids for %s", finding.Kind, finding.NormalizedPath, action) + } + if strings.TrimSpace(finding.Kind) == "repo_orphan_live_object" { + plan.DeleteObjectIDs = append(plan.DeleteObjectIDs, objectIDs...) + plan.DeleteStorageObjectIDs = append(plan.DeleteStorageObjectIDs, objectIDs...) + plan.RepoDeletePaths = append(plan.RepoDeletePaths, finding.NormalizedPath) + return nil + } + if len(bucketURLs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing bucket object URLs for %s", finding.Kind, finding.NormalizedPath, action) + } + plan.DeleteObjectIDs = append(plan.DeleteObjectIDs, objectIDs...) + plan.DeleteBucketObjects = append(plan.DeleteBucketObjects, bucketURLs...) + plan.RepoDeletePaths = append(plan.RepoDeletePaths, finding.NormalizedPath) + case storageActionRemoveBrokenAccessURLs: + return addAccessMethodRepairToPlan(plan, finding) + default: + return fmt.Errorf("unsupported cleanup action %q for finding kind %q", action, finding.Kind) + } + return nil +} + +func storageApplyFindingObjectIDs(finding GitStorageCleanupApplyFinding) []string { + objectIDs := append([]string(nil), finding.ObjectIDs...) + for _, record := range finding.Records { + objectIDs = append(objectIDs, record.ObjectID) + } + return uniqueStrings(objectIDs) +} + +func addAccessMethodRepairToPlan(plan *storageCleanupApplyPlan, finding GitStorageCleanupApplyFinding) error { + records := finding.Records + if len(records) == 0 { + for _, objectID := range finding.ObjectIDs { + records = append(records, GitStorageCleanupRecordAudit{ + ObjectID: objectID, + AccessURLs: append([]string(nil), finding.AccessURLs...), + }) + } + } + if len(records) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing records for %s", finding.Kind, finding.NormalizedPath, storageActionRemoveBrokenAccessURLs) + } + for _, record := range records { + objectID := strings.TrimSpace(record.ObjectID) + if objectID == "" { + return fmt.Errorf("cleanup finding %q at %q has a record without object_id", finding.Kind, finding.NormalizedPath) + } + if len(brokenAccessURLsForRecord(record)) == 0 { + return fmt.Errorf("cleanup finding %q at %q record %q is missing broken access URL evidence", finding.Kind, finding.NormalizedPath, objectID) + } + remaining := remainingAccessMethodsAfterBrokenRemoval(record) + if len(remaining) == 0 { + plan.DeleteObjectIDs = append(plan.DeleteObjectIDs, objectID) + plan.RepoDeletePaths = append(plan.RepoDeletePaths, finding.NormalizedPath) + continue + } + plan.UpdateAccessMethods[objectID] = remaining + plan.UpdatedRecordIDs = append(plan.UpdatedRecordIDs, objectID) + } + return nil +} + +func remainingAccessMethodsAfterBrokenRemoval(record GitStorageCleanupRecordAudit) []gintegrationsyfon.ProjectAccessMethod { + methods := projectAccessMethodsFromCleanupMethods(record.AccessMethods) + if len(methods) == 0 { + for _, accessURL := range record.AccessURLs { + if trimmed := strings.TrimSpace(accessURL); trimmed != "" { + methods = append(methods, gintegrationsyfon.ProjectAccessMethod{URL: trimmed}) } - default: - manualPaths = append(manualPaths, finding.Public.NormalizedPath) } } - toDelete = uniqueStrings(toDelete) - toDeleteWithStorage = uniqueStrings(toDeleteWithStorage) + brokenURLs := brokenAccessURLsForRecord(record) + remaining := make([]gintegrationsyfon.ProjectAccessMethod, 0, len(methods)) + for _, method := range methods { + url := strings.TrimSpace(method.URL) + if url == "" { + continue + } + if _, broken := brokenURLs[normalizeCleanupSelectionKey(url)]; broken { + continue + } + remaining = append(remaining, method) + } + return remaining +} + +func brokenAccessURLsForRecord(record GitStorageCleanupRecordAudit) map[string]struct{} { + broken := make(map[string]struct{}) + for _, probe := range record.AccessProbes { + url := normalizeCleanupSelectionKey(probe.URL) + if url == "" { + continue + } + if accessProbeIsBroken(probe) { + broken[url] = struct{}{} + } + } + for _, accessURL := range record.AccessURLs { + if strings.TrimSpace(accessURL) == "" { + continue + } + if len(record.AccessProbes) == 0 && recordStatusMeansBrokenAccess(record.Status) { + broken[normalizeCleanupSelectionKey(accessURL)] = struct{}{} + } + } + return broken +} + +func recordStatusMeansBrokenAccess(status string) bool { + switch strings.TrimSpace(status) { + case "missing", "error": + return true + default: + return false + } +} + +func accessProbeIsBroken(probe GitStorageCleanupAccessProbe) bool { + switch strings.TrimSpace(probe.ErrorKind) { + case "missing_access_url", "credential_missing": + return true + } + switch strings.TrimSpace(probe.Status) { + case "missing", "forbidden", "unsupported", "invalid", "error": + return true + } + return false +} + +func (service *StorageAnalyticsService) executeStorageCleanupApplyPlan(ctx context.Context, authorizationHeader string, organization string, project string, plan storageCleanupApplyPlan, dryRun bool) (*GitStorageCleanupApplyResponse, error) { + toDelete := uniqueStrings(plan.DeleteObjectIDs) + toDeleteWithStorage := uniqueStrings(plan.DeleteStorageObjectIDs) toDeleteMetadataOnly := differenceStrings(toDelete, toDeleteWithStorage) - toDeleteBucketObjects = uniqueStrings(toDeleteBucketObjects) - repoDeletePaths = uniqueStrings(repoDeletePaths) - deletedBucketObjectURLs = uniqueStrings(deletedBucketObjectURLs) - updatedRecordIDs = uniqueStrings(updatedRecordIDs) - manualPaths = uniqueStrings(manualPaths) - skippedPaths = uniqueStrings(skippedPaths) + toDeleteBucketObjects := uniqueStrings(plan.DeleteBucketObjects) + repoDeletePaths := uniqueStrings(plan.RepoDeletePaths) + deletedBucketObjectURLs := make([]string, 0) + updatedRecordIDs := uniqueStrings(plan.UpdatedRecordIDs) + manualPaths := uniqueStrings(plan.ManualPaths) + skippedPaths := uniqueStrings(plan.SkippedPaths) purgeResults := make([]GitStorageCleanupPurgeResult, 0, len(toDelete)+len(toDeleteBucketObjects)) if dryRun { for _, objectID := range toDelete { @@ -451,8 +704,8 @@ func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, DryRun: true, }, nil } - if len(toUpdate) > 0 { - if err := service.storage.BulkUpdateAccessMethods(ctx, authorizationHeader, toUpdate); err != nil { + if len(plan.UpdateAccessMethods) > 0 { + if err := service.storage.BulkUpdateAccessMethods(ctx, authorizationHeader, plan.UpdateAccessMethods); err != nil { return nil, fmt.Errorf("update syfon access methods: %w", err) } } @@ -485,7 +738,7 @@ func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, } deletedBucketObjectURLs = uniqueStrings(deletedBucketObjectURLs) } - if len(toUpdate) > 0 || len(toDelete) > 0 { + if len(plan.UpdateAccessMethods) > 0 || len(toDelete) > 0 { service.evictProjectJoinCache(organization, project) } for _, objectID := range toDelete { @@ -508,6 +761,71 @@ func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, }, nil } +func indexCleanupSelection(paths []string) map[string]struct{} { + index := make(map[string]struct{}, len(paths)) + for _, path := range paths { + normalized := normalizeCleanupSelectionKey(path) + if normalized == "" { + continue + } + index[normalized] = struct{}{} + } + return index +} + +func storageApplyFindingSelected(selected map[string]struct{}, finding GitStorageCleanupApplyFinding) bool { + for _, candidate := range storageApplyFindingSelectionCandidates(finding) { + if _, ok := selected[candidate]; ok { + return true + } + } + return false +} + +func storageApplyFindingSelectionCandidates(finding GitStorageCleanupApplyFinding) []string { + candidates := []string{ + finding.NormalizedPath, + finding.BucketObjectURL, + } + candidates = append(candidates, finding.ObjectIDs...) + candidates = append(candidates, finding.BucketObjectURLs...) + candidates = append(candidates, finding.AccessURLs...) + if finding.Evidence != nil { + candidates = append(candidates, finding.Evidence.AccessURLs...) + candidates = append(candidates, finding.Evidence.BucketObjectURLs...) + candidates = append(candidates, finding.Evidence.ObjectIDs...) + } + return uniqueCleanupSelectionCandidates(candidates) +} + +func storageApplyFindingBucketObjectURLs(finding GitStorageCleanupApplyFinding) []string { + candidates := []string{ + finding.BucketObjectURL, + } + if canonicalStorageURL("", "", finding.NormalizedPath) != "" { + candidates = append(candidates, finding.NormalizedPath) + } + candidates = append(candidates, finding.BucketObjectURLs...) + candidates = append(candidates, finding.AccessURLs...) + if finding.Evidence != nil { + candidates = append(candidates, finding.Evidence.BucketObjectURLs...) + candidates = append(candidates, finding.Evidence.AccessURLs...) + } + return uniqueCleanupSelectionCandidates(candidates) +} + +func cleanupActionForApplyFinding(index map[string]string, finding GitStorageCleanupApplyFinding) string { + for _, path := range storageApplyFindingSelectionCandidates(finding) { + if action := strings.TrimSpace(index[cleanupActionKey(finding.Kind, path)]); action != "" { + return action + } + if action := strings.TrimSpace(index[cleanupActionKey("", path)]); action != "" { + return action + } + } + return "" +} + func indexCleanupActions(actions []GitStorageCleanupApplyAction) map[string]string { if len(actions) == 0 { return nil @@ -1031,16 +1349,20 @@ func buildProjectDiffAuditModel(gitSubpath string, inventory []RepoInventoryFile func buildCleanupAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, selectedRepoPaths []string, checkStorage bool) *cleanupAuditModel { allowed := make(map[string]struct{}, len(selectedRepoPaths)) for _, path := range selectedRepoPaths { - if normalized := normalizeRepoSubpath(path); normalized != "" { + if normalized := normalizeCleanupSelectionKey(path); normalized != "" { allowed[normalized] = struct{}{} } } - includePath := func(path string) bool { + includePath := func(paths ...string) bool { if len(allowed) == 0 { return true } - _, ok := allowed[normalizeRepoSubpath(path)] - return ok + for _, path := range paths { + if _, ok := allowed[normalizeCleanupSelectionKey(path)]; ok { + return true + } + } + return false } findings := make([]cleanupFindingModel, 0) countsByKind := map[string]int{ @@ -1134,7 +1456,7 @@ func buildCleanupAuditModel(gitSubpath string, inventory []RepoInventoryFile, re continue } displayPath := orphanDisplayPath(checksum, recordSourcePaths(matches)) - if !includePath(displayPath) { + if !includePath(cleanupSelectionCandidatesForRecords(checksum, displayPath, matches)...) { continue } kind := "repo_orphan_stale_record" @@ -1168,7 +1490,7 @@ func buildCleanupAuditModel(gitSubpath string, inventory []RepoInventoryFile, re if _, ok := referencedBucketURLs[objectURL]; ok { continue } - if !includePath(objectURL) { + if !includePath(cleanupSelectionCandidatesForBucketObject(objectURL, bucketObjectsByURL[objectURL])...) { continue } public := buildBucketOnlyFinding(bucketObjectsByURL[objectURL]) @@ -1219,6 +1541,62 @@ func buildCleanupAuditModel(gitSubpath string, inventory []RepoInventoryFile, re } } +func normalizeCleanupSelectionKey(raw string) string { + trimmed := strings.Trim(strings.TrimSpace(raw), "/") + if trimmed == "" { + return "" + } + if objectURL := canonicalStorageURL("", "", trimmed); objectURL != "" { + return objectURL + } + return trimmed +} + +func cleanupSelectionCandidatesForRecords(checksum string, displayPath string, matches []projectRecordState) []string { + candidates := []string{displayPath} + normalizedChecksum := normalizeAnalyticsChecksum(checksum) + if normalizedChecksum != "" { + candidates = append(candidates, normalizedChecksum, "sha256/"+normalizedChecksum) + } + for _, match := range matches { + candidates = append(candidates, match.ObjectID) + candidates = append(candidates, match.AccessURLs...) + candidates = append(candidates, match.CanonicalAccessURLs...) + candidates = append(candidates, accessURLsForStorage(match)...) + for _, probe := range match.AccessProbes { + candidates = append(candidates, probe.ObjectURL) + candidates = append(candidates, canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL)) + } + } + return uniqueCleanupSelectionCandidates(candidates) +} + +func cleanupSelectionCandidatesForBucketObject(objectURL string, item gintegrationsyfon.ProjectBucketObject) []string { + return uniqueCleanupSelectionCandidates([]string{ + objectURL, + item.ObjectURL, + canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL), + item.Key, + }) +} + +func uniqueCleanupSelectionCandidates(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + normalized := normalizeCleanupSelectionKey(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + func bucketObjectHasCompleteChain(matches []projectRecordState, repoPathsByChecksum map[string][]string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { for _, match := range matches { if classifyStorageFinding(match, bucketObjectsByURL) != storageFindingNone { @@ -1242,19 +1620,7 @@ func buildCleanupFinding(kind string, normalizedPath string, matches []projectRe if checksum == "" { checksum = strings.TrimSpace(match.Checksum) } - records = append(records, GitStorageCleanupRecordAudit{ - ObjectID: match.ObjectID, - Checksum: strings.TrimSpace(match.Checksum), - NormalizedPath: normalizedPath, - CleanupScope: cleanupScope, - AccessProbes: accessProbesForRecord(match), - Status: accessStatusForRecord(match), - Error: accessErrorForRecord(match), - SizeBytes: match.Size, - LastUpdated: formatOptionalTime(match.UpdatedAt), - DownloadCount: match.Usage.DownloadCount, - LastDownload: formatOptionalTime(match.Usage.LastDownloadTime), - }) + records = append(records, cleanupRecordAuditForRecord(normalizedPath, cleanupScope, match)) totalBytes += match.Size totalDownloads += match.Usage.DownloadCount latestUpdate = laterTime(latestUpdate, match.UpdatedAt) @@ -1286,6 +1652,54 @@ func buildCleanupFinding(kind string, normalizedPath string, matches []projectRe } } +func cleanupRecordAuditForRecord(normalizedPath string, cleanupScope string, record projectRecordState) GitStorageCleanupRecordAudit { + return GitStorageCleanupRecordAudit{ + ObjectID: record.ObjectID, + Checksum: strings.TrimSpace(record.Checksum), + NormalizedPath: normalizedPath, + CleanupScope: cleanupScope, + AccessURLs: uniqueStrings(record.AccessURLs), + AccessMethods: cleanupAccessMethodsFromProjectMethods(record.AccessMethods), + AccessProbes: accessProbesForRecord(record), + Status: accessStatusForRecord(record), + Error: accessErrorForRecord(record), + SizeBytes: record.Size, + LastUpdated: formatOptionalTime(record.UpdatedAt), + DownloadCount: record.Usage.DownloadCount, + LastDownload: formatOptionalTime(record.Usage.LastDownloadTime), + } +} + +func cleanupAccessMethodsFromProjectMethods(methods []gintegrationsyfon.ProjectAccessMethod) []GitStorageCleanupAccessMethod { + out := make([]GitStorageCleanupAccessMethod, 0, len(methods)) + for _, method := range methods { + out = append(out, GitStorageCleanupAccessMethod{ + AccessID: strings.TrimSpace(method.AccessID), + Type: strings.TrimSpace(method.Type), + URL: strings.TrimSpace(method.URL), + Headers: append([]string(nil), method.Headers...), + }) + } + return out +} + +func projectAccessMethodsFromCleanupMethods(methods []GitStorageCleanupAccessMethod) []gintegrationsyfon.ProjectAccessMethod { + out := make([]gintegrationsyfon.ProjectAccessMethod, 0, len(methods)) + for _, method := range methods { + url := strings.TrimSpace(method.URL) + if url == "" { + continue + } + out = append(out, gintegrationsyfon.ProjectAccessMethod{ + AccessID: strings.TrimSpace(method.AccessID), + Type: strings.TrimSpace(method.Type), + URL: url, + Headers: append([]string(nil), method.Headers...), + }) + } + return out +} + func repoOrphanAction(kind string) string { if kind == "repo_orphan_live_object" { return "Delete Syfon record and purge storage object" @@ -1294,43 +1708,13 @@ func repoOrphanAction(kind string) string { } func storageCleanupActionSupport(kind string) (string, []string, string, bool) { - switch strings.TrimSpace(kind) { - case "broken_bucket_mapping": - return storageActionabilityAutoRepair, []string{ - storageActionRemoveBrokenAccessURLs, - storageActionDeleteSyfonRecord, - storageActionInspectEvidence, - }, storageActionRemoveBrokenAccessURLs, true - case "storage_validation_mismatch": - return storageActionabilityManualChoice, []string{ - storageActionDeleteSyfonRecord, - storageActionDeleteBucketObject, - storageActionDeleteBoth, - storageActionInspectEvidence, - }, "", true - default: - return storageActionabilityInspectOnly, []string{storageActionInspectEvidence}, storageActionInspectEvidence, false - } + policy := storageRepairPolicyForKind(kind) + return policy.actionability, append([]string(nil), policy.actions...), policy.defaultAction, policy.supportsDryRun } func storageChainActionSupport(kind string) (string, []string, string, bool) { - switch strings.TrimSpace(kind) { - case "syfon_broken_bucket_mapping": - return storageActionabilityAutoRepair, []string{ - storageActionRemoveBrokenAccessURLs, - storageActionDeleteSyfonRecord, - storageActionInspectEvidence, - }, storageActionRemoveBrokenAccessURLs, true - case "git_syfon_metadata_mismatch": - return storageActionabilityManualChoice, []string{ - storageActionDeleteSyfonRecord, - storageActionDeleteBucketObject, - storageActionDeleteBoth, - storageActionInspectEvidence, - }, "", true - default: - return storageActionabilityInspectOnly, []string{storageActionInspectEvidence}, storageActionInspectEvidence, false - } + policy := storageRepairPolicyForKind(kind) + return policy.actionability, append([]string(nil), policy.actions...), policy.defaultAction, policy.supportsDryRun } func compareRecordState(left projectRecordState, right projectRecordState) int { @@ -1694,6 +2078,7 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, Checksum: strings.TrimSpace(record.Checksum), SourcePaths: uniqueStrings(gitPaths), ObjectIDs: objectIDs, + Records: []GitStorageCleanupRecordAudit{cleanupRecordAuditForRecord(path, "access_url", record)}, AccessURLs: accessURLs, BucketObjectURL: primaryProbe.bucketObjectURL, ResolvedBucket: primaryProbe.probe.Bucket, diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index bc3f68e..1f5a383 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -46,6 +46,7 @@ type fakeStorageAnalyticsBackend struct { listProbeCalls int listProbeItems []gintegrationsyfon.BulkStorageProbeItem deletedIDs []string + deletedStorageIDs []string updatedAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod deletedBucketObjects []string } @@ -202,6 +203,9 @@ func (fake *fakeStorageAnalyticsBackend) BulkListStorageObjects(ctx context.Cont func (fake *fakeStorageAnalyticsBackend) BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error { fake.deletedIDs = append(fake.deletedIDs, objectIDs...) + if deleteStorageData { + fake.deletedStorageIDs = append(fake.deletedStorageIDs, objectIDs...) + } return nil } @@ -277,13 +281,20 @@ func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { t.Fatalf("unexpected summary bytes/downloads: %+v", summary) } - children, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 10, "bytes", "desc") + usageCallsBeforeChildren := backend.listProjectFileUsageCalls + children, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 10, "bytes", "desc", "") if err != nil { t.Fatalf("build storage children: %v", err) } + if backend.listProjectFileUsageCalls != usageCallsBeforeChildren { + t.Fatalf("expected storage children to skip project-wide file usage, got calls before=%d after=%d", usageCallsBeforeChildren, backend.listProjectFileUsageCalls) + } if len(children.Items) != 4 { t.Fatalf("expected 4 child rows, got %+v", children.Items) } + if children.HasMore || children.NextCursor != "" { + t.Fatalf("expected unpaginated children response, got %+v", children) + } if children.Items[0].Path != "data/c.txt" || children.Items[1].Path != "data/nested" { t.Fatalf("unexpected child ordering: %+v", children.Items) } @@ -328,7 +339,23 @@ func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { assertHasCleanupFinding(t, cleanup.Findings, "broken_access_url_error", "data/c.txt") assertHasCleanupFinding(t, cleanup.Findings, "repo_orphan_stale_record", "s3://bucket/orphan") - applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, nil, mirrorPath, repo, hash, true, true, true, false, false, true) + applyResult, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + nil, + nil, + []GitStorageCleanupApplyFinding{ + {Kind: "stale_duplicate_record", NormalizedPath: "data/nested/b.txt", ObjectIDs: []string{"obj-b-old"}}, + {Kind: "repo_orphan_stale_record", NormalizedPath: "s3://bucket/orphan", ObjectIDs: []string{"obj-orphan"}, AccessURLs: []string{"s3://bucket/orphan"}}, + }, + true, + true, + false, + false, + true, + ) if err != nil { t.Fatalf("apply cleanup dry run: %v", err) } @@ -343,11 +370,174 @@ func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { } } -func TestApplyStorageCleanupRepairsBrokenBucketMappings(t *testing.T) { +func TestBuildStorageChildrenEnrichesOnlySelectedPage(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ - "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/big/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 600), + "data/big/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 300), + "data/other.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 50), + "data/small.txt": lfsPointer("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", 100), + "outside/file.txt": lfsPointer("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", 1000), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj"}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj"}, + {ObjectID: "obj-c", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj"}, + {ObjectID: "obj-d", Checksum: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Organization: "org", Project: "proj"}, + {ObjectID: "obj-e", Checksum: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", Organization: "org", Project: "proj"}, + }, + } + service := NewStorageAnalyticsService(backend) + + children, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "") + if err != nil { + t.Fatalf("build storage children: %v", err) + } + if len(children.Items) != 1 || children.Items[0].Path != "data/big" { + t.Fatalf("expected only biggest child page, got %+v", children.Items) + } + if !children.HasMore || children.NextCursor == "" { + t.Fatalf("expected next cursor for truncated children page, got %+v", children) + } + expectedChecksums := []string{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + } + if strings.Join(backend.bulkChecksums, ",") != strings.Join(expectedChecksums, ",") { + t.Fatalf("expected page-scoped checksums %v, got %v", expectedChecksums, backend.bulkChecksums) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected storage children to skip project file usage, got %d calls", backend.listProjectFileUsageCalls) + } +} + +func TestBuildStorageChildrenCursorPagination(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "data/c.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 100), }) + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + + first, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "") + if err != nil { + t.Fatalf("build first storage children page: %v", err) + } + if len(first.Items) != 2 || first.Items[0].Path != "data/a.txt" || first.Items[1].Path != "data/b.txt" { + t.Fatalf("unexpected first page: %+v", first.Items) + } + if !first.HasMore || first.NextCursor == "" { + t.Fatalf("expected next cursor on first page, got %+v", first) + } + + second, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", first.NextCursor) + if err != nil { + t.Fatalf("build second storage children page: %v", err) + } + if len(second.Items) != 1 || second.Items[0].Path != "data/c.txt" { + t.Fatalf("unexpected second page: %+v", second.Items) + } + if second.HasMore || second.NextCursor != "" { + t.Fatalf("expected final page without cursor, got %+v", second) + } + if second.Items[0].Path == first.Items[0].Path || second.Items[0].Path == first.Items[1].Path { + t.Fatalf("expected cursor pages not to duplicate rows: first=%+v second=%+v", first.Items, second.Items) + } +} + +func TestBuildStorageChildrenRejectsStaleCursor(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + wrongHash := plumbing.NewHash("cccccccccccccccccccccccccccccccccccccccc") + staleCursors := map[string]string{ + "commit": buildStorageChildrenCursor(wrongHash, "data", "bytes", "desc", 1), + "path": buildStorageChildrenCursor(hash, "other", "bytes", "desc", 1), + "sort_by": buildStorageChildrenCursor(hash, "data", "name", "desc", 1), + "sort_order": buildStorageChildrenCursor(hash, "data", "bytes", "asc", 1), + "invalid": "not-base64", + } + for name, cursor := range staleCursors { + t.Run(name, func(t *testing.T) { + _, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", cursor) + if err == nil { + t.Fatalf("expected stale cursor %q to fail", name) + } + }) + } +} + +func TestApplyStorageCleanupMatchesSelectedRepoOrphanByAnyAccessURL(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + + applyResult, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"s3://bucket/orphan-selected"}, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "repo_orphan_stale_record", + NormalizedPath: "s3://bucket/orphan-primary", + ObjectIDs: []string{"obj-orphan"}, + AccessURLs: []string{"s3://bucket/orphan-primary", "s3://bucket/orphan-selected"}, + }}, + true, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply cleanup: %v", err) + } + if !contains(applyResult.DeletedRecordIDs, "obj-orphan") { + t.Fatalf("expected selected orphan to be deleted, got %+v", applyResult) + } + if len(applyResult.SkippedPaths) != 0 || len(applyResult.ManualPaths) != 0 { + t.Fatalf("expected selected orphan not to be skipped or manual, got %+v", applyResult) + } + if !contains(backend.deletedIDs, "obj-orphan") { + t.Fatalf("expected syfon delete call for selected orphan, got %+v", backend.deletedIDs) + } +} + +func TestApplyStorageCleanupRejectsUnmatchedSelectedPaths(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + + _, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"s3://other-bucket/missing"}, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "repo_orphan_stale_record", + NormalizedPath: "s3://bucket/a", + ObjectIDs: []string{"obj-a"}, + AccessURLs: []string{"s3://bucket/a"}, + }}, + true, + false, + false, + false, + false, + ) + if err == nil { + t.Fatalf("expected unmatched selected path to fail") + } + if !strings.Contains(err.Error(), "selected cleanup paths did not match provided cleanup findings") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestApplyStorageCleanupRepairsBrokenBucketMappings(t *testing.T) { now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) backend := &fakeStorageAnalyticsBackend{ projectRecords: []gintegrationsyfon.ProjectRecord{ @@ -397,7 +587,46 @@ func TestApplyStorageCleanupRepairsBrokenBucketMappings(t *testing.T) { } service := NewStorageAnalyticsService(backend) - applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", []string{"data/a.txt", "data/b.txt"}, nil, mirrorPath, repo, hash, true, false, false, false, true, false) + applyResult, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"data/a.txt", "data/b.txt"}, + nil, + []GitStorageCleanupApplyFinding{ + { + Kind: "broken_bucket_mapping", + NormalizedPath: "data/a.txt", + ObjectIDs: []string{"obj-a"}, + Records: []GitStorageCleanupRecordAudit{{ + ObjectID: "obj-a", + AccessURLs: []string{"s3://legacy/a", "s3://bucket/a"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/a"}, {URL: "s3://bucket/a"}}, + AccessProbes: []GitStorageCleanupAccessProbe{ + {URL: "s3://legacy/a", Status: "error", ErrorKind: "credential_missing"}, + {URL: "s3://bucket/a", Status: "present"}, + }, + }}, + }, + { + Kind: "broken_bucket_mapping", + NormalizedPath: "data/b.txt", + ObjectIDs: []string{"obj-b"}, + Records: []GitStorageCleanupRecordAudit{{ + ObjectID: "obj-b", + AccessURLs: []string{"s3://legacy/b"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/b"}}, + AccessProbes: []GitStorageCleanupAccessProbe{{URL: "s3://legacy/b", Status: "error", ErrorKind: "credential_missing"}}, + }}, + }, + }, + false, + false, + false, + true, + false, + ) if err != nil { t.Fatalf("apply cleanup broken bucket mapping repair: %v", err) } @@ -417,9 +646,6 @@ func TestApplyStorageCleanupRepairsBrokenBucketMappings(t *testing.T) { } func TestApplyStorageCleanupDeletesBucketOnlyObjects(t *testing.T) { - repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ - "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), - }) now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) backend := &fakeStorageAnalyticsBackend{ projectRecords: []gintegrationsyfon.ProjectRecord{ @@ -443,7 +669,25 @@ func TestApplyStorageCleanupDeletesBucketOnlyObjects(t *testing.T) { } service := NewStorageAnalyticsService(backend) - applyResult, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", refName, "data", []string{"s3://bucket/orphan"}, nil, mirrorPath, repo, hash, true, false, false, true, false, false) + applyResult, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"s3://bucket/orphan"}, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "bucket_only_object", + NormalizedPath: "s3://bucket/orphan", + BucketObjectURL: "s3://bucket/orphan", + BucketObjectURLs: []string{"s3://bucket/orphan"}, + }}, + false, + false, + true, + false, + false, + ) if err != nil { t.Fatalf("apply cleanup bucket only delete: %v", err) } @@ -528,9 +772,6 @@ func TestBuildStorageCleanupAuditReportsStorageProbeFailures(t *testing.T) { } func TestApplyStorageCleanupSupportsMetadataMismatchActions(t *testing.T) { - repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ - "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), - }) now := time.Date(2026, 7, 2, 12, 0, 0, 0, time.UTC) backend := &fakeStorageAnalyticsBackend{ projectRecords: []gintegrationsyfon.ProjectRecord{ @@ -551,20 +792,23 @@ func TestApplyStorageCleanupSupportsMetadataMismatchActions(t *testing.T) { }, } service := NewStorageAnalyticsService(backend) + applyFindings := []GitStorageCleanupApplyFinding{{ + Kind: "storage_validation_mismatch", + NormalizedPath: "data/b.txt", + ObjectIDs: []string{"obj-b"}, + BucketObjectURL: "s3://bucket/b", + BucketObjectURLs: []string{"s3://bucket/b"}, + AccessURLs: []string{"s3://bucket/b"}, + }} deleteRecordOnly, err := service.ApplyStorageCleanup( context.Background(), "Bearer token", "org", "proj", - refName, - "data", []string{"data/b.txt"}, []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/b.txt", Action: "delete_syfon_record"}}, - mirrorPath, - repo, - hash, - true, + applyFindings, false, false, false, @@ -586,14 +830,9 @@ func TestApplyStorageCleanupSupportsMetadataMismatchActions(t *testing.T) { "Bearer token", "org", "proj", - refName, - "data", []string{"data/b.txt"}, []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/b.txt", Action: "delete_bucket_object"}}, - mirrorPath, - repo, - hash, - true, + applyFindings, false, false, false, @@ -615,14 +854,9 @@ func TestApplyStorageCleanupSupportsMetadataMismatchActions(t *testing.T) { "Bearer token", "org", "proj", - refName, - "data", []string{"data/b.txt"}, []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/b.txt", Action: "delete_both"}}, - mirrorPath, - repo, - hash, - true, + applyFindings, false, false, false, @@ -640,6 +874,209 @@ func TestApplyStorageCleanupSupportsMetadataMismatchActions(t *testing.T) { } } +func TestStorageRepairPolicyCoversFindingKinds(t *testing.T) { + tests := []struct { + kind string + defaultAction string + actions []string + actionability string + }{ + {"bucket_only_object", storageActionDeleteBucketObject, []string{storageActionDeleteBucketObject}, storageActionabilityAutoRepair}, + {"bucket_syfon_no_git", storageActionDeleteBoth, []string{storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject}, storageActionabilityAutoRepair}, + {"repo_orphan_live_object", storageActionDeleteBoth, []string{storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject}, storageActionabilityAutoRepair}, + {"repo_orphan_stale_record", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"stale_duplicate_record", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"live_duplicate_conflict", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, + {"broken_access_url_error", storageActionRemoveBrokenAccessURLs, []string{storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"broken_bucket_mapping", storageActionRemoveBrokenAccessURLs, []string{storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"syfon_broken_bucket_mapping", storageActionRemoveBrokenAccessURLs, []string{storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"storage_validation_mismatch", "", []string{storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionDeleteBoth}, storageActionabilityManualChoice}, + {"git_syfon_metadata_mismatch", "", []string{storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionDeleteBoth}, storageActionabilityManualChoice}, + {"storage_object_missing", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"syfon_git_no_bucket", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"syfon_missing_bucket_object", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"git_only_no_syfon", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, + {"storage_probe_error", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, + {"probe_error", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, + } + for _, test := range tests { + t.Run(test.kind, func(t *testing.T) { + policy := storageRepairPolicyForKind(test.kind) + if policy.defaultAction != test.defaultAction { + t.Fatalf("expected default %q, got %q", test.defaultAction, policy.defaultAction) + } + if policy.actionability != test.actionability { + t.Fatalf("expected actionability %q, got %q", test.actionability, policy.actionability) + } + for _, action := range test.actions { + if !contains(policy.actions, action) { + t.Fatalf("expected action %q in %+v", action, policy.actions) + } + } + }) + } +} + +func TestApplyStorageCleanupRepairMatrix(t *testing.T) { + tests := []struct { + name string + finding GitStorageCleanupApplyFinding + actions []GitStorageCleanupApplyAction + wantDeletedIDs []string + wantStorageIDs []string + wantBucketURLs []string + wantUpdatedRecordID string + wantManualPath string + }{ + {name: "bucket only", finding: applyFinding("bucket_only_object", "s3://bucket/only", nil, []string{"s3://bucket/only"}), wantBucketURLs: []string{"s3://bucket/only"}}, + {name: "bucket syfon no git", finding: applyFinding("bucket_syfon_no_git", "s3://bucket/no-git", []string{"obj-a"}, []string{"s3://bucket/no-git"}), wantDeletedIDs: []string{"obj-a"}, wantBucketURLs: []string{"s3://bucket/no-git"}}, + {name: "repo orphan live", finding: applyFinding("repo_orphan_live_object", "s3://bucket/live", []string{"obj-live"}, nil), wantDeletedIDs: []string{"obj-live"}, wantStorageIDs: []string{"obj-live"}}, + {name: "repo orphan stale", finding: applyFinding("repo_orphan_stale_record", "s3://bucket/stale", []string{"obj-stale"}, nil), wantDeletedIDs: []string{"obj-stale"}}, + {name: "stale duplicate", finding: applyFinding("stale_duplicate_record", "data/a.txt", []string{"obj-old"}, nil), wantDeletedIDs: []string{"obj-old"}}, + {name: "storage missing", finding: applyFinding("storage_object_missing", "data/missing.txt", []string{"obj-missing"}, nil), wantDeletedIDs: []string{"obj-missing"}}, + {name: "syfon git no bucket", finding: applyFinding("syfon_git_no_bucket", "data/missing.txt", []string{"obj-missing"}, nil), wantDeletedIDs: []string{"obj-missing"}}, + {name: "syfon missing bucket object", finding: applyFinding("syfon_missing_bucket_object", "s3://bucket/missing", []string{"obj-missing"}, nil), wantDeletedIDs: []string{"obj-missing"}}, + {name: "git syfon mismatch delete both", finding: applyFinding("git_syfon_metadata_mismatch", "data/mismatch.txt", []string{"obj-mm"}, []string{"s3://bucket/mismatch"}), actions: []GitStorageCleanupApplyAction{{Kind: "git_syfon_metadata_mismatch", NormalizedPath: "data/mismatch.txt", Action: storageActionDeleteBoth}}, wantDeletedIDs: []string{"obj-mm"}, wantBucketURLs: []string{"s3://bucket/mismatch"}}, + {name: "storage mismatch delete both", finding: applyFinding("storage_validation_mismatch", "data/mismatch.txt", []string{"obj-mm"}, []string{"s3://bucket/mismatch"}), actions: []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/mismatch.txt", Action: storageActionDeleteBoth}}, wantDeletedIDs: []string{"obj-mm"}, wantBucketURLs: []string{"s3://bucket/mismatch"}}, + {name: "live duplicate conflict", finding: applyFinding("live_duplicate_conflict", "data/dup.txt", []string{"obj-a", "obj-b"}, nil), wantManualPath: "data/dup.txt"}, + {name: "git only no syfon", finding: applyFinding("git_only_no_syfon", "data/git-only.txt", nil, nil), wantManualPath: "data/git-only.txt"}, + {name: "probe error", finding: applyFinding("probe_error", "data/probe.txt", []string{"obj-probe"}, nil), wantManualPath: "data/probe.txt"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + result, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", nil, test.actions, []GitStorageCleanupApplyFinding{test.finding}, false, false, false, false, false) + if err != nil { + t.Fatalf("apply cleanup: %v", err) + } + assertStringSet(t, "deleted IDs", result.DeletedRecordIDs, test.wantDeletedIDs) + assertStringSet(t, "storage IDs", backend.deletedStorageIDs, test.wantStorageIDs) + assertStringSet(t, "bucket URLs", result.DeletedBucketObjectURLs, test.wantBucketURLs) + if test.wantManualPath != "" && !contains(result.ManualPaths, test.wantManualPath) { + t.Fatalf("expected manual path %q, got %+v", test.wantManualPath, result.ManualPaths) + } + if test.wantUpdatedRecordID != "" && !contains(result.UpdatedRecordIDs, test.wantUpdatedRecordID) { + t.Fatalf("expected updated record %q, got %+v", test.wantUpdatedRecordID, result.UpdatedRecordIDs) + } + }) + } +} + +func TestApplyStorageCleanupPrunesBrokenAccessMethods(t *testing.T) { + tests := []struct { + name string + record GitStorageCleanupRecordAudit + wantDeleted []string + wantRemaining []string + wantErrSnippet string + }{ + { + name: "keeps good access method", + record: GitStorageCleanupRecordAudit{ + ObjectID: "obj-a", + AccessURLs: []string{"s3://legacy/a", "s3://bucket/a"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/a"}, {URL: "s3://bucket/a"}}, + AccessProbes: []GitStorageCleanupAccessProbe{ + {URL: "s3://legacy/a", Status: "error", ErrorKind: "credential_missing"}, + {URL: "s3://bucket/a", Status: "present"}, + }, + }, + wantRemaining: []string{"s3://bucket/a"}, + }, + { + name: "deletes record when only access method is broken", + record: GitStorageCleanupRecordAudit{ + ObjectID: "obj-b", + AccessURLs: []string{"s3://legacy/b"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/b"}}, + AccessProbes: []GitStorageCleanupAccessProbe{{URL: "s3://legacy/b", Status: "error", ErrorKind: "credential_missing"}}, + }, + wantDeleted: []string{"obj-b"}, + }, + { + name: "missing evidence fails", + record: GitStorageCleanupRecordAudit{ + ObjectID: "obj-c", + AccessURLs: []string{"s3://legacy/c"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/c"}}, + }, + wantErrSnippet: "missing broken access URL evidence", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + _, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + nil, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "broken_access_url_error", + NormalizedPath: "data/a.txt", + ObjectIDs: []string{test.record.ObjectID}, + Records: []GitStorageCleanupRecordAudit{test.record}, + }}, + false, + false, + false, + false, + false, + ) + if test.wantErrSnippet != "" { + if err == nil || !strings.Contains(err.Error(), test.wantErrSnippet) { + t.Fatalf("expected error containing %q, got %v", test.wantErrSnippet, err) + } + return + } + if err != nil { + t.Fatalf("apply cleanup: %v", err) + } + assertStringSet(t, "deleted IDs", backend.deletedIDs, test.wantDeleted) + if len(test.wantRemaining) > 0 { + methods := backend.updatedAccessMethods[test.record.ObjectID] + got := make([]string, 0, len(methods)) + for _, method := range methods { + got = append(got, method.URL) + } + assertStringSet(t, "remaining access methods", got, test.wantRemaining) + } + }) + } +} + +func TestApplyStorageCleanupValidation(t *testing.T) { + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + tests := []struct { + name string + finding GitStorageCleanupApplyFinding + actions []GitStorageCleanupApplyAction + wantErr string + }{ + {name: "unknown kind", finding: applyFinding("unknown_kind", "data/a.txt", []string{"obj-a"}, nil), wantErr: "unsupported cleanup finding kind"}, + {name: "unsupported action", finding: applyFinding("bucket_only_object", "s3://bucket/a", nil, []string{"s3://bucket/a"}), actions: []GitStorageCleanupApplyAction{{Kind: "bucket_only_object", NormalizedPath: "s3://bucket/a", Action: storageActionDeleteSyfonRecord}}, wantErr: "not supported"}, + {name: "action not advertised", finding: func() GitStorageCleanupApplyFinding { + finding := applyFinding("bucket_only_object", "s3://bucket/a", nil, []string{"s3://bucket/a"}) + finding.AvailableActions = []string{storageActionInspectEvidence} + return finding + }(), wantErr: "not advertised"}, + {name: "missing object ids", finding: applyFinding("repo_orphan_stale_record", "s3://bucket/a", nil, nil), wantErr: "missing object_ids"}, + {name: "missing bucket urls", finding: applyFinding("bucket_only_object", "data/a.txt", nil, nil), wantErr: "missing bucket object URLs"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", nil, test.actions, []GitStorageCleanupApplyFinding{test.finding}, false, false, false, false, false) + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("expected error containing %q, got %v", test.wantErr, err) + } + }) + } +} + func TestBuildStorageCleanupAuditFlagsRecordWhenAnyAccessProbeIsDead(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -1200,7 +1637,7 @@ func TestBuildStorageChainAuditCachesProjectRecordsPerPathPrefix(t *testing.T) { } } -func TestStorageSummaryAndChildrenShareInflightJoinWork(t *testing.T) { +func TestStorageChildrenSkipsSummaryJoinUsageWork(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), }) @@ -1221,7 +1658,7 @@ func TestStorageSummaryAndChildrenShareInflightJoinWork(t *testing.T) { errCh <- err }() go func() { - _, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1000, "bytes", "desc") + _, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1000, "bytes", "desc", "") errCh <- err }() for i := 0; i < 2; i++ { @@ -1229,11 +1666,11 @@ func TestStorageSummaryAndChildrenShareInflightJoinWork(t *testing.T) { t.Fatalf("expected concurrent storage analytics request to succeed, got %v", err) } } - if backend.bulkGetProjectRecordsCalls != 1 { - t.Fatalf("expected concurrent summary/children requests to share one checksum lookup, got %d", backend.bulkGetProjectRecordsCalls) + if backend.bulkGetProjectRecordsCalls != 2 { + t.Fatalf("expected summary and page-scoped children checksum lookups, got %d", backend.bulkGetProjectRecordsCalls) } if backend.listProjectFileUsageCalls != 1 { - t.Fatalf("expected concurrent summary/children requests to share one usage lookup, got %d", backend.listProjectFileUsageCalls) + t.Fatalf("expected only summary to perform usage lookup, got %d", backend.listProjectFileUsageCalls) } } @@ -1789,6 +2226,90 @@ func TestBuildStorageChainAuditReturnsFullFindings(t *testing.T) { } } +func TestApplyStorageCleanupDeletesSelectedBucketSyfonNoGitChainFinding(t *testing.T) { + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-no-git", + Checksum: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + Organization: "org", + Project: "proj", + Size: 150, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/no-git"}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://bucket/no-git", + Bucket: "bucket", + Key: "no-git", + Path: "no-git", + SizeBytes: 150, + MetaSHA256: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + response, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"s3://bucket/no-git"}, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "bucket_syfon_no_git", + NormalizedPath: "s3://bucket/no-git", + ObjectIDs: []string{"obj-no-git"}, + BucketObjectURL: "s3://bucket/no-git", + BucketObjectURLs: []string{"s3://bucket/no-git"}, + AccessURLs: []string{"s3://bucket/no-git"}, + }}, + true, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply cleanup: %v", err) + } + if !contains(response.DeletedRecordIDs, "obj-no-git") { + t.Fatalf("expected selected Syfon record to be deleted, got %+v", response) + } + if !contains(response.DeletedBucketObjectURLs, "s3://bucket/no-git") { + t.Fatalf("expected selected bucket object to be deleted, got %+v", response) + } + if !contains(backend.deletedIDs, "obj-no-git") { + t.Fatalf("expected backend Syfon delete, got %+v", backend.deletedIDs) + } + if !contains(backend.deletedBucketObjects, "s3://bucket/no-git") { + t.Fatalf("expected backend bucket delete, got %+v", backend.deletedBucketObjects) + } +} + +func TestBuildStorageChainAuditMarksBucketSyfonNoGitActionable(t *testing.T) { + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupport("bucket_syfon_no_git") + if actionability != storageActionabilityAutoRepair { + t.Fatalf("expected auto-repair actionability, got %q", actionability) + } + if defaultAction != storageActionDeleteBoth { + t.Fatalf("expected delete-both default action, got %q", defaultAction) + } + if !supportsDryRun { + t.Fatal("expected bucket+Syfon/no-Git to support dry-run") + } + if !contains(availableActions, storageActionDeleteBoth) || !contains(availableActions, storageActionDeleteSyfonRecord) || !contains(availableActions, storageActionDeleteBucketObject) { + t.Fatalf("expected destructive chain actions, got %+v", availableActions) + } +} + func TestBuildStorageCleanupAuditTreatsMissingProbeEvidenceAsProbeError(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -1972,6 +2493,46 @@ func int64Ptr(value int64) *int64 { return ©Value } +func applyFinding(kind string, normalizedPath string, objectIDs []string, bucketURLs []string) GitStorageCleanupApplyFinding { + policy := storageRepairPolicyForKind(kind) + return GitStorageCleanupApplyFinding{ + Kind: kind, + NormalizedPath: normalizedPath, + ObjectIDs: append([]string(nil), objectIDs...), + BucketObjectURL: firstString(bucketURLs), + BucketObjectURLs: append([]string(nil), bucketURLs...), + AccessURLs: append([]string(nil), bucketURLs...), + AvailableActions: append([]string(nil), policy.actions...), + DefaultAction: policy.defaultAction, + Evidence: &GitAuditEvidence{ + ObjectIDs: append([]string(nil), objectIDs...), + AccessURLs: append([]string(nil), bucketURLs...), + BucketObjectURLs: append([]string(nil), bucketURLs...), + }, + } +} + +func firstString(values []string) string { + if len(values) == 0 { + return "" + } + return values[0] +} + +func assertStringSet(t *testing.T, label string, got []string, want []string) { + t.Helper() + got = uniqueStrings(got) + want = uniqueStrings(want) + if len(got) != len(want) { + t.Fatalf("%s: expected %v, got %v", label, want, got) + } + for _, value := range want { + if !contains(got, value) { + t.Fatalf("%s: expected %v, got %v", label, want, got) + } + } +} + func contains(values []string, target string) bool { for _, value := range values { if value == target { diff --git a/internal/git/storage_children.go b/internal/git/storage_children.go new file mode 100644 index 0000000..fd6de56 --- /dev/null +++ b/internal/git/storage_children.go @@ -0,0 +1,191 @@ +package git + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + "github.com/go-git/go-git/v5/plumbing" +) + +type storageChildrenCursor struct { + CommitHash string `json:"commit_hash"` + GitSubpath string `json:"git_subpath"` + SortBy string `json:"sort_by"` + SortOrder string `json:"sort_order"` + Offset int `json:"offset"` +} + +type storageChildrenPage struct { + items []storageAggregate + hasMore bool + nextCursor string +} + +func buildStorageChildrenCursor(hash plumbing.Hash, gitSubpath string, sortBy string, sortOrder string, offset int) string { + cursor := storageChildrenCursor{ + CommitHash: hash.String(), + GitSubpath: normalizeRepoSubpath(gitSubpath), + SortBy: strings.TrimSpace(sortBy), + SortOrder: strings.TrimSpace(sortOrder), + Offset: offset, + } + contentBytes, err := json.Marshal(cursor) + if err != nil { + return "" + } + return base64.RawURLEncoding.EncodeToString(contentBytes) +} + +func decodeStorageChildrenCursor(raw string) (storageChildrenCursor, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return storageChildrenCursor{}, nil + } + contentBytes, err := base64.RawURLEncoding.DecodeString(trimmed) + if err != nil { + return storageChildrenCursor{}, fmt.Errorf("decode storage children cursor: %w", err) + } + cursor := storageChildrenCursor{} + if err := json.Unmarshal(contentBytes, &cursor); err != nil { + return storageChildrenCursor{}, fmt.Errorf("decode storage children cursor JSON: %w", err) + } + if cursor.Offset < 0 { + return storageChildrenCursor{}, fmt.Errorf("storage children cursor offset must be non-negative") + } + return cursor, nil +} + +func storageChildrenPageForRequest(items []storageAggregate, hash plumbing.Hash, gitSubpath string, sortBy string, sortOrder string, limit int, rawCursor string) (storageChildrenPage, error) { + if limit <= 0 { + limit = len(items) + } + cursor, err := decodeStorageChildrenCursor(rawCursor) + if err != nil { + return storageChildrenPage{}, err + } + offset := cursor.Offset + if strings.TrimSpace(rawCursor) != "" { + expected := storageChildrenCursor{ + CommitHash: hash.String(), + GitSubpath: normalizeRepoSubpath(gitSubpath), + SortBy: strings.TrimSpace(sortBy), + SortOrder: strings.TrimSpace(sortOrder), + } + if cursor.CommitHash != expected.CommitHash || cursor.GitSubpath != expected.GitSubpath || cursor.SortBy != expected.SortBy || cursor.SortOrder != expected.SortOrder { + return storageChildrenPage{}, fmt.Errorf("storage children cursor does not match current request") + } + } + if offset > len(items) { + offset = len(items) + } + end := offset + limit + if end > len(items) { + end = len(items) + } + pageItems := append([]storageAggregate(nil), items[offset:end]...) + page := storageChildrenPage{ + items: pageItems, + hasMore: end < len(items), + } + if page.hasMore { + page.nextCursor = buildStorageChildrenCursor(hash, gitSubpath, sortBy, sortOrder, end) + } + return page, nil +} + +func filterInventoryForStorageChildren(inventory []RepoInventoryFile, children []storageAggregate) []RepoInventoryFile { + if len(children) == 0 || len(inventory) == 0 { + return nil + } + selectedFiles := make(map[string]struct{}) + selectedDirectories := make([]string, 0) + for _, child := range children { + childPath := normalizeRepoSubpath(child.path) + if childPath == "" { + continue + } + if child.rowType == "file" { + selectedFiles[childPath] = struct{}{} + continue + } + selectedDirectories = append(selectedDirectories, childPath+"/") + } + filtered := make([]RepoInventoryFile, 0) + for _, item := range inventory { + itemPath := normalizeRepoSubpath(item.RepoPath) + if _, ok := selectedFiles[itemPath]; ok { + filtered = append(filtered, item) + continue + } + for _, prefix := range selectedDirectories { + if strings.HasPrefix(itemPath, prefix) { + filtered = append(filtered, item) + break + } + } + } + return filtered +} + +func (service *StorageAnalyticsService) enrichStorageChildrenPage(ctx context.Context, authorizationHeader string, organization string, project string, gitSubpath string, inventory []RepoInventoryFile, pageItems []storageAggregate) ([]storageAggregate, error) { + checksums := uniqueInventoryChecksums(inventory) + if len(checksums) == 0 { + return pageItems, nil + } + recordsByChecksumRaw, err := service.storage.BulkGetProjectRecordsByChecksum(ctx, authorizationHeader, organization, project, checksums) + if err != nil { + return nil, fmt.Errorf("lookup syfon project records by checksum: %w", err) + } + recordsByChecksum := make(map[string][]projectRecordState, len(recordsByChecksumRaw)) + for _, records := range recordsByChecksumRaw { + for _, record := range records { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + record.Checksum = normalizedChecksum + recordsByChecksum[normalizedChecksum] = append(recordsByChecksum[normalizedChecksum], projectRecordState{ + ProjectRecord: record, + }) + } + } + return aggregateImmediateChildren(gitSubpath, inventory, recordsByChecksum, nil, pageItems), nil +} + +func uniqueInventoryChecksums(inventory []RepoInventoryFile) []string { + checksums := make([]string, 0, len(inventory)) + seen := make(map[string]struct{}, len(inventory)) + for _, item := range inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" { + continue + } + if _, ok := seen[checksum]; ok { + continue + } + seen[checksum] = struct{}{} + checksums = append(checksums, checksum) + } + return checksums +} + +func storageChildrenItemsFromAggregates(aggregates []storageAggregate) []GitStorageChildResponseItem { + items := make([]GitStorageChildResponseItem, 0, len(aggregates)) + for _, agg := range aggregates { + items = append(items, GitStorageChildResponseItem{ + Name: agg.name, + Path: agg.path, + Type: agg.rowType, + FileCount: agg.fileCount, + RecordCount: agg.recordCount, + TotalBytes: agg.totalBytes, + DownloadCount: agg.downloadCount, + LastDownloadTime: formatOptionalTime(agg.lastDownload), + LatestUpdateTime: formatOptionalTime(agg.latestUpdate), + }) + } + return items +} diff --git a/internal/git/types.go b/internal/git/types.go index cd8c8c5..33b1fbe 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -330,7 +330,9 @@ type GitStorageChildResponseItem struct { } type GitStorageChildrenResponse struct { - Items []GitStorageChildResponseItem `json:"items"` + Items []GitStorageChildResponseItem `json:"items"` + HasMore bool `json:"has_more"` + NextCursor string `json:"next_cursor,omitempty"` } type GitProjectDiffAuditRequest struct { @@ -403,26 +405,27 @@ type GitStorageChainAuditRequest struct { } type GitStorageChainFinding struct { - Kind string `json:"kind"` - NormalizedPath string `json:"normalized_path"` - Checksum string `json:"checksum,omitempty"` - SourcePaths []string `json:"source_paths,omitempty"` - ObjectIDs []string `json:"object_ids"` - AccessURLs []string `json:"access_urls,omitempty"` - BucketObjectURL string `json:"bucket_object_url,omitempty"` - ResolvedBucket string `json:"resolved_bucket,omitempty"` - ResolvedKey string `json:"resolved_key,omitempty"` - ProbeStatus string `json:"probe_status,omitempty"` - ErrorKind string `json:"error_kind,omitempty"` - Error string `json:"error,omitempty"` - RecordCount int `json:"record_count"` - SizeBytes int64 `json:"size_bytes,omitempty"` - RecommendedAction string `json:"recommended_action"` - Actionability string `json:"actionability,omitempty"` - AvailableActions []string `json:"available_actions,omitempty"` - DefaultAction string `json:"default_action,omitempty"` - SupportsDryRun bool `json:"supports_dry_run,omitempty"` - Evidence *GitAuditEvidence `json:"evidence,omitempty"` + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids"` + Records []GitStorageCleanupRecordAudit `json:"records,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + BucketObjectURL string `json:"bucket_object_url,omitempty"` + ResolvedBucket string `json:"resolved_bucket,omitempty"` + ResolvedKey string `json:"resolved_key,omitempty"` + ProbeStatus string `json:"probe_status,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + Error string `json:"error,omitempty"` + RecordCount int `json:"record_count"` + SizeBytes int64 `json:"size_bytes,omitempty"` + RecommendedAction string `json:"recommended_action"` + Actionability string `json:"actionability,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + SupportsDryRun bool `json:"supports_dry_run,omitempty"` + Evidence *GitAuditEvidence `json:"evidence,omitempty"` } type GitStorageChainAuditSummary struct { @@ -481,18 +484,27 @@ type GitStorageCleanupAccessProbe struct { ValidationMismatches []string `json:"validation_mismatches,omitempty"` } +type GitStorageCleanupAccessMethod struct { + AccessID string `json:"access_id,omitempty"` + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Headers []string `json:"headers,omitempty"` +} + type GitStorageCleanupRecordAudit struct { - ObjectID string `json:"object_id"` - Checksum string `json:"checksum,omitempty"` - NormalizedPath string `json:"normalized_path,omitempty"` - CleanupScope string `json:"cleanup_scope"` - AccessProbes []GitStorageCleanupAccessProbe `json:"access_probes"` - Status string `json:"status,omitempty"` - Error string `json:"error,omitempty"` - SizeBytes int64 `json:"size,omitempty"` - LastUpdated string `json:"updated_time,omitempty"` - DownloadCount int64 `json:"download_count,omitempty"` - LastDownload string `json:"last_download_time,omitempty"` + ObjectID string `json:"object_id"` + Checksum string `json:"checksum,omitempty"` + NormalizedPath string `json:"normalized_path,omitempty"` + CleanupScope string `json:"cleanup_scope"` + AccessURLs []string `json:"access_urls,omitempty"` + AccessMethods []GitStorageCleanupAccessMethod `json:"access_methods,omitempty"` + AccessProbes []GitStorageCleanupAccessProbe `json:"access_probes"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` + SizeBytes int64 `json:"size,omitempty"` + LastUpdated string `json:"updated_time,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` } type GitStorageCleanupFinding struct { @@ -521,6 +533,19 @@ type GitStorageCleanupApplyAction struct { NormalizedPath string `json:"normalized_path,omitempty"` } +type GitStorageCleanupApplyFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + ObjectIDs []string `json:"object_ids,omitempty"` + Records []GitStorageCleanupRecordAudit `json:"records,omitempty"` + BucketObjectURL string `json:"bucket_object_url,omitempty"` + BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + Evidence *GitAuditEvidence `json:"evidence,omitempty"` +} + type GitStorageCleanupAuditSummary struct { CountsByKind map[string]int `json:"counts_by_kind"` TotalFindings int `json:"total_findings"` @@ -539,15 +564,16 @@ type GitStorageCleanupAuditResponse struct { } type GitStorageCleanupApplyRequest struct { - GitSubpath string `json:"git_subpath,omitempty"` - Ref string `json:"ref,omitempty"` - DeleteRepoOrphans bool `json:"delete_repo_orphans,omitempty"` - DeleteStaleDuplicates bool `json:"delete_stale_duplicates,omitempty"` - DeleteBucketOnlyObjects bool `json:"delete_bucket_only_objects,omitempty"` - RepairBrokenBucketMappings bool `json:"repair_broken_bucket_mappings,omitempty"` - DryRun bool `json:"dry_run,omitempty"` - SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` - Actions []GitStorageCleanupApplyAction `json:"actions,omitempty"` + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + DeleteRepoOrphans bool `json:"delete_repo_orphans,omitempty"` + DeleteStaleDuplicates bool `json:"delete_stale_duplicates,omitempty"` + DeleteBucketOnlyObjects bool `json:"delete_bucket_only_objects,omitempty"` + RepairBrokenBucketMappings bool `json:"repair_broken_bucket_mappings,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` + Actions []GitStorageCleanupApplyAction `json:"actions,omitempty"` + Findings []GitStorageCleanupApplyFinding `json:"findings,omitempty"` } type GitStorageCleanupPurgeResult struct { diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index 4afb940..a186225 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -504,14 +504,16 @@ func (manager *Manager) BulkDeleteObjects(ctx context.Context, authorizationHead if len(normalized) == 0 { return nil } - requestBody := drsapi.BulkDeleteObjectsJSONRequestBody{BulkObjectIds: normalized} - if deleteStorageData { - requestBody.DeleteStorageData = &deleteStorageData - } resp, err := manager.drsClient(authorizationHeader) if err != nil { return err } + deleteMetadata := true + requestBody := drsapi.BulkDeleteObjectsJSONRequestBody{ + BulkObjectIds: normalized, + DeleteObjectMetadata: &deleteMetadata, + DeleteStorageData: &deleteStorageData, + } response, err := resp.BulkDeleteObjectsWithResponse(ctx, requestBody) if err != nil { return fmt.Errorf("bulk delete syfon objects: %w", err) @@ -924,7 +926,7 @@ func (manager *Manager) bucketsService(authorizationHeader string) (*syfonservic } func (manager *Manager) drsClient(authorizationHeader string) (*drsapi.ClientWithResponses, error) { - clientBaseURL, err := manager.clientBaseURL() + clientBaseURL, err := manager.drsAPIBaseURL() if err != nil { return nil, err } @@ -942,6 +944,17 @@ func (manager *Manager) drsClient(authorizationHeader string) (*drsapi.ClientWit return client, nil } +func (manager *Manager) drsAPIBaseURL() (string, error) { + clientBaseURL, err := manager.clientBaseURL() + if err != nil { + return "", err + } + if strings.HasSuffix(clientBaseURL, "/ga4gh/drs/v1") { + return clientBaseURL, nil + } + return strings.TrimRight(clientBaseURL+"/ga4gh/drs/v1", "/"), nil +} + func (manager *Manager) clientBaseURL() (string, error) { if manager.baseURL == "" { return "", fmt.Errorf("SYFON_DATA_API_BASE_URL is not configured") diff --git a/internal/integrations/syfon/adapter_test.go b/internal/integrations/syfon/adapter_test.go index 928f943..a7d63b6 100644 --- a/internal/integrations/syfon/adapter_test.go +++ b/internal/integrations/syfon/adapter_test.go @@ -158,6 +158,60 @@ func TestBulkGetProjectRecordsByChecksumAllowsLegacyUnscopedResults(t *testing.T } } +func TestBulkDeleteObjectsUsesBulkDRSDeleteEndpoint(t *testing.T) { + t.Helper() + + var requests []string + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + requests = append(requests, r.Method+" "+r.URL.Path) + if r.Method != http.MethodPut { + t.Fatalf("unexpected method %s", r.Method) + } + if r.URL.Path != "/ga4gh/drs/v1/objects/delete" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + + var req struct { + BulkObjectIDs []string `json:"bulk_object_ids"` + DeleteObjectMetadata *bool `json:"delete_object_metadata"` + DeleteStorageData *bool `json:"delete_storage_data"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if !reflect.DeepEqual(req.BulkObjectIDs, []string{"obj-a", "obj-b"}) { + t.Fatalf("unexpected bulk object ids: %#v", req.BulkObjectIDs) + } + if req.DeleteObjectMetadata == nil || !*req.DeleteObjectMetadata { + t.Fatalf("expected delete_object_metadata=true, got %#v", req.DeleteObjectMetadata) + } + if req.DeleteStorageData == nil || !*req.DeleteStorageData { + t.Fatalf("expected delete_storage_data=true, got %#v", req.DeleteStorageData) + } + + return &http.Response{ + StatusCode: http.StatusNoContent, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(nil)), + }, nil + })} + + manager := NewManager("http://syfon.example/data", client) + if err := manager.BulkDeleteObjects( + context.Background(), + "Bearer token", + []string{"obj-a", "obj-b", "obj-a"}, + true, + ); err != nil { + t.Fatalf("BulkDeleteObjects returned error: %v", err) + } + + expected := []string{"PUT /ga4gh/drs/v1/objects/delete"} + if !reflect.DeepEqual(requests, expected) { + t.Fatalf("unexpected delete requests: %#v", requests) + } +} + func TestBulkProbeStorageObjectsBatchesRequests(t *testing.T) { t.Helper() diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index 9405147..f578732 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -17,6 +17,15 @@ import ( ) const defaultStorageChainFindingLimit = 500 +const defaultStorageChildrenLimit = 100 +const maxStorageChildrenLimit = 1000 + +type storageChildrenRequestOptions struct { + limit int + cursor string + sortBy string + sortOrder string +} func (handler *Handler) handleGitProjectStorageSummaryGET(ctx fiber.Ctx) error { projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) @@ -37,15 +46,10 @@ func (handler *Handler) handleGitProjectStorageChildrenGET(ctx fiber.Ctx) error return errResponse.Write(ctx) } gitSubpath := normalizeAnalyticsSubpath(strings.TrimSpace(ctx.Query("git_subpath"))) - limit := 1000 - if rawLimit := strings.TrimSpace(ctx.Query("limit")); rawLimit != "" { - parsed, err := strconv.Atoi(rawLimit) - if err != nil || parsed <= 0 { - response := httputil.NewError("invalid_request", "limit must be a positive integer", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) - response.WriteLog(handler.logger) - return response.Write(ctx) - } - limit = parsed + options, errResponse := parseStorageChildrenRequestOptions(ctx, projectCtx.projectID) + if errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) } response, err := handler.storageAnalytics.BuildStorageChildren( ctx.Context(), @@ -57,9 +61,10 @@ func (handler *Handler) handleGitProjectStorageChildrenGET(ctx fiber.Ctx) error projectCtx.mirrorPath, projectCtx.repo, projectCtx.hash, - limit, - strings.TrimSpace(ctx.Query("sort_by")), - strings.TrimSpace(ctx.Query("sort_order")), + options.limit, + options.sortBy, + options.sortOrder, + options.cursor, ) if err != nil { return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) @@ -67,6 +72,33 @@ func (handler *Handler) handleGitProjectStorageChildrenGET(ctx fiber.Ctx) error return httputil.JSON(response, http.StatusOK).Write(ctx) } +func parseStorageChildrenRequestOptions(ctx fiber.Ctx, projectID string) (storageChildrenRequestOptions, *httputil.ErrorResponse) { + limit, err := parseStorageChildrenLimit(strings.TrimSpace(ctx.Query("limit"))) + if err != nil { + return storageChildrenRequestOptions{}, httputil.NewError("invalid_request", err.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + } + return storageChildrenRequestOptions{ + limit: limit, + cursor: strings.TrimSpace(ctx.Query("cursor")), + sortBy: strings.TrimSpace(ctx.Query("sort_by")), + sortOrder: strings.TrimSpace(ctx.Query("sort_order")), + }, nil +} + +func parseStorageChildrenLimit(rawLimit string) (int, error) { + if rawLimit == "" { + return defaultStorageChildrenLimit, nil + } + parsed, err := strconv.Atoi(rawLimit) + if err != nil || parsed <= 0 { + return 0, fmt.Errorf("limit must be a positive integer") + } + if parsed > maxStorageChildrenLimit { + return maxStorageChildrenLimit, nil + } + return parsed, nil +} + func (handler *Handler) handleGitProjectDiffAuditPOST(ctx fiber.Ctx) error { projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) if errResponse != nil { @@ -246,19 +278,49 @@ func (handler *Handler) handleGitProjectStorageCleanupApplyPOST(ctx fiber.Ctx) e requestBody.Actions[i].Kind = strings.TrimSpace(requestBody.Actions[i].Kind) requestBody.Actions[i].Action = strings.TrimSpace(requestBody.Actions[i].Action) } + for i := range requestBody.Findings { + requestBody.Findings[i].Kind = strings.TrimSpace(requestBody.Findings[i].Kind) + requestBody.Findings[i].NormalizedPath = normalizeAnalyticsSubpath(requestBody.Findings[i].NormalizedPath) + requestBody.Findings[i].ObjectIDs = normalizeStringList(requestBody.Findings[i].ObjectIDs) + requestBody.Findings[i].BucketObjectURL = strings.TrimSpace(requestBody.Findings[i].BucketObjectURL) + requestBody.Findings[i].BucketObjectURLs = normalizeAnalyticsPathList(requestBody.Findings[i].BucketObjectURLs) + requestBody.Findings[i].AccessURLs = normalizeAnalyticsPathList(requestBody.Findings[i].AccessURLs) + requestBody.Findings[i].AvailableActions = normalizeStringList(requestBody.Findings[i].AvailableActions) + requestBody.Findings[i].DefaultAction = strings.TrimSpace(requestBody.Findings[i].DefaultAction) + if requestBody.Findings[i].Evidence != nil { + requestBody.Findings[i].Evidence.ObjectIDs = normalizeStringList(requestBody.Findings[i].Evidence.ObjectIDs) + requestBody.Findings[i].Evidence.AccessURLs = normalizeAnalyticsPathList(requestBody.Findings[i].Evidence.AccessURLs) + requestBody.Findings[i].Evidence.BucketObjectURLs = normalizeAnalyticsPathList(requestBody.Findings[i].Evidence.BucketObjectURLs) + requestBody.Findings[i].Evidence.SourcePaths = normalizeAnalyticsPathList(requestBody.Findings[i].Evidence.SourcePaths) + } + for j := range requestBody.Findings[i].Records { + record := &requestBody.Findings[i].Records[j] + record.ObjectID = strings.TrimSpace(record.ObjectID) + record.Checksum = strings.TrimSpace(record.Checksum) + record.NormalizedPath = normalizeAnalyticsSubpath(record.NormalizedPath) + record.CleanupScope = strings.TrimSpace(record.CleanupScope) + record.AccessURLs = normalizeAnalyticsPathList(record.AccessURLs) + for k := range record.AccessMethods { + record.AccessMethods[k].AccessID = strings.TrimSpace(record.AccessMethods[k].AccessID) + record.AccessMethods[k].Type = strings.TrimSpace(record.AccessMethods[k].Type) + record.AccessMethods[k].URL = strings.TrimSpace(record.AccessMethods[k].URL) + record.AccessMethods[k].Headers = normalizeStringList(record.AccessMethods[k].Headers) + } + for k := range record.AccessProbes { + record.AccessProbes[k].URL = strings.TrimSpace(record.AccessProbes[k].URL) + record.AccessProbes[k].Status = strings.TrimSpace(record.AccessProbes[k].Status) + record.AccessProbes[k].ErrorKind = strings.TrimSpace(record.AccessProbes[k].ErrorKind) + } + } + } response, err := handler.storageAnalytics.ApplyStorageCleanup( ctx.Context(), projectCtx.authorizationHeader, projectCtx.organization, projectCtx.project, - projectCtx.refName, - requestBody.GitSubpath, requestBody.SelectedRepoPaths, requestBody.Actions, - projectCtx.mirrorPath, - projectCtx.repo, - projectCtx.hash, - true, + requestBody.Findings, requestBody.DeleteRepoOrphans, requestBody.DeleteStaleDuplicates, requestBody.DeleteBucketOnlyObjects, @@ -409,10 +471,34 @@ func normalizeAnalyticsPathList(values []string) []string { return out } +func normalizeStringList(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + normalized := strings.TrimSpace(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + func (handler *Handler) writeGitAnalyticsError(ctx fiber.Ctx, projectID string, ref string, gitSubpath string, err error) error { statusCode := http.StatusBadGateway errorType := "integration_error" - if strings.Contains(strings.ToLower(err.Error()), "git tree path") { + errorMessage := strings.ToLower(err.Error()) + if strings.Contains(errorMessage, "storage children cursor") { + statusCode = http.StatusBadRequest + errorType = "invalid_request" + } else if strings.Contains(errorMessage, "cleanup apply") || strings.Contains(errorMessage, "cleanup finding") || strings.Contains(errorMessage, "cleanup action") || strings.Contains(errorMessage, "unsupported cleanup") || strings.Contains(errorMessage, "selected cleanup paths") { + statusCode = http.StatusBadRequest + errorType = "invalid_request" + } else if strings.Contains(errorMessage, "git tree path") { statusCode = http.StatusNotFound errorType = "not_found" } diff --git a/internal/server/http/git/storage_analytics_options_test.go b/internal/server/http/git/storage_analytics_options_test.go new file mode 100644 index 0000000..afdaf39 --- /dev/null +++ b/internal/server/http/git/storage_analytics_options_test.go @@ -0,0 +1,102 @@ +package git + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "net/http/httptest" + "os" + "testing" + + geckologging "github.com/calypr/gecko/internal/logging" + "github.com/gofiber/fiber/v3" +) + +func TestParseStorageChildrenLimit(t *testing.T) { + tests := []struct { + name string + raw string + want int + wantError bool + }{ + {name: "default", raw: "", want: defaultStorageChildrenLimit}, + {name: "explicit", raw: "25", want: 25}, + {name: "cap", raw: "5000", want: maxStorageChildrenLimit}, + {name: "invalid", raw: "nope", wantError: true}, + {name: "zero", raw: "0", wantError: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := parseStorageChildrenLimit(test.raw) + if test.wantError { + if err == nil { + t.Fatalf("expected error for limit %q", test.raw) + } + return + } + if err != nil { + t.Fatalf("parse limit: %v", err) + } + if got != test.want { + t.Fatalf("expected limit %d, got %d", test.want, got) + } + }) + } +} + +func TestParseStorageChildrenRequestOptionsReadsCursorAndSort(t *testing.T) { + app := fiber.New() + app.Get("/storage/children", func(ctx fiber.Ctx) error { + options, errResponse := parseStorageChildrenRequestOptions(ctx, "org/proj") + if errResponse != nil { + return errResponse.Write(ctx) + } + return ctx.JSON(map[string]any{ + "cursor": options.cursor, + "limit": options.limit, + "sort_by": options.sortBy, + "sort_order": options.sortOrder, + }) + }) + req := httptest.NewRequest(http.MethodGet, "/storage/children?limit=15&cursor=abc123&sort_by=name&sort_order=asc", nil) + resp, err := app.Test(req) + if err != nil { + t.Fatalf("run request: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + var body map[string]any + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + if body["cursor"] != "abc123" || body["sort_by"] != "name" || body["sort_order"] != "asc" || body["limit"].(float64) != 15 { + t.Fatalf("unexpected parsed options: %+v", body) + } +} + +func TestWriteGitAnalyticsErrorClassifiesCleanupApplyValidation(t *testing.T) { + handler := &Handler{logger: &geckologging.Handler{Logger: log.New(os.Stdout, "", 0)}} + app := fiber.New() + app.Post("/apply", func(ctx fiber.Ctx) error { + return handler.writeGitAnalyticsError(ctx, "org/proj", "main", "data", fmt.Errorf("cleanup apply requires findings from a prior audit; refusing to rebuild audit during apply")) + }) + resp, err := app.Test(httptest.NewRequest(http.MethodPost, "/apply", nil)) + if err != nil { + t.Fatalf("run request: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected status 400, got %d", resp.StatusCode) + } + var body map[string]any + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + errorBody, ok := body["error"].(map[string]any) + if !ok || errorBody["type"] != "invalid_request" { + t.Fatalf("expected invalid_request error, got %+v", body) + } +} From eab30efa132883cdc38af9e743a9d705a111ebbb Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Sat, 4 Jul 2026 12:52:46 -0700 Subject: [PATCH 18/36] improve gecko storage directory path serving --- internal/git/storage_analytics.go | 39 ++++++ internal/git/storage_analytics_test.go | 116 ++++++++++++++++++ internal/git/types.go | 5 + internal/server/http/git/register.go | 1 + internal/server/http/git/storage_analytics.go | 32 +++++ 5 files changed, 193 insertions(+) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 2690e04..edeb9e6 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -267,6 +267,44 @@ func (service *StorageAnalyticsService) BuildStorageChildren(ctx context.Context }, nil } +func (service *StorageAnalyticsService) BuildStorageFolder(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string, cursor string) (*GitStorageFolderResponse, error) { + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + if err != nil { + return nil, err + } + directory, err := repoDirectoryAggregate(index, gitSubpath) + if err != nil { + return nil, err + } + summaryAgg := summarizeSubtree(gitSubpath, inventory, recordsByChecksum, usageByObjectID, directory.DirectChildCount) + aggregates := cloneDirectoryChildren(directory.Children) + sortStorageAggregates(aggregates, sortBy, sortOrder) + page, err := storageChildrenPageForRequest(aggregates, hash, gitSubpath, sortBy, sortOrder, limit, cursor) + if err != nil { + return nil, err + } + pageInventory := filterInventoryForStorageChildren(inventory, page.items) + enriched := aggregateImmediateChildren(gitSubpath, pageInventory, recordsByChecksum, usageByObjectID, page.items) + return &GitStorageFolderResponse{ + Summary: GitStorageSummaryResponse{ + Path: summaryAgg.path, + FileCount: summaryAgg.fileCount, + RecordCount: summaryAgg.recordCount, + DirectChildCount: directory.DirectChildCount, + TotalBytes: summaryAgg.totalBytes, + DownloadCount: summaryAgg.downloadCount, + LastDownloadTime: formatOptionalTime(summaryAgg.lastDownload), + LatestUpdateTime: formatOptionalTime(summaryAgg.latestUpdate), + DuplicatePathCount: summaryAgg.duplicateCount, + }, + Children: GitStorageChildrenResponse{ + Items: storageChildrenItemsFromAggregates(enriched), + HasMore: page.hasMore, + NextCursor: page.nextCursor, + }, + }, nil +} + func (service *StorageAnalyticsService) BuildProjectDiffAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitProjectDiffAuditResponse, error) { _, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) if err != nil { @@ -395,6 +433,7 @@ func filterStorageChainSummary(summary GitStorageChainAuditSummary, findings []G for kind := range summary.CountsByKind { filteredCounts[kind] = 0 } + filteredCounts["bucket_syfon_git_complete"] = summary.CountsByKind["bucket_syfon_git_complete"] for _, finding := range findings { filteredCounts[finding.Kind]++ } diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 1f5a383..cd4176e 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -445,6 +445,94 @@ func TestBuildStorageChildrenCursorPagination(t *testing.T) { } } +func TestBuildStorageFolderCombinesSummaryAndChildren(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "data/c.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj"}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj"}, + {ObjectID: "obj-c", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj"}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 3}, + "obj-b": {ObjectID: "obj-b", DownloadCount: 2}, + "obj-c": {ObjectID: "obj-c", DownloadCount: 1}, + }, + } + service := NewStorageAnalyticsService(backend) + + summary, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build storage summary: %v", err) + } + children, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "") + if err != nil { + t.Fatalf("build storage children: %v", err) + } + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "") + if err != nil { + t.Fatalf("build storage folder: %v", err) + } + if folder.Summary != *summary { + t.Fatalf("expected folder summary to match summary endpoint\nfolder=%+v\nsummary=%+v", folder.Summary, *summary) + } + if len(folder.Children.Items) != len(children.Items) || folder.Children.HasMore != children.HasMore || folder.Children.NextCursor != children.NextCursor { + t.Fatalf("expected folder children page shape to match children endpoint\nfolder=%+v\nchildren=%+v", folder.Children, *children) + } + for i := range children.Items { + if folder.Children.Items[i].Path != children.Items[i].Path || folder.Children.Items[i].Type != children.Items[i].Type { + t.Fatalf("expected folder child ordering to match children endpoint\nfolder=%+v\nchildren=%+v", folder.Children, *children) + } + } + + next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", folder.Children.NextCursor) + if err != nil { + t.Fatalf("build storage folder next page: %v", err) + } + if len(next.Children.Items) != 1 || next.Children.Items[0].Path != "data/c.txt" { + t.Fatalf("unexpected next folder page: %+v", next.Children) + } + if next.Children.HasMore || next.Children.NextCursor != "" { + t.Fatalf("expected final folder page without cursor, got %+v", next.Children) + } +} + +func TestBuildStorageFolderSharesSummaryJoinWork(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj"}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj"}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 1}, + "obj-b": {ObjectID: "obj-b", DownloadCount: 1}, + }, + } + service := NewStorageAnalyticsService(backend) + + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "") + if err != nil { + t.Fatalf("build storage folder: %v", err) + } + if folder.Summary.FileCount != 2 || len(folder.Children.Items) != 1 || !folder.Children.HasMore { + t.Fatalf("unexpected folder response: %+v", folder) + } + if backend.bulkGetProjectRecordsCalls != 1 { + t.Fatalf("expected one joined Syfon record lookup for summary plus first page, got %d", backend.bulkGetProjectRecordsCalls) + } + if backend.listProjectFileUsageCalls != 1 { + t.Fatalf("expected one project usage lookup for exact summary totals, got %d", backend.listProjectFileUsageCalls) + } +} + func TestBuildStorageChildrenRejectsStaleCursor(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), @@ -917,6 +1005,34 @@ func TestStorageRepairPolicyCoversFindingKinds(t *testing.T) { } } +func TestFilterStorageChainSummaryPreservesCleanJoinCount(t *testing.T) { + summary := GitStorageChainAuditSummary{ + CountsByKind: map[string]int{ + "bucket_syfon_git_complete": 7, + "git_syfon_metadata_mismatch": 3, + "syfon_broken_bucket_mapping": 2, + }, + BucketObjectCount: 10, + SyfonRecordCount: 10, + GitTrackedFileCount: 10, + } + findings := []GitStorageChainFinding{ + {Kind: "git_syfon_metadata_mismatch", NormalizedPath: "data/a.txt"}, + } + + filtered := filterStorageChainSummary(summary, findings) + + if got := filtered.CountsByKind["bucket_syfon_git_complete"]; got != 7 { + t.Fatalf("expected clean-chain count to survive filtering, got %d in %+v", got, filtered) + } + if got := filtered.CountsByKind["git_syfon_metadata_mismatch"]; got != 1 { + t.Fatalf("expected filtered mismatch count from findings, got %d in %+v", got, filtered) + } + if got := filtered.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected filtered broken-mapping count to reset, got %d in %+v", got, filtered) + } +} + func TestApplyStorageCleanupRepairMatrix(t *testing.T) { tests := []struct { name string diff --git a/internal/git/types.go b/internal/git/types.go index 33b1fbe..4b97f77 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -335,6 +335,11 @@ type GitStorageChildrenResponse struct { NextCursor string `json:"next_cursor,omitempty"` } +type GitStorageFolderResponse struct { + Summary GitStorageSummaryResponse `json:"summary"` + Children GitStorageChildrenResponse `json:"children"` +} + type GitProjectDiffAuditRequest struct { GitSubpath string `json:"git_subpath,omitempty"` Ref string `json:"ref,omitempty"` diff --git a/internal/server/http/git/register.go b/internal/server/http/git/register.go index 4d18c3b..2562dc7 100644 --- a/internal/server/http/git/register.go +++ b/internal/server/http/git/register.go @@ -35,6 +35,7 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler gitGroup.Get("/projects/:orgTitle/:projectTitle/download/*", projectReadAuth, handler.handleGitProjectDownloadGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/storage/summary", projectReadAuth, handler.handleGitProjectStorageSummaryGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/storage/children", projectReadAuth, handler.handleGitProjectStorageChildrenGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/storage/folder", projectReadAuth, handler.handleGitProjectStorageFolderGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/thumbnail", handler.handleGitProjectThumbnailGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/presentationConfig", projectConfigReadAuth, handler.handleGitProjectPresentationConfigGET) diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index f578732..5337bf4 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -72,6 +72,38 @@ func (handler *Handler) handleGitProjectStorageChildrenGET(ctx fiber.Ctx) error return httputil.JSON(response, http.StatusOK).Write(ctx) } +func (handler *Handler) handleGitProjectStorageFolderGET(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + gitSubpath := normalizeAnalyticsSubpath(strings.TrimSpace(ctx.Query("git_subpath"))) + options, errResponse := parseStorageChildrenRequestOptions(ctx, projectCtx.projectID) + if errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + response, err := handler.storageAnalytics.BuildStorageFolder( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + gitSubpath, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + options.limit, + options.sortBy, + options.sortOrder, + options.cursor, + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + func parseStorageChildrenRequestOptions(ctx fiber.Ctx, projectID string) (storageChildrenRequestOptions, *httputil.ErrorResponse) { limit, err := parseStorageChildrenLimit(strings.TrimSpace(ctx.Query("limit"))) if err != nil { From 863b11bd65ce92378c672d0a9492ef5b8612ab60 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Sat, 4 Jul 2026 14:41:14 -0700 Subject: [PATCH 19/36] update funcs --- internal/git/storage_analytics.go | 93 ++++++++++++++++--- internal/git/storage_analytics_pipeline.go | 4 +- internal/git/storage_analytics_test.go | 24 +++-- internal/server/http/git/storage_analytics.go | 49 ++++++++-- 4 files changed, 138 insertions(+), 32 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index edeb9e6..32ebb20 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -20,6 +20,7 @@ const cleanupInactiveDays = 30 const projectJoinCacheTTL = 45 * time.Second const chainInputCacheTTL = 45 * time.Second const storageChainValidationDebugSampleLimit = 20 +const StorageFolderSummaryModeExact = "exact" const ( storageActionabilityAutoRepair = "auto_repair" @@ -107,6 +108,18 @@ type StorageAnalyticsService struct { chainInputCache map[string]cachedChainInputState } +type StorageFolderTimings struct { + DebugPrefix string + Logf func(format string, args ...any) +} + +func (timings *StorageFolderTimings) Record(stage string, duration time.Duration) { + if timings == nil || timings.Logf == nil { + return + } + timings.Logf("storage_folder_stage %s stage=%s duration_ms=%d", timings.DebugPrefix, stage, duration.Milliseconds()) +} + func NewStorageAnalyticsService(storage storageAnalyticsBackend) *StorageAnalyticsService { if storage == nil { return nil @@ -267,35 +280,87 @@ func (service *StorageAnalyticsService) BuildStorageChildren(ctx context.Context }, nil } -func (service *StorageAnalyticsService) BuildStorageFolder(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string, cursor string) (*GitStorageFolderResponse, error) { - index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) +func (service *StorageAnalyticsService) BuildStorageFolder(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string, cursor string, summaryMode string, timings *StorageFolderTimings) (*GitStorageFolderResponse, error) { + if strings.EqualFold(strings.TrimSpace(summaryMode), StorageFolderSummaryModeExact) { + exactStart := time.Now() + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + timings.Record("exact_join", time.Since(exactStart)) + if err != nil { + return nil, err + } + directoryStart := time.Now() + directory, err := repoDirectoryAggregate(index, gitSubpath) + timings.Record("directory_aggregate", time.Since(directoryStart)) + if err != nil { + return nil, err + } + summaryAgg := summarizeSubtree(gitSubpath, inventory, recordsByChecksum, usageByObjectID, directory.DirectChildCount) + pageStart := time.Now() + aggregates := cloneDirectoryChildren(directory.Children) + sortStorageAggregates(aggregates, sortBy, sortOrder) + page, err := storageChildrenPageForRequest(aggregates, hash, gitSubpath, sortBy, sortOrder, limit, cursor) + timings.Record("child_pagination", time.Since(pageStart)) + if err != nil { + return nil, err + } + enrichStart := time.Now() + pageInventory := filterInventoryForStorageChildren(inventory, page.items) + enriched := aggregateImmediateChildren(gitSubpath, pageInventory, recordsByChecksum, usageByObjectID, page.items) + timings.Record("enrich_children_page", time.Since(enrichStart)) + return &GitStorageFolderResponse{ + Summary: GitStorageSummaryResponse{ + Path: summaryAgg.path, + FileCount: summaryAgg.fileCount, + RecordCount: summaryAgg.recordCount, + DirectChildCount: directory.DirectChildCount, + TotalBytes: summaryAgg.totalBytes, + DownloadCount: summaryAgg.downloadCount, + LastDownloadTime: formatOptionalTime(summaryAgg.lastDownload), + LatestUpdateTime: formatOptionalTime(summaryAgg.latestUpdate), + DuplicatePathCount: summaryAgg.duplicateCount, + }, + Children: GitStorageChildrenResponse{ + Items: storageChildrenItemsFromAggregates(enriched), + HasMore: page.hasMore, + NextCursor: page.nextCursor, + }, + }, nil + } + + indexStart := time.Now() + index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) + timings.Record("load_repo_index", time.Since(indexStart)) if err != nil { return nil, err } + directoryStart := time.Now() directory, err := repoDirectoryAggregate(index, gitSubpath) + timings.Record("directory_aggregate", time.Since(directoryStart)) if err != nil { return nil, err } - summaryAgg := summarizeSubtree(gitSubpath, inventory, recordsByChecksum, usageByObjectID, directory.DirectChildCount) + pageStart := time.Now() aggregates := cloneDirectoryChildren(directory.Children) sortStorageAggregates(aggregates, sortBy, sortOrder) page, err := storageChildrenPageForRequest(aggregates, hash, gitSubpath, sortBy, sortOrder, limit, cursor) + timings.Record("child_pagination", time.Since(pageStart)) + if err != nil { + return nil, err + } + enrichStart := time.Now() + pageInventory := filterInventoryForStorageChildren(index.sidecar.Files, page.items) + enriched, err := service.enrichStorageChildrenPage(ctx, authorizationHeader, organization, project, gitSubpath, pageInventory, page.items) + timings.Record("enrich_children_page", time.Since(enrichStart)) if err != nil { return nil, err } - pageInventory := filterInventoryForStorageChildren(inventory, page.items) - enriched := aggregateImmediateChildren(gitSubpath, pageInventory, recordsByChecksum, usageByObjectID, page.items) + normalizedPath := normalizeRepoSubpath(gitSubpath) return &GitStorageFolderResponse{ Summary: GitStorageSummaryResponse{ - Path: summaryAgg.path, - FileCount: summaryAgg.fileCount, - RecordCount: summaryAgg.recordCount, - DirectChildCount: directory.DirectChildCount, - TotalBytes: summaryAgg.totalBytes, - DownloadCount: summaryAgg.downloadCount, - LastDownloadTime: formatOptionalTime(summaryAgg.lastDownload), - LatestUpdateTime: formatOptionalTime(summaryAgg.latestUpdate), - DuplicatePathCount: summaryAgg.duplicateCount, + Path: normalizedPath, + FileCount: directory.FileCount, + DirectChildCount: directory.DirectChildCount, + TotalBytes: directory.TotalBytes, }, Children: GitStorageChildrenResponse{ Items: storageChildrenItemsFromAggregates(enriched), diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 898cc5c..0d62b5c 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -947,9 +947,9 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum bucketMatches := matchedBucketObjectURLs(record, index.bucketObjectsByURL) switch classifyStorageFinding(record, index.bucketObjectsByURL) { case storageFindingBrokenBucketMap: - findings := buildChainRecordFindings("syfon_broken_bucket_mapping", record, gitPaths, bucketMatches, "Syfon access URL did not resolve through a configured bucket mapping.") + findings := buildChainRecordFindingsWithOptions("syfon_broken_bucket_mapping", record, gitPaths, bucketMatches, "Syfon access URL did not resolve through a configured bucket mapping.", true) acc.findings = append(acc.findings, findings...) - acc.addCount("syfon_broken_bucket_mapping", chainPathCount(gitPaths)) + acc.addCount("syfon_broken_bucket_mapping", len(findings)) case storageFindingObjectMissing: if len(gitPaths) > 0 { findings := buildChainRecordFindings("syfon_git_no_bucket", record, gitPaths, bucketMatches, "Git and Syfon matched, but the mapped bucket object does not exist.") diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index cd4176e..f58bbb1 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -473,7 +473,7 @@ func TestBuildStorageFolderCombinesSummaryAndChildren(t *testing.T) { if err != nil { t.Fatalf("build storage children: %v", err) } - folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "") + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "", StorageFolderSummaryModeExact, nil) if err != nil { t.Fatalf("build storage folder: %v", err) } @@ -489,7 +489,7 @@ func TestBuildStorageFolderCombinesSummaryAndChildren(t *testing.T) { } } - next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", folder.Children.NextCursor) + next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", folder.Children.NextCursor, StorageFolderSummaryModeExact, nil) if err != nil { t.Fatalf("build storage folder next page: %v", err) } @@ -501,7 +501,7 @@ func TestBuildStorageFolderCombinesSummaryAndChildren(t *testing.T) { } } -func TestBuildStorageFolderSharesSummaryJoinWork(t *testing.T) { +func TestBuildStorageFolderFastModeEnrichesOnlySelectedPage(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), @@ -518,18 +518,28 @@ func TestBuildStorageFolderSharesSummaryJoinWork(t *testing.T) { } service := NewStorageAnalyticsService(backend) - folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "") + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", nil) if err != nil { t.Fatalf("build storage folder: %v", err) } if folder.Summary.FileCount != 2 || len(folder.Children.Items) != 1 || !folder.Children.HasMore { t.Fatalf("unexpected folder response: %+v", folder) } + if folder.Summary.RecordCount != 0 || folder.Summary.DownloadCount != 0 || folder.Summary.DuplicatePathCount != 0 { + t.Fatalf("fast folder summary should not fake exact Syfon totals, got %+v", folder.Summary) + } + if folder.Children.Items[0].Path != "data/b.txt" || folder.Children.Items[0].RecordCount != 1 { + t.Fatalf("expected first page to be enriched with selected child record count, got %+v", folder.Children.Items) + } if backend.bulkGetProjectRecordsCalls != 1 { - t.Fatalf("expected one joined Syfon record lookup for summary plus first page, got %d", backend.bulkGetProjectRecordsCalls) + t.Fatalf("expected one page-scoped Syfon record lookup, got %d", backend.bulkGetProjectRecordsCalls) } - if backend.listProjectFileUsageCalls != 1 { - t.Fatalf("expected one project usage lookup for exact summary totals, got %d", backend.listProjectFileUsageCalls) + expectedChecksums := []string{"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} + if strings.Join(backend.bulkChecksums, ",") != strings.Join(expectedChecksums, ",") { + t.Fatalf("expected page-scoped checksums %v, got %v", expectedChecksums, backend.bulkChecksums) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected fast folder to skip project usage lookup, got %d calls", backend.listProjectFileUsageCalls) } } diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index 5337bf4..f240200 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -21,10 +21,11 @@ const defaultStorageChildrenLimit = 100 const maxStorageChildrenLimit = 1000 type storageChildrenRequestOptions struct { - limit int - cursor string - sortBy string - sortOrder string + limit int + cursor string + sortBy string + sortOrder string + summaryMode string } func (handler *Handler) handleGitProjectStorageSummaryGET(ctx fiber.Ctx) error { @@ -73,16 +74,38 @@ func (handler *Handler) handleGitProjectStorageChildrenGET(ctx fiber.Ctx) error } func (handler *Handler) handleGitProjectStorageFolderGET(ctx fiber.Ctx) error { + resolveStart := time.Now() projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) if errResponse != nil { return errResponse.Write(ctx) } + resolveDuration := time.Since(resolveStart) gitSubpath := normalizeAnalyticsSubpath(strings.TrimSpace(ctx.Query("git_subpath"))) options, errResponse := parseStorageChildrenRequestOptions(ctx, projectCtx.projectID) if errResponse != nil { errResponse.WriteLog(handler.logger) return errResponse.Write(ctx) } + summaryMode := strings.TrimSpace(options.summaryMode) + if summaryMode == "" { + summaryMode = "fast" + } + timings := &gitcore.StorageFolderTimings{ + DebugPrefix: fmt.Sprintf( + "project_id=%s ref=%s git_subpath=%q limit=%d sort_by=%q sort_order=%q cursor=%t summary_mode=%s", + projectCtx.projectID, + projectCtx.refName, + gitSubpath, + options.limit, + options.sortBy, + options.sortOrder, + options.cursor != "", + summaryMode, + ), + Logf: handler.logger.Info, + } + timings.Record("resolve_git_context", resolveDuration) + buildStart := time.Now() response, err := handler.storageAnalytics.BuildStorageFolder( ctx.Context(), projectCtx.authorizationHeader, @@ -97,11 +120,18 @@ func (handler *Handler) handleGitProjectStorageFolderGET(ctx fiber.Ctx) error { options.sortBy, options.sortOrder, options.cursor, + options.summaryMode, + timings, ) + timings.Record("build_folder", time.Since(buildStart)) if err != nil { return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) } - return httputil.JSON(response, http.StatusOK).Write(ctx) + writeStart := time.Now() + writeErr := httputil.JSON(response, http.StatusOK).Write(ctx) + timings.Record("write_response", time.Since(writeStart)) + handler.logger.Info("storage_folder_request_complete %s", timings.DebugPrefix) + return writeErr } func parseStorageChildrenRequestOptions(ctx fiber.Ctx, projectID string) (storageChildrenRequestOptions, *httputil.ErrorResponse) { @@ -110,10 +140,11 @@ func parseStorageChildrenRequestOptions(ctx fiber.Ctx, projectID string) (storag return storageChildrenRequestOptions{}, httputil.NewError("invalid_request", err.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) } return storageChildrenRequestOptions{ - limit: limit, - cursor: strings.TrimSpace(ctx.Query("cursor")), - sortBy: strings.TrimSpace(ctx.Query("sort_by")), - sortOrder: strings.TrimSpace(ctx.Query("sort_order")), + limit: limit, + cursor: strings.TrimSpace(ctx.Query("cursor")), + sortBy: strings.TrimSpace(ctx.Query("sort_by")), + sortOrder: strings.TrimSpace(ctx.Query("sort_order")), + summaryMode: strings.TrimSpace(ctx.Query("summary_mode")), }, nil } From db72864ef12cc6bc85c94d692c6ecb6e40cb8f38 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Sat, 4 Jul 2026 16:07:39 -0700 Subject: [PATCH 20/36] cleanup api to be faster --- internal/git/storage_analytics.go | 32 +-- internal/git/storage_analytics_test.go | 245 ++++++++++++++++-- internal/git/storage_children.go | 87 ++++++- internal/git/storage_index.go | 92 ++++++- internal/git/types.go | 1 + internal/server/http/git/storage_analytics.go | 2 +- 6 files changed, 404 insertions(+), 55 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 32ebb20..1631dcd 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -21,6 +21,8 @@ const projectJoinCacheTTL = 45 * time.Second const chainInputCacheTTL = 45 * time.Second const storageChainValidationDebugSampleLimit = 20 const StorageFolderSummaryModeExact = "exact" +const StorageFolderSummarySourceGitIndex = "git_index" +const StorageFolderSummarySourceExactJoin = "exact_join" const ( storageActionabilityAutoRepair = "auto_repair" @@ -310,6 +312,7 @@ func (service *StorageAnalyticsService) BuildStorageFolder(ctx context.Context, return &GitStorageFolderResponse{ Summary: GitStorageSummaryResponse{ Path: summaryAgg.path, + Source: StorageFolderSummarySourceExactJoin, FileCount: summaryAgg.fileCount, RecordCount: summaryAgg.recordCount, DirectChildCount: directory.DirectChildCount, @@ -328,45 +331,34 @@ func (service *StorageAnalyticsService) BuildStorageFolder(ctx context.Context, } indexStart := time.Now() - index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) + index, err := loadOrBuildRepoAnalyticsIndexWithTimings(ctx, mirrorPath, ref, repo, hash, timings) timings.Record("load_repo_index", time.Since(indexStart)) if err != nil { return nil, err } directoryStart := time.Now() - directory, err := repoDirectoryAggregate(index, gitSubpath) + directory, err := repoDirectoryServingIndex(index, gitSubpath) timings.Record("directory_aggregate", time.Since(directoryStart)) if err != nil { return nil, err } pageStart := time.Now() - aggregates := cloneDirectoryChildren(directory.Children) - sortStorageAggregates(aggregates, sortBy, sortOrder) - page, err := storageChildrenPageForRequest(aggregates, hash, gitSubpath, sortBy, sortOrder, limit, cursor) + children, err := storageChildrenResponseForServingIndex(directory, hash, gitSubpath, sortBy, sortOrder, limit, cursor) timings.Record("child_pagination", time.Since(pageStart)) if err != nil { return nil, err } - enrichStart := time.Now() - pageInventory := filterInventoryForStorageChildren(index.sidecar.Files, page.items) - enriched, err := service.enrichStorageChildrenPage(ctx, authorizationHeader, organization, project, gitSubpath, pageInventory, page.items) - timings.Record("enrich_children_page", time.Since(enrichStart)) - if err != nil { - return nil, err - } + timings.Record("git_index_remote_enrichment", 0) normalizedPath := normalizeRepoSubpath(gitSubpath) return &GitStorageFolderResponse{ Summary: GitStorageSummaryResponse{ Path: normalizedPath, - FileCount: directory.FileCount, - DirectChildCount: directory.DirectChildCount, - TotalBytes: directory.TotalBytes, - }, - Children: GitStorageChildrenResponse{ - Items: storageChildrenItemsFromAggregates(enriched), - HasMore: page.hasMore, - NextCursor: page.nextCursor, + Source: StorageFolderSummarySourceGitIndex, + FileCount: directory.directory.FileCount, + DirectChildCount: directory.directory.DirectChildCount, + TotalBytes: directory.directory.TotalBytes, }, + Children: children, }, nil } diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index f58bbb1..9670a92 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -477,7 +477,9 @@ func TestBuildStorageFolderCombinesSummaryAndChildren(t *testing.T) { if err != nil { t.Fatalf("build storage folder: %v", err) } - if folder.Summary != *summary { + expectedSummary := *summary + expectedSummary.Source = StorageFolderSummarySourceExactJoin + if folder.Summary != expectedSummary { t.Fatalf("expected folder summary to match summary endpoint\nfolder=%+v\nsummary=%+v", folder.Summary, *summary) } if len(folder.Children.Items) != len(children.Items) || folder.Children.HasMore != children.HasMore || folder.Children.NextCursor != children.NextCursor { @@ -501,48 +503,138 @@ func TestBuildStorageFolderCombinesSummaryAndChildren(t *testing.T) { } } -func TestBuildStorageFolderFastModeEnrichesOnlySelectedPage(t *testing.T) { +func TestBuildStorageFolderDefaultModeUsesGitIndexOnly(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ - "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), - "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "other/c.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 50), }) backend := &fakeStorageAnalyticsBackend{ projectRecords: []gintegrationsyfon.ProjectRecord{ {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj"}, {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj"}, + {ObjectID: "obj-c", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj"}, }, usageByObject: map[string]gintegrationsyfon.FileUsage{ "obj-a": {ObjectID: "obj-a", DownloadCount: 1}, "obj-b": {ObjectID: "obj-b", DownloadCount: 1}, + "obj-c": {ObjectID: "obj-c", DownloadCount: 1}, }, } service := NewStorageAnalyticsService(backend) - folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", nil) + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", nil) if err != nil { t.Fatalf("build storage folder: %v", err) } - if folder.Summary.FileCount != 2 || len(folder.Children.Items) != 1 || !folder.Children.HasMore { + if folder.Summary.Source != StorageFolderSummarySourceGitIndex { + t.Fatalf("expected default folder summary source %q, got %+v", StorageFolderSummarySourceGitIndex, folder.Summary) + } + if folder.Summary.FileCount != 3 || folder.Summary.RecordCount != 0 || folder.Summary.DownloadCount != 0 || folder.Summary.DuplicatePathCount != 0 { + t.Fatalf("git-index folder summary should not fake exact Syfon totals, got %+v", folder.Summary) + } + if len(folder.Children.Items) != 1 || !folder.Children.HasMore { t.Fatalf("unexpected folder response: %+v", folder) } - if folder.Summary.RecordCount != 0 || folder.Summary.DownloadCount != 0 || folder.Summary.DuplicatePathCount != 0 { - t.Fatalf("fast folder summary should not fake exact Syfon totals, got %+v", folder.Summary) + first := folder.Children.Items[0] + if first.Path != "data" || first.Type != "directory" || first.FileCount != 2 || first.TotalBytes != 300 { + t.Fatalf("expected first page to use Git directory aggregate, got %+v", folder.Children.Items) } - if folder.Children.Items[0].Path != "data/b.txt" || folder.Children.Items[0].RecordCount != 1 { - t.Fatalf("expected first page to be enriched with selected child record count, got %+v", folder.Children.Items) + if first.RecordCount != 0 || first.DownloadCount != 0 || first.LastDownloadTime != "" { + t.Fatalf("git-index child should not include Syfon/download enrichment, got %+v", first) } - if backend.bulkGetProjectRecordsCalls != 1 { - t.Fatalf("expected one page-scoped Syfon record lookup, got %d", backend.bulkGetProjectRecordsCalls) + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected default folder mode to skip Syfon record lookup, got %d", backend.bulkGetProjectRecordsCalls) } - expectedChecksums := []string{"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} - if strings.Join(backend.bulkChecksums, ",") != strings.Join(expectedChecksums, ",") { - t.Fatalf("expected page-scoped checksums %v, got %v", expectedChecksums, backend.bulkChecksums) + if len(backend.bulkChecksums) != 0 { + t.Fatalf("expected default folder mode to avoid descendant checksum expansion, got %v", backend.bulkChecksums) } if backend.listProjectFileUsageCalls != 0 { - t.Fatalf("expected fast folder to skip project usage lookup, got %d calls", backend.listProjectFileUsageCalls) + t.Fatalf("expected default folder mode to skip project usage lookup, got %d calls", backend.listProjectFileUsageCalls) + } + if backend.listProjectBucketObjectsCalls != 0 || backend.listProjectBucketSummaryCalls != 0 || backend.probeCalls != 0 { + t.Fatalf("expected default folder mode to skip bucket APIs, got objects=%d summary=%d probes=%d", backend.listProjectBucketObjectsCalls, backend.listProjectBucketSummaryCalls, backend.probeCalls) + } +} + +func TestBuildStorageFolderDefaultModeUsesPreSortedServingIndex(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/alpha.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), + "data/bravo.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 100), + "data/charlie.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 200), + }) + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + + tests := map[string]struct { + sortBy string + sortOrder string + expected []string + }{ + "bytes desc": {sortBy: "bytes", sortOrder: "desc", expected: []string{"data/alpha.txt", "data/charlie.txt", "data/bravo.txt"}}, + "bytes asc": {sortBy: "bytes", sortOrder: "asc", expected: []string{"data/bravo.txt", "data/charlie.txt", "data/alpha.txt"}}, + "name asc": {sortBy: "name", sortOrder: "asc", expected: []string{"data/alpha.txt", "data/bravo.txt", "data/charlie.txt"}}, + "name desc": {sortBy: "name", sortOrder: "desc", expected: []string{"data/charlie.txt", "data/bravo.txt", "data/alpha.txt"}}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, test.sortBy, test.sortOrder, "", "", nil) + if err != nil { + t.Fatalf("build storage folder: %v", err) + } + if got := storageChildResponsePaths(folder.Children.Items); strings.Join(got, ",") != strings.Join(test.expected[:2], ",") { + t.Fatalf("unexpected first page order: got %v want %v", got, test.expected[:2]) + } + if !folder.Children.HasMore || folder.Children.NextCursor == "" { + t.Fatalf("expected first page cursor, got %+v", folder.Children) + } + next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, test.sortBy, test.sortOrder, folder.Children.NextCursor, "", nil) + if err != nil { + t.Fatalf("build storage folder next page: %v", err) + } + if got := storageChildResponsePaths(next.Children.Items); strings.Join(got, ",") != test.expected[2] { + t.Fatalf("unexpected next page order: got %v want %v", got, test.expected[2:]) + } + if next.Children.HasMore || next.Children.NextCursor != "" { + t.Fatalf("expected final page without cursor, got %+v", next.Children) + } + }) + } +} + +func TestBuildStorageFolderDefaultModeUsesSidecarServingIndex(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + if err := PersistRepoAnalyticsIndex(context.Background(), mirrorPath, repo, refName, hash); err != nil { + t.Fatalf("persist repo analytics index: %v", err) + } + repoAnalyticsIndexCache.mu.Lock() + repoAnalyticsIndexCache.entries = map[string]*repoAnalyticsIndex{} + repoAnalyticsIndexCache.inflight = map[string]*inflightRepoAnalyticsIndex{} + repoAnalyticsIndexCache.mu.Unlock() + + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", nil) + if err != nil { + t.Fatalf("build storage folder from sidecar: %v", err) + } + if got := storageChildResponsePaths(folder.Children.Items); strings.Join(got, ",") != "data/b.txt" { + t.Fatalf("expected sidecar serving index to preserve bytes desc order, got %v", got) + } + if !folder.Children.HasMore || folder.Children.NextCursor == "" { + t.Fatalf("expected sidecar serving index cursor, got %+v", folder.Children) } } +func storageChildResponsePaths(items []GitStorageChildResponseItem) []string { + paths := make([]string, 0, len(items)) + for _, item := range items { + paths = append(paths, item.Path) + } + return paths +} + func TestBuildStorageChildrenRejectsStaleCursor(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), @@ -1969,6 +2061,84 @@ func TestBuildStorageChainAuditFiltersFindingsByKind(t *testing.T) { } } +func TestBuildStorageChainAuditFilteredBrokenMappingBypassesDefaultTruncation(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/unrelated.txt": lfsPointer("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 1), + }) + now := time.Date(2026, 7, 4, 12, 0, 0, 0, time.UTC) + records := make([]gintegrationsyfon.ProjectRecord, 0, 2) + probeResults := make(map[string]gintegrationsyfon.BulkStorageProbeResult) + for i := 0; i < 2; i++ { + checksum := fmt.Sprintf("%064x", i+1) + objectID := fmt.Sprintf("obj-broken-%03d", i) + objectURL := fmt.Sprintf("s3://legacy-bucket/broken-%03d.txt", i) + records = append(records, gintegrationsyfon.ProjectRecord{ + ObjectID: objectID, + Checksum: checksum, + Organization: "org", + Project: "proj", + Size: 100 + int64(i), + UpdatedAt: &now, + AccessURLs: []string{objectURL}, + }) + probeResults[storageProbeRequestKey(objectURL, 100+int64(i), checksum)] = gintegrationsyfon.BulkStorageProbeResult{ + ID: storageProbeRequestKey(objectURL, 100+int64(i), checksum), + ObjectURL: objectURL, + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "legacy-bucket"`, + ValidationStatus: "unverifiable", + } + } + bucketObjects := make([]gintegrationsyfon.ProjectBucketObject, 0, 501) + for i := 0; i < 501; i++ { + bucketObjects = append(bucketObjects, gintegrationsyfon.ProjectBucketObject{ + ObjectURL: fmt.Sprintf("s3://bucket/loose-%03d.txt", i), + Bucket: "bucket", + Key: fmt.Sprintf("loose-%03d.txt", i), + Path: fmt.Sprintf("loose-%03d.txt", i), + SizeBytes: 10, + }) + } + backend := &fakeStorageAnalyticsBackend{ + projectRecords: records, + bucketObjects: bucketObjects, + probeResults: probeResults, + } + service := NewStorageAnalyticsService(backend) + + truncated, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + FindingLimit: 500, + }) + if err != nil { + t.Fatalf("build truncated chain audit: %v", err) + } + if !truncated.Summary.FindingsTruncated || truncated.Summary.ReturnedFindings != 500 { + t.Fatalf("expected default-like truncated response, got %+v", truncated.Summary) + } + if got := truncated.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 2 { + t.Fatalf("expected summary to retain broken mapping count, got %+v", truncated.Summary) + } + assertNoChainFinding(t, truncated.Findings, "syfon_broken_bucket_mapping") + + filtered, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + FindingKind: "syfon_broken_bucket_mapping", + FindingLimit: -1, + }) + if err != nil { + t.Fatalf("build filtered chain audit: %v", err) + } + if filtered.Summary.FindingsTruncated || filtered.Summary.ReturnedFindings != 2 { + t.Fatalf("expected full filtered response, got %+v", filtered.Summary) + } + if len(filtered.Findings) != 2 { + t.Fatalf("expected filtered broken mapping detail rows, got %+v", filtered.Findings) + } + assertHasChainFinding(t, filtered.Findings, "syfon_broken_bucket_mapping", "syfon/obj-broken-000") + assertHasChainFinding(t, filtered.Findings, "syfon_broken_bucket_mapping", "syfon/obj-broken-001") +} + func TestBuildStorageChainAuditUsesScopedProjectRecordsForGitJoin(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -2553,6 +2723,49 @@ func TestPersistRepoAnalyticsIndexAndLoadExistingDirectoryWithoutLFSFiles(t *tes } } +func BenchmarkBuildStorageFolderDefaultModeLargeDirectoryPage(b *testing.B) { + children := make([]GitRepoAnalyticsChild, 1000) + for index := 0; index < 1000; index++ { + children[index] = GitRepoAnalyticsChild{ + Name: fmt.Sprintf("file-%04d.txt", index), + Path: fmt.Sprintf("data/file-%04d.txt", index), + Type: "file", + FileCount: 1, + TotalBytes: int64(index + 1), + } + } + hash := plumbing.NewHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + mirrorPath := filepath.Join(b.TempDir(), "mirror.git") + index := repoAnalyticsIndexFromSidecar(GitRepoAnalyticsIndexSidecar{ + SchemaVersion: repoAnalyticsIndexSchemaVersion, + CommitHash: hash.String(), + RefName: "main", + GeneratedAt: time.Now().UTC(), + Directories: []GitRepoAnalyticsDirectory{ + { + Path: "data", + DirectChildCount: len(children), + FileCount: len(children), + TotalBytes: 500500, + Children: children, + }, + }, + }) + repoAnalyticsIndexCache.put(mirrorPath, hash, index) + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + if _, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", "main", "data", mirrorPath, nil, hash, 100, "bytes", "desc", "", "", nil); err != nil { + b.Fatalf("warm storage folder index: %v", err) + } + + b.ReportAllocs() + b.ResetTimer() + for index := 0; index < b.N; index++ { + if _, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", "main", "data", mirrorPath, nil, hash, 100, "bytes", "desc", "", "", nil); err != nil { + b.Fatalf("build storage folder: %v", err) + } + } +} + func buildAnalyticsMirror(t *testing.T, files map[string]string) (*gogit.Repository, string, string, plumbing.Hash) { t.Helper() tempDir := t.TempDir() diff --git a/internal/git/storage_children.go b/internal/git/storage_children.go index fd6de56..6ea65da 100644 --- a/internal/git/storage_children.go +++ b/internal/git/storage_children.go @@ -62,22 +62,10 @@ func storageChildrenPageForRequest(items []storageAggregate, hash plumbing.Hash, if limit <= 0 { limit = len(items) } - cursor, err := decodeStorageChildrenCursor(rawCursor) + offset, err := storageChildrenCursorOffset(hash, gitSubpath, sortBy, sortOrder, rawCursor) if err != nil { return storageChildrenPage{}, err } - offset := cursor.Offset - if strings.TrimSpace(rawCursor) != "" { - expected := storageChildrenCursor{ - CommitHash: hash.String(), - GitSubpath: normalizeRepoSubpath(gitSubpath), - SortBy: strings.TrimSpace(sortBy), - SortOrder: strings.TrimSpace(sortOrder), - } - if cursor.CommitHash != expected.CommitHash || cursor.GitSubpath != expected.GitSubpath || cursor.SortBy != expected.SortBy || cursor.SortOrder != expected.SortOrder { - return storageChildrenPage{}, fmt.Errorf("storage children cursor does not match current request") - } - } if offset > len(items) { offset = len(items) } @@ -96,6 +84,79 @@ func storageChildrenPageForRequest(items []storageAggregate, hash plumbing.Hash, return page, nil } +func storageChildrenCursorOffset(hash plumbing.Hash, gitSubpath string, sortBy string, sortOrder string, rawCursor string) (int, error) { + cursor, err := decodeStorageChildrenCursor(rawCursor) + if err != nil { + return 0, err + } + if strings.TrimSpace(rawCursor) == "" { + return 0, nil + } + expected := storageChildrenCursor{ + CommitHash: hash.String(), + GitSubpath: normalizeRepoSubpath(gitSubpath), + SortBy: strings.TrimSpace(sortBy), + SortOrder: strings.TrimSpace(sortOrder), + } + if cursor.CommitHash != expected.CommitHash || cursor.GitSubpath != expected.GitSubpath || cursor.SortBy != expected.SortBy || cursor.SortOrder != expected.SortOrder { + return 0, fmt.Errorf("storage children cursor does not match current request") + } + return cursor.Offset, nil +} + +func storageChildrenResponseForServingIndex(directory repoAnalyticsDirectoryServingIndex, hash plumbing.Hash, gitSubpath string, sortBy string, sortOrder string, limit int, rawCursor string) (GitStorageChildrenResponse, error) { + order := sortedOrderForStorageChildrenRequest(directory, sortBy, sortOrder) + if limit <= 0 { + limit = len(order) + } + offset, err := storageChildrenCursorOffset(hash, gitSubpath, sortBy, sortOrder, rawCursor) + if err != nil { + return GitStorageChildrenResponse{}, err + } + if offset > len(order) { + offset = len(order) + } + end := offset + limit + if end > len(order) { + end = len(order) + } + items := make([]GitStorageChildResponseItem, 0, end-offset) + for _, childIndex := range order[offset:end] { + child := directory.directory.Children[childIndex] + items = append(items, GitStorageChildResponseItem{ + Name: child.Name, + Path: child.Path, + Type: child.Type, + FileCount: child.FileCount, + TotalBytes: child.TotalBytes, + }) + } + response := GitStorageChildrenResponse{ + Items: items, + HasMore: end < len(order), + } + if response.HasMore { + response.NextCursor = buildStorageChildrenCursor(hash, gitSubpath, sortBy, sortOrder, end) + } + return response, nil +} + +func sortedOrderForStorageChildrenRequest(directory repoAnalyticsDirectoryServingIndex, sortBy string, sortOrder string) []int { + desc := !strings.EqualFold(strings.TrimSpace(sortOrder), "asc") + switch strings.ToLower(strings.TrimSpace(sortBy)) { + case "name": + if desc { + return directory.nameDesc + } + return directory.nameAsc + default: + if desc { + return directory.bytesDesc + } + return directory.bytesAsc + } +} + func filterInventoryForStorageChildren(inventory []RepoInventoryFile, children []storageAggregate) []RepoInventoryFile { if len(children) == 0 || len(inventory) == 0 { return nil diff --git a/internal/git/storage_index.go b/internal/git/storage_index.go index 48fc234..f485bf8 100644 --- a/internal/git/storage_index.go +++ b/internal/git/storage_index.go @@ -38,8 +38,17 @@ type inflightRepoAnalyticsIndex struct { } type repoAnalyticsIndex struct { - sidecar GitRepoAnalyticsIndexSidecar - directoryLookup map[string]GitRepoAnalyticsDirectory + sidecar GitRepoAnalyticsIndexSidecar + directoryLookup map[string]GitRepoAnalyticsDirectory + directoryServing map[string]repoAnalyticsDirectoryServingIndex +} + +type repoAnalyticsDirectoryServingIndex struct { + directory GitRepoAnalyticsDirectory + bytesAsc []int + bytesDesc []int + nameAsc []int + nameDesc []int } func repoAnalyticsCacheKey(mirrorPath string, hash plumbing.Hash) string { @@ -88,16 +97,23 @@ func PersistRepoAnalyticsIndex(ctx context.Context, mirrorPath string, repo *gog return nil } -func loadOrBuildRepoAnalyticsIndex(_ context.Context, mirrorPath string, ref string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, error) { +func loadOrBuildRepoAnalyticsIndex(ctx context.Context, mirrorPath string, ref string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, error) { + return loadOrBuildRepoAnalyticsIndexWithTimings(ctx, mirrorPath, ref, repo, hash, nil) +} + +func loadOrBuildRepoAnalyticsIndexWithTimings(_ context.Context, mirrorPath string, ref string, repo *gogit.Repository, hash plumbing.Hash, timings *StorageFolderTimings) (*repoAnalyticsIndex, error) { cacheKey := repoAnalyticsCacheKey(mirrorPath, hash) repoAnalyticsIndexCache.mu.Lock() if cached := repoAnalyticsIndexCache.entries[cacheKey]; cached != nil { repoAnalyticsIndexCache.mu.Unlock() + timings.Record("repo_index_memory_cache_hit", 0) return cached, nil } if inflight := repoAnalyticsIndexCache.inflight[cacheKey]; inflight != nil { repoAnalyticsIndexCache.mu.Unlock() + waitStart := time.Now() <-inflight.done + timings.Record("repo_index_inflight_wait", time.Since(waitStart)) return inflight.index, inflight.err } inflight := &inflightRepoAnalyticsIndex{done: make(chan struct{})} @@ -110,22 +126,31 @@ func loadOrBuildRepoAnalyticsIndex(_ context.Context, mirrorPath string, ref str repoAnalyticsIndexCache.mu.Unlock() }() + sidecarStart := time.Now() sidecar, err := readRepoAnalyticsIndexSidecar(mirrorPath) + timings.Record("repo_index_sidecar_read_decode", time.Since(sidecarStart)) if err == nil && sidecar.SchemaVersion == repoAnalyticsIndexSchemaVersion && sidecar.CommitHash == hash.String() { + servingStart := time.Now() index := repoAnalyticsIndexFromSidecar(sidecar) + timings.Record("repo_index_sorted_order_build", time.Since(servingStart)) repoAnalyticsIndexCache.put(mirrorPath, hash, index) inflight.index = index return index, nil } + buildStart := time.Now() index, buildErr := buildRepoAnalyticsIndex(ref, repo, hash) + timings.Record("repo_index_git_tree_rebuild", time.Since(buildStart)) if buildErr != nil { inflight.err = buildErr return nil, buildErr } + writeStart := time.Now() if writeErr := writeRepoAnalyticsIndexSidecar(mirrorPath, index.sidecar); writeErr != nil { + timings.Record("repo_index_sidecar_write", time.Since(writeStart)) inflight.err = writeErr return nil, writeErr } + timings.Record("repo_index_sidecar_write", time.Since(writeStart)) repoAnalyticsIndexCache.put(mirrorPath, hash, index) inflight.index = index return index, nil @@ -165,13 +190,61 @@ func writeRepoAnalyticsIndexSidecar(mirrorPath string, sidecar GitRepoAnalyticsI func repoAnalyticsIndexFromSidecar(sidecar GitRepoAnalyticsIndexSidecar) *repoAnalyticsIndex { directoryLookup := make(map[string]GitRepoAnalyticsDirectory, len(sidecar.Directories)) + directoryServing := make(map[string]repoAnalyticsDirectoryServingIndex, len(sidecar.Directories)) for _, directory := range sidecar.Directories { directoryLookup[directory.Path] = directory + directoryServing[directory.Path] = buildRepoAnalyticsDirectoryServingIndex(directory) } return &repoAnalyticsIndex{ - sidecar: sidecar, - directoryLookup: directoryLookup, + sidecar: sidecar, + directoryLookup: directoryLookup, + directoryServing: directoryServing, + } +} + +func buildRepoAnalyticsDirectoryServingIndex(directory GitRepoAnalyticsDirectory) repoAnalyticsDirectoryServingIndex { + return repoAnalyticsDirectoryServingIndex{ + directory: directory, + bytesAsc: sortedChildIndexes(directory.Children, "bytes", "asc"), + bytesDesc: sortedChildIndexes(directory.Children, "bytes", "desc"), + nameAsc: sortedChildIndexes(directory.Children, "name", "asc"), + nameDesc: sortedChildIndexes(directory.Children, "name", "desc"), + } +} + +func sortedChildIndexes(children []GitRepoAnalyticsChild, sortBy string, sortOrder string) []int { + indexes := make([]int, len(children)) + for index := range children { + indexes[index] = index + } + desc := !strings.EqualFold(strings.TrimSpace(sortOrder), "asc") + switch strings.ToLower(strings.TrimSpace(sortBy)) { + case "name": + sort.Slice(indexes, func(i, j int) bool { + left := strings.ToLower(children[indexes[i]].Name) + right := strings.ToLower(children[indexes[j]].Name) + if desc { + return left > right + } + return left < right + }) + default: + sort.Slice(indexes, func(i, j int) bool { + left := children[indexes[i]] + right := children[indexes[j]] + if left.TotalBytes != right.TotalBytes { + if desc { + return left.TotalBytes > right.TotalBytes + } + return left.TotalBytes < right.TotalBytes + } + if left.Type != right.Type { + return left.Type == "directory" + } + return strings.ToLower(left.Name) < strings.ToLower(right.Name) + }) } + return indexes } func buildRepoAnalyticsIndex(ref string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, error) { @@ -306,6 +379,15 @@ func repoDirectoryAggregate(index *repoAnalyticsIndex, gitSubpath string) (GitRe return directory, nil } +func repoDirectoryServingIndex(index *repoAnalyticsIndex, gitSubpath string) (repoAnalyticsDirectoryServingIndex, error) { + normalizedPath := normalizeRepoSubpath(gitSubpath) + directory, ok := index.directoryServing[normalizedPath] + if !ok { + return repoAnalyticsDirectoryServingIndex{}, fmt.Errorf("load git tree path %s: directory not found", normalizedPath) + } + return directory, nil +} + func cloneDirectoryChildren(children []GitRepoAnalyticsChild) []storageAggregate { out := make([]storageAggregate, 0, len(children)) for _, child := range children { diff --git a/internal/git/types.go b/internal/git/types.go index 4b97f77..0c34de5 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -307,6 +307,7 @@ type GitRepoAnalyticsIndexSidecar struct { type GitStorageSummaryResponse struct { Path string `json:"path"` + Source string `json:"source,omitempty"` FileCount int `json:"file_count"` RecordCount int `json:"record_count"` DirectChildCount int `json:"direct_child_count"` diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index f240200..231a0e7 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -88,7 +88,7 @@ func (handler *Handler) handleGitProjectStorageFolderGET(ctx fiber.Ctx) error { } summaryMode := strings.TrimSpace(options.summaryMode) if summaryMode == "" { - summaryMode = "fast" + summaryMode = gitcore.StorageFolderSummarySourceGitIndex } timings := &gitcore.StorageFolderTimings{ DebugPrefix: fmt.Sprintf( From 90485af9d29b38d4189f72791a1cc81f32f3c5f6 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 7 Jul 2026 13:24:55 -0700 Subject: [PATCH 21/36] optimize gecko routes in audit --- internal/git/repository.go | 28 -- internal/git/response.go | 39 +- internal/git/service_test.go | 192 -------- internal/git/storage_analytics.go | 174 +++++++- internal/git/storage_analytics_pipeline.go | 308 ++++++++++++- internal/git/storage_analytics_test.go | 411 ++++++++++++++---- internal/git/storage_analytics_timing.go | 49 +++ internal/git/tree_enrichment.go | 173 ++++++++ internal/git/tree_response_test.go | 326 ++++++++++++++ internal/integrations/syfon/adapter.go | 131 +++++- internal/integrations/syfon/adapter_test.go | 117 +++++ internal/server/http/git/storage_analytics.go | 14 +- 12 files changed, 1599 insertions(+), 363 deletions(-) create mode 100644 internal/git/tree_enrichment.go create mode 100644 internal/git/tree_response_test.go diff --git a/internal/git/repository.go b/internal/git/repository.go index 397afb1..a6a0b40 100644 --- a/internal/git/repository.go +++ b/internal/git/repository.go @@ -9,7 +9,6 @@ import ( "regexp" "strconv" "strings" - "time" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" @@ -141,33 +140,6 @@ func OpenRepository(path string) (*gogit.Repository, error) { return repo, nil } -func lookupGitPathLastModified(repo *gogit.Repository, from plumbing.Hash, path string) (*time.Time, error) { - normalizedPath := strings.Trim(strings.TrimSpace(path), "/") - if normalizedPath == "" { - return nil, nil - } - - iter, err := repo.Log(&gogit.LogOptions{ - From: from, - Order: gogit.LogOrderCommitterTime, - PathFilter: func(candidate string) bool { - trimmed := strings.Trim(strings.TrimSpace(candidate), "/") - return trimmed == normalizedPath || strings.HasPrefix(trimmed, normalizedPath+"/") - }, - }) - if err != nil { - return nil, err - } - defer iter.Close() - - commit, err := iter.Next() - if err != nil { - return nil, err - } - lastModifiedAt := commit.Committer.When.UTC() - return &lastModifiedAt, nil -} - func ParseGitLFSPointer(content []byte) *GitLFSPointerInfo { trimmed := strings.TrimSpace(string(content)) if trimmed == "" { diff --git a/internal/git/response.go b/internal/git/response.go index 4e19bee..e965a6d 100644 --- a/internal/git/response.go +++ b/internal/git/response.go @@ -63,38 +63,15 @@ func BuildGitTreeResponse(projectID string, ref string, path string, repo *gogit truncated = true } - for index := range entries { - entry := &entries[index] - if entry.Type != "blob" { - if options.IncludeLastModified { - if lastModifiedAt, err := lookupGitPathLastModified(repo, hash, entry.Path); err == nil && lastModifiedAt != nil { - entry.LastModifiedAt = lastModifiedAt - } - } - continue - } - - needsFileOpen := options.IncludeSize || options.IncludeLFSPointer - if needsFileOpen { - if file, err := tree.File(entry.Name); err == nil { - if options.IncludeSize { - entry.Size = file.Size - } - if options.IncludeLFSPointer { - if reader, err := file.Reader(); err == nil { - contentBytes, readErr := io.ReadAll(io.LimitReader(reader, 2048)) - _ = reader.Close() - if readErr == nil { - entry.LFSPointer = ParseGitLFSPointer(contentBytes) - } - } - } - } + enrichTreeEntries(tree, entries, options) + if options.IncludeLastModified { + lastModifiedByPath, err := lookupGitPathsLastModified(repo, hash, entries) + if err != nil { + return nil, fmt.Errorf("lookup git tree last modified times: %w", err) } - - if options.IncludeLastModified { - if lastModifiedAt, err := lookupGitPathLastModified(repo, hash, entry.Path); err == nil && lastModifiedAt != nil { - entry.LastModifiedAt = lastModifiedAt + for index := range entries { + if lastModifiedAt, ok := lastModifiedByPath[entries[index].Path]; ok { + entries[index].LastModifiedAt = &lastModifiedAt } } } diff --git a/internal/git/service_test.go b/internal/git/service_test.go index 5b0859e..d59a361 100644 --- a/internal/git/service_test.go +++ b/internal/git/service_test.go @@ -183,198 +183,6 @@ func TestBuildGitRefsResponseIncludesRemoteBranches(t *testing.T) { } } -func TestBuildGitResponsesDetectLFSPointers(t *testing.T) { - tempDir := t.TempDir() - sourcePath := filepath.Join(tempDir, "source") - repo, err := gogit.PlainInit(sourcePath, false) - if err != nil { - t.Fatalf("init source repo: %v", err) - } - worktree, err := repo.Worktree() - if err != nil { - t.Fatalf("load worktree: %v", err) - } - pointerContent := strings.Join([]string{ - "version https://git-lfs.github.com/spec/v1", - "oid sha256:0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83", - "size 3780184021", - "", - }, "\n") - if err := os.MkdirAll(filepath.Join(sourcePath, "data"), 0o755); err != nil { - t.Fatalf("create data dir: %v", err) - } - if err := os.WriteFile(filepath.Join(sourcePath, "data", "tcga.tumor.ensembl.tsv"), []byte(pointerContent), 0o644); err != nil { - t.Fatalf("write lfs pointer file: %v", err) - } - if _, err := worktree.Add("data/tcga.tumor.ensembl.tsv"); err != nil { - t.Fatalf("add lfs pointer file: %v", err) - } - if _, err := worktree.Commit("add lfs pointer", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { - t.Fatalf("commit lfs pointer file: %v", err) - } - - mirrorPath := filepath.Join(tempDir, "mirror.git") - if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { - t.Fatalf("sync mirror: %v", err) - } - mirrorRepo, err := OpenRepository(mirrorPath) - if err != nil { - t.Fatalf("open mirror: %v", err) - } - refName, hash, err := ResolveGitReference(mirrorRepo, "", "") - if err != nil { - t.Fatalf("resolve HEAD: %v", err) - } - treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "data", mirrorRepo, hash, GitTreeResponseOptions{ - IncludeLFSPointer: true, - IncludeSize: true, - }) - if err != nil { - t.Fatalf("build tree response: %v", err) - } - if len(treeResponse.Entries) != 1 { - t.Fatalf("expected one tree entry, got %+v", treeResponse.Entries) - } - treePointer := treeResponse.Entries[0].LFSPointer - if treePointer == nil { - t.Fatalf("expected tree entry to be marked as lfs pointer, got %+v", treeResponse.Entries[0]) - } - if treePointer.OID != "0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83" { - t.Fatalf("unexpected lfs oid: %q", treePointer.OID) - } - if treePointer.Size != 3780184021 { - t.Fatalf("unexpected lfs size: %d", treePointer.Size) - } - - fileResponse, err := BuildGitFileResponse("org-a/proj-a", refName, "data/tcga.tumor.ensembl.tsv", mirrorRepo, hash) - if err != nil { - t.Fatalf("build file response: %v", err) - } - if fileResponse.LFSPointer == nil { - t.Fatalf("expected file response to include lfs pointer metadata") - } - if fileResponse.LFSPointer.OID != treePointer.OID { - t.Fatalf("expected matching lfs oid, got %q and %q", fileResponse.LFSPointer.OID, treePointer.OID) - } -} - -func TestBuildGitTreeResponseDefaultsToCheapFields(t *testing.T) { - tempDir := t.TempDir() - sourcePath := filepath.Join(tempDir, "source") - repo, err := gogit.PlainInit(sourcePath, false) - if err != nil { - t.Fatalf("init source repo: %v", err) - } - worktree, err := repo.Worktree() - if err != nil { - t.Fatalf("load worktree: %v", err) - } - if err := os.WriteFile(filepath.Join(sourcePath, "README.md"), []byte("hello gecko"), 0o644); err != nil { - t.Fatalf("write readme: %v", err) - } - if _, err := worktree.Add("README.md"); err != nil { - t.Fatalf("add readme: %v", err) - } - if _, err := worktree.Commit("initial commit", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { - t.Fatalf("commit readme: %v", err) - } - - mirrorPath := filepath.Join(tempDir, "mirror.git") - if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { - t.Fatalf("sync mirror: %v", err) - } - mirrorRepo, err := OpenRepository(mirrorPath) - if err != nil { - t.Fatalf("open mirror: %v", err) - } - refName, hash, err := ResolveGitReference(mirrorRepo, "", "") - if err != nil { - t.Fatalf("resolve HEAD: %v", err) - } - - treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{}) - if err != nil { - t.Fatalf("build tree response: %v", err) - } - if treeResponse.EntryCount != 1 { - t.Fatalf("expected entry count 1, got %d", treeResponse.EntryCount) - } - if treeResponse.Truncated { - t.Fatal("expected non-truncated response by default") - } - if len(treeResponse.Entries) != 1 { - t.Fatalf("expected one tree entry, got %+v", treeResponse.Entries) - } - if treeResponse.Entries[0].Size != 0 { - t.Fatalf("expected default tree response to omit size, got %d", treeResponse.Entries[0].Size) - } - if treeResponse.Entries[0].LFSPointer != nil { - t.Fatalf("expected default tree response to omit lfs pointer, got %+v", treeResponse.Entries[0].LFSPointer) - } - if treeResponse.Entries[0].LastModifiedAt != nil { - t.Fatalf("expected default tree response to omit last modified, got %+v", treeResponse.Entries[0].LastModifiedAt) - } -} - -func TestBuildGitTreeResponseHonorsLimitBeforeEnrichment(t *testing.T) { - tempDir := t.TempDir() - sourcePath := filepath.Join(tempDir, "source") - repo, err := gogit.PlainInit(sourcePath, false) - if err != nil { - t.Fatalf("init source repo: %v", err) - } - worktree, err := repo.Worktree() - if err != nil { - t.Fatalf("load worktree: %v", err) - } - for _, name := range []string{"a.txt", "b.txt", "c.txt"} { - if err := os.WriteFile(filepath.Join(sourcePath, name), []byte(name), 0o644); err != nil { - t.Fatalf("write %s: %v", name, err) - } - if _, err := worktree.Add(name); err != nil { - t.Fatalf("add %s: %v", name, err) - } - } - if _, err := worktree.Commit("add files", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { - t.Fatalf("commit files: %v", err) - } - - mirrorPath := filepath.Join(tempDir, "mirror.git") - if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { - t.Fatalf("sync mirror: %v", err) - } - mirrorRepo, err := OpenRepository(mirrorPath) - if err != nil { - t.Fatalf("open mirror: %v", err) - } - refName, hash, err := ResolveGitReference(mirrorRepo, "", "") - if err != nil { - t.Fatalf("resolve HEAD: %v", err) - } - - treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{ - IncludeSize: true, - Limit: 2, - }) - if err != nil { - t.Fatalf("build tree response: %v", err) - } - if !treeResponse.Truncated { - t.Fatal("expected truncated response when limit is smaller than entry count") - } - if treeResponse.EntryCount != 3 { - t.Fatalf("expected total entry count 3, got %d", treeResponse.EntryCount) - } - if len(treeResponse.Entries) != 2 { - t.Fatalf("expected two returned entries, got %+v", treeResponse.Entries) - } - for _, entry := range treeResponse.Entries { - if entry.Size == 0 { - t.Fatalf("expected size to be included for limited entry %+v", entry) - } - } -} - func TestBuildGitManifestResponseRecursesAndPaginates(t *testing.T) { tempDir := t.TempDir() sourcePath := filepath.Join(tempDir, "source") diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 1631dcd..b2dda29 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "net/http" "path" "sort" "strings" @@ -19,6 +20,8 @@ import ( const cleanupInactiveDays = 30 const projectJoinCacheTTL = 45 * time.Second const chainInputCacheTTL = 45 * time.Second +const chainProjectRecordCacheMaxAge = 30 * time.Minute +const projectFileUsageBulkChunkSize = 5000 const storageChainValidationDebugSampleLimit = 20 const StorageFolderSummaryModeExact = "exact" const StorageFolderSummarySourceGitIndex = "git_index" @@ -89,10 +92,13 @@ type storageAnalyticsBackend interface { ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) ListProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectRecord, error) + GetProjectMetricsSummary(ctx context.Context, authorizationHeader string, organization string, project string) (*gintegrationsyfon.ProjectMetricsSummary, error) ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]gintegrationsyfon.ProjectRecord, error) + ListProjectFileUsageByObjectIDs(ctx context.Context, authorizationHeader string, organization string, project string, objectIDs []string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) ListProjectFileUsage(ctx context.Context, authorizationHeader string, organization string, project string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) + ListProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) ListProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) BulkListStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) @@ -102,12 +108,14 @@ type storageAnalyticsBackend interface { } type StorageAnalyticsService struct { - storage storageAnalyticsBackend - projectJoinMu sync.RWMutex - projectJoinCache map[string]cachedProjectJoinState - projectJoinWork map[string]*inflightProjectJoinState - chainInputMu sync.RWMutex - chainInputCache map[string]cachedChainInputState + storage storageAnalyticsBackend + projectJoinMu sync.RWMutex + projectJoinCache map[string]cachedProjectJoinState + projectJoinWork map[string]*inflightProjectJoinState + chainInputMu sync.RWMutex + chainInputCache map[string]cachedChainInputState + projectAuditCache map[string]cachedProjectAuditRecordState + projectAuditWork map[string]*inflightProjectAuditRecordState } type StorageFolderTimings struct { @@ -127,10 +135,12 @@ func NewStorageAnalyticsService(storage storageAnalyticsBackend) *StorageAnalyti return nil } return &StorageAnalyticsService{ - storage: storage, - projectJoinCache: map[string]cachedProjectJoinState{}, - projectJoinWork: map[string]*inflightProjectJoinState{}, - chainInputCache: map[string]cachedChainInputState{}, + storage: storage, + projectJoinCache: map[string]cachedProjectJoinState{}, + projectJoinWork: map[string]*inflightProjectJoinState{}, + chainInputCache: map[string]cachedChainInputState{}, + projectAuditCache: map[string]cachedProjectAuditRecordState{}, + projectAuditWork: map[string]*inflightProjectAuditRecordState{}, } } @@ -224,6 +234,25 @@ type cachedChainInputState struct { bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject } +type cachedProjectAuditRecordState struct { + records []gintegrationsyfon.ProjectRecord + validator projectAuditRecordValidator + cachedAt time.Time +} + +type inflightProjectAuditRecordState struct { + done chan struct{} + records []gintegrationsyfon.ProjectRecord + validator projectAuditRecordValidator + err error +} + +type projectAuditRecordValidator struct { + RecordCount int + RecordLatestUpdatedTime string + RecordRevision string +} + func BuildGitRepoInventory(ref string, gitSubpath string, repo *gogit.Repository, hash plumbing.Hash) ([]RepoInventoryFile, error) { index, err := buildRepoAnalyticsIndex(ref, repo, hash) if err != nil { @@ -432,13 +461,28 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co options.Timings.StageStart("chain_setup_total") inputs, err := service.loadStorageChainInputs(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, bucketMode, bucketPathPrefix, options.Timings) options.Timings.Record("chain_setup_total", time.Since(start)) + if inputs != nil && inputs.recordSet != nil { + options.Timings.RecordMemory( + "chain_setup_total", + "git_files", len(inputs.inventory), + "syfon_records", countRecordStates(inputs.recordSet.allProjectRecords), + "bucket_objects", len(inputs.bucketObjects), + ) + } if err != nil { return nil, err } storageViewStart := time.Now() options.Timings.StageStart("storage_view") - storageView, err := service.buildStorageChainView(ctx, authorizationHeader, inputs.recordSet, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.Timings) + storageView, err := service.buildStorageChainView(ctx, authorizationHeader, organization, project, inputs.recordSet, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.Timings) options.Timings.Record("storage_view", time.Since(storageViewStart)) + if storageView != nil { + options.Timings.RecordMemory( + "storage_view", + "syfon_records", countRecordStates(storageView.allProjectRecords), + "bucket_objects", len(storageView.bucketObjectsByURL), + ) + } if err != nil { return nil, err } @@ -447,6 +491,13 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co includeBucketOrigin := bucketMode != StorageChainBucketModeValidate && storageView.bucketInventoryAvailable model := buildStorageChainAuditModel(gitSubpath, inputs.inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL, includeBucketOrigin) options.Timings.Record("model_build", time.Since(modelStart)) + options.Timings.RecordMemory( + "model_build", + "total_findings", len(model.Findings), + "syfon_records", countRecordStates(storageView.allProjectRecords), + "bucket_objects", len(storageView.bucketObjectsByURL), + "git_files", len(inputs.inventory), + ) model.Summary.BucketInventoryAvailable = storageView.bucketInventoryAvailable model.Summary.BucketInventoryError = storageView.bucketInventoryError model.Summary.ValidationMode = validationMode @@ -458,10 +509,17 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co } filteredFindings := filterStorageChainFindingsByKind(model.Findings, options.FindingKind) summary := filterStorageChainSummary(model.Summary, filteredFindings) - findings := limitStorageChainFindings(filteredFindings, options.FindingLimit) + findings := limitStorageChainFindings(filteredFindings, options.FindingLimit, options.FindingKind) summary.ReturnedFindings = len(findings) summary.FindingLimit = options.FindingLimit summary.FindingsTruncated = len(findings) < len(filteredFindings) + options.Timings.RecordMemory( + "response_shape", + "total_findings", summary.TotalFindings, + "filtered_findings", len(filteredFindings), + "returned_findings", len(findings), + "finding_limit", options.FindingLimit, + ) return &GitStorageChainAuditResponse{ Findings: findings, Groups: summarizeChainIssueGroups(filteredFindings), @@ -499,10 +557,22 @@ func filterStorageChainSummary(summary GitStorageChainAuditSummary, findings []G return summary } -func limitStorageChainFindings(findings []GitStorageChainFinding, limit int) []GitStorageChainFinding { +func limitStorageChainFindings(findings []GitStorageChainFinding, limit int, kind string) []GitStorageChainFinding { if limit <= 0 || limit >= len(findings) { return append([]GitStorageChainFinding(nil), findings...) } + if strings.TrimSpace(kind) == "" { + limited := make([]GitStorageChainFinding, 0, len(findings)) + countsByKind := make(map[string]int) + for _, finding := range findings { + if countsByKind[finding.Kind] >= limit { + continue + } + limited = append(limited, finding) + countsByKind[finding.Kind]++ + } + return limited + } return append([]GitStorageChainFinding(nil), findings[:limit]...) } @@ -1015,7 +1085,8 @@ func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context inflight.err = fmt.Errorf("lookup syfon project records by checksum: %w", err) return nil, nil, inflight.err } - usageByObjectID, err := service.storage.ListProjectFileUsage(ctx, authorizationHeader, organization, project, cleanupInactiveDays) + objectIDs := projectRecordObjectIDs(recordsByChecksumRaw) + usageByObjectID, err := service.listProjectFileUsageByObjectIDs(ctx, authorizationHeader, organization, project, objectIDs, cleanupInactiveDays) if err != nil { inflight.err = fmt.Errorf("list syfon project file usage: %w", err) return nil, nil, inflight.err @@ -1046,6 +1117,40 @@ func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context return recordsByChecksum, usageByObjectID, nil } +func projectRecordObjectIDs(recordsByChecksum map[string][]gintegrationsyfon.ProjectRecord) []string { + objectIDs := make([]string, 0) + for _, records := range recordsByChecksum { + for _, record := range records { + objectIDs = append(objectIDs, record.ObjectID) + } + } + return uniqueStrings(objectIDs) +} + +func (service *StorageAnalyticsService) listProjectFileUsageByObjectIDs(ctx context.Context, authorizationHeader string, organization string, project string, objectIDs []string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) { + usageByObjectID := make(map[string]gintegrationsyfon.FileUsage) + if len(objectIDs) == 0 { + return usageByObjectID, nil + } + for start := 0; start < len(objectIDs); start += projectFileUsageBulkChunkSize { + end := start + projectFileUsageBulkChunkSize + if end > len(objectIDs) { + end = len(objectIDs) + } + chunkUsage, err := service.storage.ListProjectFileUsageByObjectIDs(ctx, authorizationHeader, organization, project, objectIDs[start:end], inactiveDays) + if err != nil { + if gintegrationsyfon.IsHTTPStatus(err, http.StatusNotFound, http.StatusMethodNotAllowed) { + return service.storage.ListProjectFileUsage(ctx, authorizationHeader, organization, project, inactiveDays) + } + return nil, err + } + for objectID, usage := range chunkUsage { + usageByObjectID[objectID] = usage + } + } + return usageByObjectID, nil +} + func (service *StorageAnalyticsService) projectJoinCacheKey(organization string, project string, commitHash string) string { return strings.TrimSpace(organization) + "/" + strings.TrimSpace(project) + "::" + strings.TrimSpace(commitHash) } @@ -1872,6 +1977,9 @@ func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[st if hasExactPathBucketMismatch(record, bucketMatches, bucketObjectsByURL) { return storageFindingBrokenBucketMap } + if recordHasMatchedCanonicalAccessProbe(record) { + return storageFindingNone + } if inventoryHasValidationMismatch(record, bucketMatches, bucketObjectsByURL) { return storageFindingValidationMismatch } @@ -1976,6 +2084,41 @@ func recordHasValidationMismatchProbe(record projectRecordState) bool { return false } +func recordHasMatchedCanonicalAccessProbe(record projectRecordState) bool { + rawURLs := make(map[string]struct{}, len(record.AccessURLs)) + for _, accessURL := range record.AccessURLs { + if trimmed := strings.TrimSpace(accessURL); trimmed != "" { + rawURLs[trimmed] = struct{}{} + } + } + canonicalURLs := make(map[string]struct{}, len(record.CanonicalAccessURLs)) + for _, accessURL := range record.CanonicalAccessURLs { + trimmed := strings.TrimSpace(accessURL) + if trimmed == "" { + continue + } + if _, raw := rawURLs[trimmed]; !raw { + canonicalURLs[trimmed] = struct{}{} + } + } + if len(canonicalURLs) == 0 { + return false + } + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Status) != "present" { + continue + } + if _, ok := canonicalURLs[strings.TrimSpace(probe.ObjectURL)]; !ok { + continue + } + switch strings.TrimSpace(probe.ValidationStatus) { + case "", "not_requested", "matched": + return true + } + } + return false +} + func inventoryHasValidationMismatch(record projectRecordState, bucketObjectURLs []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { if len(bucketObjectURLs) == 0 { return false @@ -2385,6 +2528,9 @@ func classifyRawAccessURLFindings(record projectRecordState) storageFindingKind if len(record.AccessProbes) == 0 { return storageFindingNone } + if recordHasMatchedCanonicalAccessProbe(record) { + return storageFindingNone + } probesByURL := make(map[string][]gintegrationsyfon.BulkStorageProbeResult, len(record.AccessProbes)) for _, probe := range record.AccessProbes { probesByURL[strings.TrimSpace(probe.ObjectURL)] = append(probesByURL[strings.TrimSpace(probe.ObjectURL)], probe) diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 0d62b5c..c21d154 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -100,18 +100,20 @@ func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Conte timings.StageStart("repo_index") inventory, err := service.loadStorageChainInventory(ctx, ref, gitSubpath, mirrorPath, repo, hash) timings.Record("repo_index", time.Since(start)) + timings.RecordMemory("repo_index", "git_files", len(inventory)) logStorageChainInputResult("repo_index", len(inventory), err) inventoryCh <- inventoryResult{inventory: inventory, err: err} }() go func() { start := time.Now() timings.StageStart("syfon_project_records") - recordSet, err := service.loadCachedProjectAuditRecordSet(ctx, authorizationHeader, organization, project, gitSubpath) + recordSet, err := service.loadCachedProjectAuditRecordSet(ctx, authorizationHeader, organization, project) timings.Record("syfon_project_records", time.Since(start)) recordCount := 0 if recordSet != nil { recordCount = countRecordStates(recordSet.allProjectRecords) } + timings.RecordMemory("syfon_project_records", "syfon_records", recordCount) logStorageChainInputResult("syfon_project_records", recordCount, err) recordCh <- recordResult{recordSet: recordSet, err: err} }() @@ -136,6 +138,7 @@ func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Conte timings.StageStart("syfon_bucket_inventory") bucketObjects, bucketObjectsByURL, err := service.loadCachedProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) timings.Record("syfon_bucket_inventory", time.Since(start)) + timings.RecordMemory("syfon_bucket_inventory", "bucket_objects", len(bucketObjects), "bucket_lookup", len(bucketObjectsByURL)) logStorageChainInputResult("syfon_bucket_items", len(bucketObjects), err) bucketCh <- bucketResult{bucketObjects: bucketObjects, bucketObjectsByURL: bucketObjectsByURL, err: err} }() @@ -337,22 +340,81 @@ func (service *StorageAnalyticsService) loadProjectAuditRecordSet(ctx context.Co return buildProjectAuditRecordSet(projectRecords), nil } -func (service *StorageAnalyticsService) loadCachedProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) (*storageAuditRecordSet, error) { - cacheKey := service.projectChainInputCacheKey(organization, project) + "::audit-records::" + normalizeRepoSubpath(pathPrefix) - service.chainInputMu.RLock() - cached, ok := service.chainInputCache[cacheKey] - service.chainInputMu.RUnlock() - if ok && time.Now().Before(cached.expiresAt) && cached.projectRecords != nil { - return buildProjectAuditRecordSet(cached.projectRecords), nil +func (service *StorageAnalyticsService) loadCachedProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string) (*storageAuditRecordSet, error) { + projectRecords, err := service.loadCachedProjectAuditRecords(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err } - projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project, pathPrefix) + return buildProjectAuditRecordSet(projectRecords), nil +} + +func (service *StorageAnalyticsService) loadCachedProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + summary, err := service.storage.GetProjectMetricsSummary(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("get syfon project metrics summary: %w", err) + } + if summary != nil { + validator := projectAuditRecordValidatorFromSummary(*summary) + service.chainInputMu.RLock() + cached, ok := service.projectAuditCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && cached.validator == validator && time.Since(cached.cachedAt) < chainProjectRecordCacheMaxAge { + return cached.records, nil + } + + workKey := projectAuditRecordWorkKey(cacheKey, validator) + service.chainInputMu.Lock() + if cached, ok := service.projectAuditCache[cacheKey]; ok && cached.validator == validator && time.Since(cached.cachedAt) < chainProjectRecordCacheMaxAge { + service.chainInputMu.Unlock() + return cached.records, nil + } + if inflight, ok := service.projectAuditWork[workKey]; ok { + service.chainInputMu.Unlock() + <-inflight.done + if inflight.err != nil { + return nil, inflight.err + } + return inflight.records, nil + } + inflight := &inflightProjectAuditRecordState{ + done: make(chan struct{}), + validator: validator, + } + service.projectAuditWork[workKey] = inflight + service.chainInputMu.Unlock() + defer func() { + service.chainInputMu.Lock() + delete(service.projectAuditWork, workKey) + close(inflight.done) + service.chainInputMu.Unlock() + }() + + projectRecords, err := service.storage.ListProjectAuditRecords(context.WithoutCancel(ctx), authorizationHeader, organization, project, "") + if err != nil { + inflight.err = fmt.Errorf("list syfon project audit records: %w", err) + return nil, inflight.err + } + copiedRecords := append([]gintegrationsyfon.ProjectRecord(nil), projectRecords...) + service.chainInputMu.Lock() + service.projectAuditCache[cacheKey] = cachedProjectAuditRecordState{ + records: copiedRecords, + validator: validator, + cachedAt: time.Now(), + } + service.chainInputMu.Unlock() + inflight.records = copiedRecords + return copiedRecords, nil + } + projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project, "") if err != nil { return nil, fmt.Errorf("list syfon project audit records: %w", err) } - service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { - state.projectRecords = append([]gintegrationsyfon.ProjectRecord(nil), projectRecords...) - }) - return buildProjectAuditRecordSet(projectRecords), nil + return projectRecords, nil +} + +func projectAuditRecordWorkKey(cacheKey string, validator projectAuditRecordValidator) string { + return fmt.Sprintf("%s::records::%d::%s::%s", cacheKey, validator.RecordCount, validator.RecordLatestUpdatedTime, validator.RecordRevision) } func buildProjectAuditRecordSet(projectRecords []gintegrationsyfon.ProjectRecord) *storageAuditRecordSet { @@ -367,11 +429,66 @@ func buildProjectAuditRecordSet(projectRecords []gintegrationsyfon.ProjectRecord ProjectRecord: record, }) } - allProjectRecords := cloneRecordStateMap(recordsByChecksum) return &storageAuditRecordSet{ recordsByChecksum: recordsByChecksum, - allProjectRecords: allProjectRecords, + allProjectRecords: recordsByChecksum, + } +} + +func projectAuditRecordValidatorFromSummary(summary gintegrationsyfon.ProjectMetricsSummary) projectAuditRecordValidator { + return projectAuditRecordValidator{ + RecordCount: summary.RecordCount, + RecordLatestUpdatedTime: strings.TrimSpace(summary.RecordLatestUpdatedTime), + RecordRevision: strings.TrimSpace(summary.RecordRevision), + } +} + +func filterProjectAuditRecordSet(recordSet *storageAuditRecordSet, pathPrefix string, scopes []domain.StorageBucketScope) *storageAuditRecordSet { + if recordSet == nil || normalizeRepoSubpath(pathPrefix) == "" { + return recordSet + } + filtered := make(map[string][]projectRecordState, len(recordSet.allProjectRecords)) + for checksum, group := range recordSet.allProjectRecords { + for _, record := range group { + if projectAuditRecordMatchesPathPrefix(record.ProjectRecord, pathPrefix, scopes) { + filtered[checksum] = append(filtered[checksum], record) + } + } + } + return &storageAuditRecordSet{ + recordsByChecksum: cloneRecordStateMap(filtered), + allProjectRecords: cloneRecordStateMap(filtered), + } +} + +func projectAuditRecordMatchesPathPrefix(record gintegrationsyfon.ProjectRecord, pathPrefix string, scopes []domain.StorageBucketScope) bool { + normalizedPrefix := normalizeRepoSubpath(pathPrefix) + if normalizedPrefix == "" { + return true + } + for _, accessURL := range projectAuditRecordPathURLs(record, scopes) { + _, key, ok := parseStorageURL(accessURL) + if !ok { + continue + } + key = normalizeRepoSubpath(key) + if key == normalizedPrefix || strings.HasPrefix(key, normalizedPrefix+"/") { + return true + } } + return false +} + +func projectAuditRecordPathURLs(record gintegrationsyfon.ProjectRecord, scopes []domain.StorageBucketScope) []string { + out := make([]string, 0, len(record.AccessURLs)+len(record.AccessMethods)*2) + out = append(out, record.AccessURLs...) + for _, method := range record.AccessMethods { + if trimmed := strings.TrimSpace(method.URL); trimmed != "" { + out = append(out, trimmed) + } + } + out = append(out, canonicalizeRecordAccessURLs(out, scopes)...) + return uniqueStrings(out) } func (service *StorageAnalyticsService) loadProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { @@ -383,6 +500,15 @@ func (service *StorageAnalyticsService) loadProjectBucketInventory(ctx context.C return objects, lookup, nil } +func (service *StorageAnalyticsService) loadProjectBucketValidationInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { + bucketObjects, err := service.storage.ListProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) + if err != nil { + return nil, nil, fmt.Errorf("list syfon project bucket inventory: %w", err) + } + objects, lookup := buildBucketObjectLookup(bucketObjects) + return objects, lookup, nil +} + func (service *StorageAnalyticsService) loadCachedProjectChainScopeMappings(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { cacheKey := service.projectChainInputCacheKey(organization, project) service.chainInputMu.RLock() @@ -421,6 +547,26 @@ func (service *StorageAnalyticsService) loadCachedProjectBucketInventory(ctx con return objects, lookup, nil } +func (service *StorageAnalyticsService) loadCachedProjectBucketValidationInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + "::bucket-validation-inventory::" + normalizeRepoSubpath(bucketPathPrefix) + service.chainInputMu.RLock() + cached, ok := service.chainInputCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && time.Now().Before(cached.expiresAt) && cached.bucketObjects != nil && cached.bucketObjectsByURL != nil { + objects, lookup := cloneBucketInventory(cached.bucketObjects, cached.bucketObjectsByURL) + return objects, lookup, nil + } + bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) + if err != nil { + return nil, nil, err + } + service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { + state.bucketObjects, state.bucketObjectsByURL = cloneBucketInventory(bucketObjects, bucketObjectsByURL) + }) + objects, lookup := cloneBucketInventory(bucketObjects, bucketObjectsByURL) + return objects, lookup, nil +} + func (service *StorageAnalyticsService) loadCachedProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) { cacheKey := service.projectChainInputCacheKey(organization, project) + "::bucket-summary::" + strings.TrimSpace(mode) service.chainInputMu.RLock() @@ -562,6 +708,113 @@ func (service *StorageAnalyticsService) attachProjectStorageListValidations(ctx }, nil } +func attachProjectStorageInventoryValidations(recordSet *storageAuditRecordSet, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) *storageAuditRecordSet { + if recordSet == nil { + return nil + } + attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + clone.AccessProbes = inventoryValidationProbesForRecord(record, bucketObjectsByURL) + states = append(states, clone) + } + out[checksum] = states + } + return out + } + return &storageAuditRecordSet{ + recordsByChecksum: attach(recordSet.recordsByChecksum), + allProjectRecords: attach(recordSet.allProjectRecords), + } +} + +func inventoryValidationProbesForRecord(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []gintegrationsyfon.BulkStorageProbeResult { + probes := make([]gintegrationsyfon.BulkStorageProbeResult, 0) + seen := make(map[string]struct{}) + for _, accessURL := range probeAccessURLsForRecord(record) { + objectURL := canonicalStorageURL("", "", accessURL) + if objectURL == "" { + continue + } + if _, ok := seen[objectURL]; ok { + continue + } + seen[objectURL] = struct{}{} + expectedName := expectedStorageObjectNameForListValidation(objectURL, record.Name) + if item, ok := bucketObjectsByURL[objectURL]; ok { + probes = append(probes, inventoryPresentProbe(record, objectURL, expectedName, item)) + continue + } + probes = append(probes, inventoryMissingProbe(record, objectURL, expectedName)) + } + return probes +} + +func inventoryPresentProbe(record projectRecordState, objectURL string, expectedName string, item gintegrationsyfon.ProjectBucketObject) gintegrationsyfon.BulkStorageProbeResult { + size := item.SizeBytes + sizeMatch := item.SizeBytes == record.Size + nameMatch := true + if expectedName != "" { + nameMatch = path.Base(strings.Trim(strings.TrimSpace(item.Key), "/")) == expectedName + } + mismatches := make([]string, 0, 2) + if !sizeMatch { + mismatches = append(mismatches, "size_mismatch") + } + if !nameMatch { + mismatches = append(mismatches, "name_mismatch") + } + validationStatus := "matched" + if len(mismatches) > 0 { + validationStatus = "mismatched" + } + return gintegrationsyfon.BulkStorageProbeResult{ + ID: storageListValidationRequestKey(objectURL, record.Size, expectedName), + Operation: StorageChainValidationModeList, + ObjectURL: objectURL, + Provider: strings.TrimSpace(item.Provider), + Bucket: strings.TrimSpace(item.Bucket), + Key: strings.Trim(strings.TrimSpace(item.Key), "/"), + Path: strings.TrimSpace(item.Path), + Exists: true, + Status: "present", + SizeBytes: &size, + MetaSHA256: strings.TrimSpace(item.MetaSHA256), + ETag: strings.TrimSpace(item.ETag), + LastModified: strings.TrimSpace(item.LastModified), + ValidationStatus: validationStatus, + SizeMatch: &sizeMatch, + NameMatch: &nameMatch, + ValidationMismatches: mismatches, + } +} + +func inventoryMissingProbe(record projectRecordState, objectURL string, expectedName string) gintegrationsyfon.BulkStorageProbeResult { + bucket, key, _ := parseStorageURL(objectURL) + sizeMatch := false + nameMatch := false + return gintegrationsyfon.BulkStorageProbeResult{ + ID: storageListValidationRequestKey(objectURL, record.Size, expectedName), + Operation: StorageChainValidationModeList, + ObjectURL: objectURL, + Provider: "s3", + Bucket: bucket, + Key: key, + Path: path.Base(key), + Exists: false, + Status: "not_found", + Error: fmt.Sprintf("object %q was not found in project bucket inventory", objectURL), + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + SizeMatch: &sizeMatch, + NameMatch: &nameMatch, + ValidationMismatches: []string{}, + } +} + func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet) (*storageAuditStorageView, error) { scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) if err != nil { @@ -608,7 +861,7 @@ func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context return view, nil } -func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { +func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { recordSet = applyScopeCanonicalization(recordSet, scopes) view := &storageAuditStorageView{ scopes: append([]domain.StorageBucketScope(nil), scopes...), @@ -626,6 +879,23 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex if validationMode == StorageChainValidationModeInventory { return view, nil } + if validationMode == StorageChainValidationModeList { + inventoryStart := time.Now() + validationObjects, validationObjectsByURL, err := service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + timings.Record("syfon_bucket_validation_inventory", time.Since(inventoryStart)) + timings.RecordMemory("syfon_bucket_validation_inventory", "bucket_objects", len(validationObjects), "bucket_lookup", len(validationObjectsByURL)) + if err != nil { + return nil, err + } + view.bucketObjects, view.bucketObjectsByURL = cloneBucketInventory(validationObjects, validationObjectsByURL) + validateStart := time.Now() + probedRecordSet := attachProjectStorageInventoryValidations(recordSet, view.bucketObjectsByURL) + timings.Record("inventory_list_validation", time.Since(validateStart)) + timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL)) + view.recordsByChecksum = probedRecordSet.recordsByChecksum + view.allProjectRecords = probedRecordSet.allProjectRecords + return view, nil + } probeStart := time.Now() var ( probedRecordSet *storageAuditRecordSet @@ -641,6 +911,9 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex stage = "bulk_list_validation" } timings.Record(stage, time.Since(probeStart)) + if probedRecordSet != nil { + timings.RecordMemory(stage, "syfon_records", countRecordStates(probedRecordSet.allProjectRecords)) + } if probeErr != nil { return nil, probeErr } @@ -666,6 +939,9 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex probeStart := time.Now() probedSubset, probeErr := service.attachProjectStorageProbes(ctx, authorizationHeader, probeCandidates) timings.Record("bulk_probe", time.Since(probeStart)) + if probedSubset != nil { + timings.RecordMemory("bulk_probe", "probe_records", countRecordStates(probedSubset.allProjectRecords)) + } if probeErr != nil { return nil, probeErr } diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 9670a92..09c649b 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -3,10 +3,12 @@ package git import ( "context" "fmt" + "net/http" "os" "path/filepath" "strconv" "strings" + "sync" "testing" "time" @@ -18,37 +20,45 @@ import ( ) type fakeStorageAnalyticsBackend struct { - projectRecords []gintegrationsyfon.ProjectRecord - bulkRecords map[string][]gintegrationsyfon.ProjectRecord - buckets map[string]domain.StorageBucket - bucketScopes map[string][]domain.StorageBucketScope - projectScopes []domain.StorageBucketScope - usageByObject map[string]gintegrationsyfon.FileUsage - bulkChecksums []string - probeResults map[string]gintegrationsyfon.BulkStorageProbeResult - listProbeResults map[string]gintegrationsyfon.BulkStorageProbeResult - bucketObjects []gintegrationsyfon.ProjectBucketObject - listProjectBucketObjectsErr error - listBucketsCalls int - listBucketScopesCalls int - listProjectAuditRecordsCalls int - listProjectAuditRecordsPathPrefix string - listProjectScopesCalls int - listProjectFileUsageCalls int - listProjectBucketObjectsCalls int - listProjectBucketObjectsPathPrefix string - listProjectBucketSummaryCalls int - listProjectBucketSummaryMode string - bulkGetProjectRecordsCalls int - bulkGetDelay time.Duration - probeCalls int - probeItems []gintegrationsyfon.BulkStorageProbeItem - listProbeCalls int - listProbeItems []gintegrationsyfon.BulkStorageProbeItem - deletedIDs []string - deletedStorageIDs []string - updatedAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod - deletedBucketObjects []string + projectRecords []gintegrationsyfon.ProjectRecord + projectMetricsSummary *gintegrationsyfon.ProjectMetricsSummary + bulkRecords map[string][]gintegrationsyfon.ProjectRecord + buckets map[string]domain.StorageBucket + bucketScopes map[string][]domain.StorageBucketScope + projectScopes []domain.StorageBucketScope + usageByObject map[string]gintegrationsyfon.FileUsage + bulkChecksums []string + probeResults map[string]gintegrationsyfon.BulkStorageProbeResult + listProbeResults map[string]gintegrationsyfon.BulkStorageProbeResult + bucketObjects []gintegrationsyfon.ProjectBucketObject + listProjectBucketObjectsErr error + listBucketsCalls int + listBucketScopesCalls int + listProjectAuditRecordsCalls int + listProjectAuditRecordsPathPrefix string + listProjectAuditRecordsDelay time.Duration + getProjectMetricsSummaryCalls int + listProjectScopesCalls int + listProjectFileUsageCalls int + listProjectFileUsageByObjectIDsCalls int + listProjectFileUsageObjectIDs []string + listProjectFileUsageByObjectIDsErr error + listProjectBucketObjectsCalls int + listProjectBucketObjectsPathPrefix string + listProjectBucketInventoryCalls int + listProjectBucketInventoryPathPrefix string + listProjectBucketSummaryCalls int + listProjectBucketSummaryMode string + bulkGetProjectRecordsCalls int + bulkGetDelay time.Duration + probeCalls int + probeItems []gintegrationsyfon.BulkStorageProbeItem + listProbeCalls int + listProbeItems []gintegrationsyfon.BulkStorageProbeItem + deletedIDs []string + deletedStorageIDs []string + updatedAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod + deletedBucketObjects []string } func (fake *fakeStorageAnalyticsBackend) ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) { @@ -72,9 +82,34 @@ func (fake *fakeStorageAnalyticsBackend) ListProjectRecords(ctx context.Context, func (fake *fakeStorageAnalyticsBackend) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectRecord, error) { fake.listProjectAuditRecordsCalls++ fake.listProjectAuditRecordsPathPrefix = pathPrefix + if fake.listProjectAuditRecordsDelay > 0 { + time.Sleep(fake.listProjectAuditRecordsDelay) + } return append([]gintegrationsyfon.ProjectRecord(nil), fake.projectRecords...), nil } +func (fake *fakeStorageAnalyticsBackend) GetProjectMetricsSummary(ctx context.Context, authorizationHeader string, organization string, project string) (*gintegrationsyfon.ProjectMetricsSummary, error) { + fake.getProjectMetricsSummaryCalls++ + if fake.projectMetricsSummary != nil { + summary := *fake.projectMetricsSummary + return &summary, nil + } + latest := "" + for _, record := range fake.projectRecords { + if record.UpdatedAt == nil { + continue + } + formatted := record.UpdatedAt.UTC().Format(time.RFC3339Nano) + if formatted > latest { + latest = formatted + } + } + return &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: len(fake.projectRecords), + RecordLatestUpdatedTime: latest, + }, nil +} + func (fake *fakeStorageAnalyticsBackend) ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { fake.listProjectScopesCalls++ if fake.projectScopes != nil { @@ -126,6 +161,21 @@ func (fake *fakeStorageAnalyticsBackend) ListProjectFileUsage(ctx context.Contex return out, nil } +func (fake *fakeStorageAnalyticsBackend) ListProjectFileUsageByObjectIDs(ctx context.Context, authorizationHeader string, organization string, project string, objectIDs []string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) { + fake.listProjectFileUsageByObjectIDsCalls++ + fake.listProjectFileUsageObjectIDs = append(fake.listProjectFileUsageObjectIDs, objectIDs...) + if fake.listProjectFileUsageByObjectIDsErr != nil { + return nil, fake.listProjectFileUsageByObjectIDsErr + } + out := make(map[string]gintegrationsyfon.FileUsage, len(objectIDs)) + for _, objectID := range objectIDs { + if usage, ok := fake.usageByObject[objectID]; ok { + out[objectID] = usage + } + } + return out, nil +} + func (fake *fakeStorageAnalyticsBackend) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) { fake.listProjectBucketObjectsCalls++ fake.listProjectBucketObjectsPathPrefix = pathPrefix @@ -135,6 +185,15 @@ func (fake *fakeStorageAnalyticsBackend) ListProjectBucketObjects(ctx context.Co return append([]gintegrationsyfon.ProjectBucketObject(nil), fake.bucketObjects...), nil } +func (fake *fakeStorageAnalyticsBackend) ListProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) { + fake.listProjectBucketInventoryCalls++ + fake.listProjectBucketInventoryPathPrefix = pathPrefix + if fake.listProjectBucketObjectsErr != nil { + return nil, fake.listProjectBucketObjectsErr + } + return append([]gintegrationsyfon.ProjectBucketObject(nil), fake.bucketObjects...), nil +} + func (fake *fakeStorageAnalyticsBackend) ListProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) { fake.listProjectBucketSummaryCalls++ fake.listProjectBucketSummaryMode = mode @@ -280,6 +339,12 @@ func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { if summary.TotalBytes != 650 || summary.DownloadCount != 15 { t.Fatalf("unexpected summary bytes/downloads: %+v", summary) } + if backend.listProjectFileUsageByObjectIDsCalls != 1 { + t.Fatalf("expected exact summary to use bulk file usage lookup, got %d", backend.listProjectFileUsageByObjectIDsCalls) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected exact summary to skip paginated project usage lookup, got %d", backend.listProjectFileUsageCalls) + } usageCallsBeforeChildren := backend.listProjectFileUsageCalls children, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 10, "bytes", "desc", "") @@ -1571,8 +1636,8 @@ func TestBuildStorageChainAuditUsesProjectAuditSourcesAndTargetsProbes(t *testin if backend.listProjectAuditRecordsCalls != 1 { t.Fatalf("expected one project audit record call, got %d", backend.listProjectAuditRecordsCalls) } - if backend.listProjectAuditRecordsPathPrefix != "data" { - t.Fatalf("expected audit path prefix to scope project records, got %q", backend.listProjectAuditRecordsPathPrefix) + if backend.listProjectAuditRecordsPathPrefix != "" { + t.Fatalf("expected audit to load project-scoped records for cache reuse, got %q", backend.listProjectAuditRecordsPathPrefix) } if backend.listProjectScopesCalls != 1 { t.Fatalf("expected one project scope call, got %d", backend.listProjectScopesCalls) @@ -1644,23 +1709,7 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { }, bucketObjects: []gintegrationsyfon.ProjectBucketObject{ {ObjectURL: "s3://bucket/present.txt", Bucket: "bucket", Key: "present.txt", Path: "present.txt", SizeBytes: 100}, - }, - listProbeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ - storageListValidationRequestKey("s3://bucket/mismatch.txt", 200, "expected.txt"): { - ID: storageListValidationRequestKey("s3://bucket/mismatch.txt", 200, "expected.txt"), - ObjectURL: "s3://bucket/mismatch.txt", - Provider: "s3", - Bucket: "bucket", - Key: "mismatch.txt", - Path: "mismatch.txt", - Exists: true, - Status: "present", - ValidationStatus: "mismatched", - SizeBytes: int64Ptr(200), - SizeMatch: ptrBool(true), - NameMatch: ptrBool(false), - ValidationMismatches: []string{"name"}, - }, + {ObjectURL: "s3://bucket/mismatch.txt", Bucket: "bucket", Key: "mismatch.txt", Path: "mismatch.txt", SizeBytes: 200}, }, } service := NewStorageAnalyticsService(backend) @@ -1671,7 +1720,7 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { "org", "proj", refName, - "data", + "", mirrorPath, repo, hash, @@ -1681,7 +1730,10 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { t.Fatalf("build validate chain audit: %v", err) } if backend.listProjectBucketObjectsCalls != 0 { - t.Fatalf("expected validate mode to skip recursive bucket inventory, got %d calls", backend.listProjectBucketObjectsCalls) + t.Fatalf("expected validate mode to skip browse bucket inventory endpoint, got %d calls", backend.listProjectBucketObjectsCalls) + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected validate mode to load project inventory once, got %d calls", backend.listProjectBucketInventoryCalls) } if backend.listProjectBucketSummaryCalls != 0 { t.Fatalf("expected validate mode to skip bucket summary preflight, got %d calls", backend.listProjectBucketSummaryCalls) @@ -1689,11 +1741,8 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { if backend.probeCalls != 0 { t.Fatalf("expected validate mode to skip HEAD probes, got %d calls", backend.probeCalls) } - if backend.listProbeCalls != 1 || len(backend.listProbeItems) != 3 { - t.Fatalf("expected three bulk LIST validation items, got calls=%d items=%+v", backend.listProbeCalls, backend.listProbeItems) - } - if backend.listProbeItems[0].ExpectedName == "" && backend.listProbeItems[1].ExpectedName == "" { - t.Fatalf("expected LIST validation items to include expected Syfon names, got %+v", backend.listProbeItems) + if backend.listProbeCalls != 0 || len(backend.listProbeItems) != 0 { + t.Fatalf("expected project inventory validation to skip bulk LIST calls, got calls=%d items=%+v", backend.listProbeCalls, backend.listProbeItems) } if chain.Summary.BucketPathExists != nil || chain.Summary.BucketSummaryMode != "" { t.Fatalf("expected validate mode not to claim bucket prefix summary, got %+v", chain.Summary) @@ -1701,21 +1750,21 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { if chain.Summary.ValidationMode != StorageChainValidationModeList { t.Fatalf("expected LIST validation mode, got %+v", chain.Summary) } - if chain.Summary.BucketObjectCount != 3 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { - t.Fatalf("expected validate mode to count validated objects without bucket-only inventory, got summary %+v", chain.Summary) + if chain.Summary.BucketObjectCount != 2 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { + t.Fatalf("expected validate mode to count project inventory without bucket-only findings, got summary %+v", chain.Summary) } if chain.Summary.CountsByKind["bucket_syfon_git_complete"] != 1 { t.Fatalf("expected validate mode to count clean Syfon/Git/bucket join, got summary %+v", chain.Summary) } - if chain.Summary.CountsByKind["bucket_syfon_no_git"] != 1 { - t.Fatalf("expected validate mode to report Syfon-backed no-Git record, got summary %+v", chain.Summary) + if chain.Summary.CountsByKind["syfon_missing_bucket_object"] != 1 { + t.Fatalf("expected validate mode to report missing Syfon bucket object, got summary %+v", chain.Summary) } if got := chain.Summary.CountsByKind["git_syfon_metadata_mismatch"]; got != 1 { t.Fatalf("expected one LIST-derived metadata mismatch, got summary %+v", chain.Summary) } assertNoChainFinding(t, chain.Findings, "bucket_only_object") assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/mismatch.txt") - assertHasChainFinding(t, chain.Findings, "bucket_syfon_no_git", "s3://bucket/syfon-only.txt") + assertHasChainFinding(t, chain.Findings, "syfon_missing_bucket_object", "s3://bucket/syfon-only.txt") } func TestBuildStorageChainAuditMetadataModeUsesMetadataValidation(t *testing.T) { @@ -1816,8 +1865,11 @@ func TestBuildStorageChainAuditCachesProjectChainInputs(t *testing.T) { if backend.listProjectAuditRecordsCalls != 1 { t.Fatalf("expected cached project records, got %d calls", backend.listProjectAuditRecordsCalls) } - if backend.listProjectAuditRecordsPathPrefix != "data" { - t.Fatalf("expected cached project records to use data subpath, got %q", backend.listProjectAuditRecordsPathPrefix) + if backend.listProjectAuditRecordsPathPrefix != "" { + t.Fatalf("expected cached project records to load the full project, got %q", backend.listProjectAuditRecordsPathPrefix) + } + if backend.getProjectMetricsSummaryCalls != 2 { + t.Fatalf("expected one cheap summary check per audit, got %d", backend.getProjectMetricsSummaryCalls) } if backend.listProjectScopesCalls != 1 { t.Fatalf("expected cached project scopes, got %d calls", backend.listProjectScopesCalls) @@ -1850,8 +1902,91 @@ func TestBuildStorageChainAuditCachesProjectRecordsPerPathPrefix(t *testing.T) { if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "DATA", mirrorPath, repo, hash, options); err != nil { t.Fatalf("build second chain audit: %v", err) } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected project-scoped record cache to serve both prefixes, got %d calls", backend.listProjectAuditRecordsCalls) + } +} + +func TestBuildStorageChainAuditCoalescesConcurrentProjectRecordRefresh(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + listProjectAuditRecordsDelay: 50 * time.Millisecond, + projectMetricsSummary: &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 1, + RecordLatestUpdatedTime: "2026-07-01T00:00:00Z", + }, + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + } + service := NewStorageAnalyticsService(backend) + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + var wg sync.WaitGroup + errs := make(chan error, 2) + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options) + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("build concurrent chain audit: %v", err) + } + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected concurrent audits to share one project record refresh, got %d calls", backend.listProjectAuditRecordsCalls) + } + if backend.getProjectMetricsSummaryCalls != 2 { + t.Fatalf("expected each audit to use the cheap metrics validator, got %d calls", backend.getProjectMetricsSummaryCalls) + } +} + +func TestBuildStorageChainAuditRefreshesProjectRecordsWhenMetricsValidatorChanges(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + backend := &fakeStorageAnalyticsBackend{ + projectMetricsSummary: &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 1, + RecordLatestUpdatedTime: "2026-07-01T00:00:00Z", + }, + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + } + service := NewStorageAnalyticsService(backend) + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build first chain audit: %v", err) + } + backend.projectMetricsSummary = &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 2, + RecordLatestUpdatedTime: "2026-07-02T00:00:00Z", + } + backend.projectRecords = append(backend.projectRecords, gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, AccessURLs: []string{"s3://bucket/data/b.txt"}, + }) + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build second chain audit: %v", err) + } if backend.listProjectAuditRecordsCalls != 2 { - t.Fatalf("expected path-scoped project record cache to distinguish prefixes, got %d calls", backend.listProjectAuditRecordsCalls) + t.Fatalf("expected changed metrics validator to refresh project records, got %d calls", backend.listProjectAuditRecordsCalls) } } @@ -1887,8 +2022,67 @@ func TestStorageChildrenSkipsSummaryJoinUsageWork(t *testing.T) { if backend.bulkGetProjectRecordsCalls != 2 { t.Fatalf("expected summary and page-scoped children checksum lookups, got %d", backend.bulkGetProjectRecordsCalls) } - if backend.listProjectFileUsageCalls != 1 { - t.Fatalf("expected only summary to perform usage lookup, got %d", backend.listProjectFileUsageCalls) + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected paginated usage lookup to remain unused, got %d", backend.listProjectFileUsageCalls) + } + if backend.listProjectFileUsageByObjectIDsCalls != 1 { + t.Fatalf("expected only summary to perform bulk usage lookup, got %d", backend.listProjectFileUsageByObjectIDsCalls) + } +} + +func TestListProjectFileUsageByObjectIDsChunksRequests(t *testing.T) { + objectIDs := make([]string, 0, projectFileUsageBulkChunkSize+1) + for i := 0; i < projectFileUsageBulkChunkSize+1; i++ { + objectIDs = append(objectIDs, fmt.Sprintf("obj-%04d", i)) + } + backend := &fakeStorageAnalyticsBackend{ + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-0000": {ObjectID: "obj-0000", DownloadCount: 1}, + "obj-5000": {ObjectID: "obj-5000", DownloadCount: 2}, + }, + } + service := NewStorageAnalyticsService(backend) + + usage, err := service.listProjectFileUsageByObjectIDs(context.Background(), "Bearer token", "org", "proj", objectIDs, cleanupInactiveDays) + if err != nil { + t.Fatalf("list bulk usage: %v", err) + } + if backend.listProjectFileUsageByObjectIDsCalls != 2 { + t.Fatalf("expected two bulk usage chunks, got %d", backend.listProjectFileUsageByObjectIDsCalls) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected no paginated fallback, got %d", backend.listProjectFileUsageCalls) + } + if len(backend.listProjectFileUsageObjectIDs) != projectFileUsageBulkChunkSize+1 { + t.Fatalf("expected all ids to be requested, got %d", len(backend.listProjectFileUsageObjectIDs)) + } + if usage["obj-0000"].DownloadCount != 1 || usage["obj-5000"].DownloadCount != 2 { + t.Fatalf("unexpected usage map: %+v", usage) + } +} + +func TestListProjectFileUsageByObjectIDsFallsBackWhenBulkEndpointMissing(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{ + listProjectFileUsageByObjectIDsErr: &gintegrationsyfon.HTTPError{ + Method: "POST", + Path: "/index/v1/metrics/files/bulk", + Status: http.StatusNotFound, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 9}, + }, + } + service := NewStorageAnalyticsService(backend) + + usage, err := service.listProjectFileUsageByObjectIDs(context.Background(), "Bearer token", "org", "proj", []string{"obj-a"}, cleanupInactiveDays) + if err != nil { + t.Fatalf("list fallback usage: %v", err) + } + if backend.listProjectFileUsageByObjectIDsCalls != 1 || backend.listProjectFileUsageCalls != 1 { + t.Fatalf("expected one bulk attempt and one paginated fallback, got bulk=%d paginated=%d", backend.listProjectFileUsageByObjectIDsCalls, backend.listProjectFileUsageCalls) + } + if usage["obj-a"].DownloadCount != 9 { + t.Fatalf("unexpected fallback usage: %+v", usage) } } @@ -2061,7 +2255,7 @@ func TestBuildStorageChainAuditFiltersFindingsByKind(t *testing.T) { } } -func TestBuildStorageChainAuditFilteredBrokenMappingBypassesDefaultTruncation(t *testing.T) { +func TestBuildStorageChainAuditDefaultTruncationKeepsRowsPerIssueKind(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/unrelated.txt": lfsPointer("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 1), }) @@ -2114,13 +2308,14 @@ func TestBuildStorageChainAuditFilteredBrokenMappingBypassesDefaultTruncation(t if err != nil { t.Fatalf("build truncated chain audit: %v", err) } - if !truncated.Summary.FindingsTruncated || truncated.Summary.ReturnedFindings != 500 { - t.Fatalf("expected default-like truncated response, got %+v", truncated.Summary) + if !truncated.Summary.FindingsTruncated || truncated.Summary.ReturnedFindings != 503 { + t.Fatalf("expected per-kind truncated response, got %+v", truncated.Summary) } if got := truncated.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 2 { t.Fatalf("expected summary to retain broken mapping count, got %+v", truncated.Summary) } - assertNoChainFinding(t, truncated.Findings, "syfon_broken_bucket_mapping") + assertHasChainFinding(t, truncated.Findings, "syfon_broken_bucket_mapping", "syfon/obj-broken-000") + assertHasChainFinding(t, truncated.Findings, "syfon_broken_bucket_mapping", "syfon/obj-broken-001") filtered, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ FindingKind: "syfon_broken_bucket_mapping", @@ -2340,6 +2535,72 @@ func TestBuildStorageChainAuditFlagsExactPathMismatchWhenHashExistsElsewhereInBu assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "CONFIG/cbds-BForePC.json") } +func TestClassifyStorageFindingSuppressesRawURLFailuresWhenScopedProbeMatches(t *testing.T) { + record := projectRecordState{ + ProjectRecord: gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Size: 100, + AccessURLs: []string{"s3://bforepc-prod/JHU/slide.ome.tiff"}, + }, + CanonicalAccessURLs: []string{"s3://bforepc/bforepc-prod/JHU/slide.ome.tiff"}, + AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ + { + ObjectURL: "s3://bforepc-prod/JHU/slide.ome.tiff", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + ValidationStatus: "unverifiable", + }, + { + ObjectURL: "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + Bucket: "bforepc", + Key: "bforepc-prod/JHU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + + if got := classifyStorageFinding(record, nil); got != storageFindingNone { + t.Fatalf("expected scoped matched probe to suppress raw URL credential miss, got %s", got) + } +} + +func TestClassifyStorageFindingSuppressesRawURLMismatchWhenScopedProbeMatches(t *testing.T) { + record := projectRecordState{ + ProjectRecord: gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Size: 100, + AccessURLs: []string{"s3://bforepc-prod/JHU/slide.ome.tiff"}, + }, + CanonicalAccessURLs: []string{"s3://bforepc/bforepc-prod/JHU/slide.ome.tiff"}, + AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ + { + ObjectURL: "s3://bforepc-prod/JHU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + { + ObjectURL: "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + Bucket: "bforepc", + Key: "bforepc-prod/JHU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + + if got := classifyStorageFinding(record, nil); got != storageFindingNone { + t.Fatalf("expected scoped matched probe to suppress raw URL mismatch, got %s", got) + } +} + func TestBuildStorageChainAuditNormalizesChecksumJoinKeys(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -2438,7 +2699,7 @@ func TestBuildStorageChainAuditFailsWhenProjectBucketListDenied(t *testing.T) { }}, }, usageByObject: map[string]gintegrationsyfon.FileUsage{}, - listProjectBucketObjectsErr: fmt.Errorf("list syfon project bucket objects: syfon POST /data/inspect/project-bucket failed with status 409: provider rejected bucket inventory request for s3://bucket/prefix; mapped bucket target may be missing or inaccessible"), + listProjectBucketObjectsErr: fmt.Errorf("list syfon project bucket inventory: syfon POST /data/inspect/project-bucket/inventory failed with status 409: provider rejected bucket inventory request for s3://bucket/prefix; mapped bucket target may be missing or inaccessible"), probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ storageProbeRequestKey("s3://bucket/prefix/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { ID: storageProbeRequestKey("s3://bucket/prefix/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), @@ -2453,15 +2714,15 @@ func TestBuildStorageChainAuditFailsWhenProjectBucketListDenied(t *testing.T) { } service := NewStorageAnalyticsService(backend) - _, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}) if err == nil { t.Fatal("expected bucket inventory failure to hard-error chain audit") } if !strings.Contains(err.Error(), "mapped bucket target may be missing or inaccessible") { t.Fatalf("expected bucket inventory error detail, got %v", err) } - if backend.listProjectBucketObjectsCalls != 1 { - t.Fatalf("expected one bucket inventory attempt, got %d", backend.listProjectBucketObjectsCalls) + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected one bucket validation inventory attempt, got %d", backend.listProjectBucketInventoryCalls) } if backend.probeCalls != 0 { t.Fatalf("expected no probe fallback after bucket inventory failure, got %d", backend.probeCalls) diff --git a/internal/git/storage_analytics_timing.go b/internal/git/storage_analytics_timing.go index 5d63f52..98f565b 100644 --- a/internal/git/storage_analytics_timing.go +++ b/internal/git/storage_analytics_timing.go @@ -1,6 +1,8 @@ package git import ( + "fmt" + "runtime" "strings" "sync" "time" @@ -65,6 +67,53 @@ func (timings *StorageChainAuditTimings) Record(stage string, duration time.Dura } } +func (timings *StorageChainAuditTimings) RecordMemory(stage string, fields ...any) { + if timings == nil || timings.Logf == nil { + return + } + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + prefix := strings.TrimSpace(timings.DebugPrefix) + normalizedStage := strings.TrimSpace(stage) + extra := formatStorageChainMemoryFields(fields...) + timings.Logf( + "storage_chain_audit_memory %s stage=%s alloc_mib=%d heap_alloc_mib=%d heap_inuse_mib=%d heap_sys_mib=%d stack_sys_mib=%d sys_mib=%d next_gc_mib=%d num_gc=%d%s", + prefix, + normalizedStage, + bytesToMiB(stats.Alloc), + bytesToMiB(stats.HeapAlloc), + bytesToMiB(stats.HeapInuse), + bytesToMiB(stats.HeapSys), + bytesToMiB(stats.StackSys), + bytesToMiB(stats.Sys), + bytesToMiB(stats.NextGC), + stats.NumGC, + extra, + ) +} + +func bytesToMiB(bytes uint64) uint64 { + return bytes / 1024 / 1024 +} + +func formatStorageChainMemoryFields(fields ...any) string { + if len(fields) == 0 { + return "" + } + parts := make([]string, 0, len(fields)/2) + for i := 0; i+1 < len(fields); i += 2 { + key := strings.TrimSpace(fmt.Sprint(fields[i])) + if key == "" { + continue + } + parts = append(parts, fmt.Sprintf("%s=%v", key, fields[i+1])) + } + if len(parts) == 0 { + return "" + } + return " " + strings.Join(parts, " ") +} + func (timings *StorageChainAuditTimings) Snapshot() []StorageChainAuditStageTiming { if timings == nil { return nil diff --git a/internal/git/tree_enrichment.go b/internal/git/tree_enrichment.go new file mode 100644 index 0000000..edbf49d --- /dev/null +++ b/internal/git/tree_enrichment.go @@ -0,0 +1,173 @@ +package git + +import ( + "io" + "sort" + "strings" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +func enrichTreeEntries(tree *object.Tree, entries []GitTreeEntry, options GitTreeResponseOptions) { + if !options.IncludeSize && !options.IncludeLFSPointer { + return + } + for index := range entries { + entry := &entries[index] + if entry.Type != "blob" { + continue + } + + file, err := tree.File(entry.Name) + if err != nil { + continue + } + if options.IncludeSize { + entry.Size = file.Size + } + if options.IncludeLFSPointer { + if reader, err := file.Reader(); err == nil { + contentBytes, readErr := io.ReadAll(io.LimitReader(reader, 2048)) + _ = reader.Close() + if readErr == nil { + entry.LFSPointer = ParseGitLFSPointer(contentBytes) + } + } + } + } +} + +func lookupGitPathsLastModified(repo *gogit.Repository, from plumbing.Hash, entries []GitTreeEntry) (map[string]time.Time, error) { + targets := buildGitTreeLastModifiedTargets(entries) + if targets.remaining == 0 { + return map[string]time.Time{}, nil + } + + iter, err := repo.Log(&gogit.LogOptions{ + From: from, + Order: gogit.LogOrderCommitterTime, + }) + if err != nil { + return nil, err + } + defer iter.Close() + + for targets.remaining > 0 { + commit, err := iter.Next() + if err != nil { + if err == io.EOF { + return targets.found, nil + } + return nil, err + } + modifiedAt := commit.Committer.When.UTC() + if commit.NumParents() == 0 { + targets.markAll(modifiedAt) + return targets.found, nil + } + changedPaths, err := commitChangedPaths(commit) + if err != nil { + return nil, err + } + for _, changedPath := range changedPaths { + targets.mark(changedPath, modifiedAt) + if targets.remaining == 0 { + return targets.found, nil + } + } + } + return targets.found, nil +} + +type gitTreeLastModifiedTargets struct { + exact map[string]string + dirs []string + found map[string]time.Time + remaining int +} + +func buildGitTreeLastModifiedTargets(entries []GitTreeEntry) *gitTreeLastModifiedTargets { + targets := &gitTreeLastModifiedTargets{ + exact: make(map[string]string, len(entries)), + found: make(map[string]time.Time, len(entries)), + } + for _, entry := range entries { + normalizedPath := strings.Trim(strings.TrimSpace(entry.Path), "/") + if normalizedPath == "" { + continue + } + targets.exact[normalizedPath] = normalizedPath + if entry.Type == "tree" { + targets.dirs = append(targets.dirs, normalizedPath) + } + targets.remaining++ + } + sort.Slice(targets.dirs, func(i, j int) bool { + return len(targets.dirs[i]) > len(targets.dirs[j]) + }) + return targets +} + +func (targets *gitTreeLastModifiedTargets) mark(changedPath string, modifiedAt time.Time) { + normalizedPath := strings.Trim(strings.TrimSpace(changedPath), "/") + if normalizedPath == "" { + return + } + if targetPath, ok := targets.exact[normalizedPath]; ok { + targets.setFound(targetPath, modifiedAt) + } + for _, dirPath := range targets.dirs { + if normalizedPath == dirPath || strings.HasPrefix(normalizedPath, dirPath+"/") { + targets.setFound(dirPath, modifiedAt) + } + } +} + +func (targets *gitTreeLastModifiedTargets) setFound(path string, modifiedAt time.Time) { + if _, exists := targets.found[path]; exists { + return + } + targets.found[path] = modifiedAt + targets.remaining-- +} + +func (targets *gitTreeLastModifiedTargets) markAll(modifiedAt time.Time) { + for targetPath := range targets.exact { + targets.setFound(targetPath, modifiedAt) + } +} + +func commitChangedPaths(commit *object.Commit) ([]string, error) { + if commit.NumParents() == 0 { + return nil, nil + } + parent, err := commit.Parent(0) + if err != nil { + return nil, err + } + parentTree, err := parent.Tree() + if err != nil { + return nil, err + } + commitTree, err := commit.Tree() + if err != nil { + return nil, err + } + changes, err := parentTree.Diff(commitTree) + if err != nil { + return nil, err + } + paths := make([]string, 0, len(changes)) + for _, change := range changes { + if change.From.Name != "" { + paths = append(paths, change.From.Name) + } + if change.To.Name != "" && change.To.Name != change.From.Name { + paths = append(paths, change.To.Name) + } + } + return paths, nil +} diff --git a/internal/git/tree_response_test.go b/internal/git/tree_response_test.go new file mode 100644 index 0000000..1d5db65 --- /dev/null +++ b/internal/git/tree_response_test.go @@ -0,0 +1,326 @@ +package git + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" +) + +func TestBuildGitResponsesDetectLFSPointers(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + pointerContent := strings.Join([]string{ + "version https://git-lfs.github.com/spec/v1", + "oid sha256:0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83", + "size 3780184021", + "", + }, "\n") + if err := os.MkdirAll(filepath.Join(sourcePath, "data"), 0o755); err != nil { + t.Fatalf("create data dir: %v", err) + } + if err := os.WriteFile(filepath.Join(sourcePath, "data", "tcga.tumor.ensembl.tsv"), []byte(pointerContent), 0o644); err != nil { + t.Fatalf("write lfs pointer file: %v", err) + } + if _, err := worktree.Add("data/tcga.tumor.ensembl.tsv"); err != nil { + t.Fatalf("add lfs pointer file: %v", err) + } + if _, err := worktree.Commit("add lfs pointer", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit lfs pointer file: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "data", mirrorRepo, hash, GitTreeResponseOptions{ + IncludeLFSPointer: true, + IncludeSize: true, + }) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if len(treeResponse.Entries) != 1 { + t.Fatalf("expected one tree entry, got %+v", treeResponse.Entries) + } + treePointer := treeResponse.Entries[0].LFSPointer + if treePointer == nil { + t.Fatalf("expected tree entry to be marked as lfs pointer, got %+v", treeResponse.Entries[0]) + } + if treePointer.OID != "0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83" { + t.Fatalf("unexpected lfs oid: %q", treePointer.OID) + } + if treePointer.Size != 3780184021 { + t.Fatalf("unexpected lfs size: %d", treePointer.Size) + } + + fileResponse, err := BuildGitFileResponse("org-a/proj-a", refName, "data/tcga.tumor.ensembl.tsv", mirrorRepo, hash) + if err != nil { + t.Fatalf("build file response: %v", err) + } + if fileResponse.LFSPointer == nil { + t.Fatalf("expected file response to include lfs pointer metadata") + } + if fileResponse.LFSPointer.OID != treePointer.OID { + t.Fatalf("expected matching lfs oid, got %q and %q", fileResponse.LFSPointer.OID, treePointer.OID) + } +} + +func TestBuildGitTreeResponseDefaultsToCheapFields(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + if err := os.WriteFile(filepath.Join(sourcePath, "README.md"), []byte("hello gecko"), 0o644); err != nil { + t.Fatalf("write readme: %v", err) + } + if _, err := worktree.Add("README.md"); err != nil { + t.Fatalf("add readme: %v", err) + } + if _, err := worktree.Commit("initial commit", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit readme: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{}) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if treeResponse.EntryCount != 1 { + t.Fatalf("expected entry count 1, got %d", treeResponse.EntryCount) + } + if treeResponse.Truncated { + t.Fatal("expected non-truncated response by default") + } + if len(treeResponse.Entries) != 1 { + t.Fatalf("expected one tree entry, got %+v", treeResponse.Entries) + } + if treeResponse.Entries[0].Size != 0 { + t.Fatalf("expected default tree response to omit size, got %d", treeResponse.Entries[0].Size) + } + if treeResponse.Entries[0].LFSPointer != nil { + t.Fatalf("expected default tree response to omit lfs pointer, got %+v", treeResponse.Entries[0].LFSPointer) + } + if treeResponse.Entries[0].LastModifiedAt != nil { + t.Fatalf("expected default tree response to omit last modified, got %+v", treeResponse.Entries[0].LastModifiedAt) + } +} + +func TestBuildGitTreeResponseHonorsLimitBeforeEnrichment(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + pointerContent := strings.Join([]string{ + "version https://git-lfs.github.com/spec/v1", + "oid sha256:0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83", + "size 3780184021", + "", + }, "\n") + for name, content := range map[string]string{ + "a.txt": pointerContent, + "b.txt": pointerContent, + "c.txt": "regular file", + } { + if err := os.WriteFile(filepath.Join(sourcePath, name), []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + if _, err := worktree.Add(name); err != nil { + t.Fatalf("add %s: %v", name, err) + } + } + committedAt := time.Date(2026, 7, 6, 12, 0, 0, 0, time.UTC) + if _, err := worktree.Commit("add files", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: committedAt}}); err != nil { + t.Fatalf("commit files: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{ + IncludeLFSPointer: true, + IncludeLastModified: true, + IncludeSize: true, + Limit: 2, + }) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if !treeResponse.Truncated { + t.Fatal("expected truncated response when limit is smaller than entry count") + } + if treeResponse.EntryCount != 3 { + t.Fatalf("expected total entry count 3, got %d", treeResponse.EntryCount) + } + if len(treeResponse.Entries) != 2 { + t.Fatalf("expected two returned entries, got %+v", treeResponse.Entries) + } + if got := []string{treeResponse.Entries[0].Name, treeResponse.Entries[1].Name}; strings.Join(got, ",") != "a.txt,b.txt" { + t.Fatalf("expected limited page to contain only first two entries, got %v", got) + } + for _, entry := range treeResponse.Entries { + if entry.Size == 0 { + t.Fatalf("expected size to be included for limited entry %+v", entry) + } + if entry.LFSPointer == nil { + t.Fatalf("expected lfs pointer to be included for limited entry %+v", entry) + } + if entry.LastModifiedAt == nil || !entry.LastModifiedAt.Equal(committedAt) { + t.Fatalf("expected last modified %s for limited entry, got %+v", committedAt, entry.LastModifiedAt) + } + } +} + +func TestBuildGitTreeResponseBatchesLastModifiedForFilesAndDirectories(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + + initialCommitAt := time.Date(2026, 7, 5, 10, 0, 0, 0, time.UTC) + if err := os.MkdirAll(filepath.Join(sourcePath, "data"), 0o755); err != nil { + t.Fatalf("create data dir: %v", err) + } + for path, content := range map[string]string{ + "README.md": "readme", + "data/a.txt": "a", + } { + fullPath := filepath.Join(sourcePath, filepath.FromSlash(path)) + if err := os.WriteFile(fullPath, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + if _, err := worktree.Add(path); err != nil { + t.Fatalf("add %s: %v", path, err) + } + } + if _, err := worktree.Commit("initial files", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: initialCommitAt}}); err != nil { + t.Fatalf("commit initial files: %v", err) + } + + directoryCommitAt := time.Date(2026, 7, 6, 11, 0, 0, 0, time.UTC) + nestedPath := filepath.Join(sourcePath, "data", "nested", "b.txt") + if err := os.MkdirAll(filepath.Dir(nestedPath), 0o755); err != nil { + t.Fatalf("create nested dir: %v", err) + } + if err := os.WriteFile(nestedPath, []byte("b"), 0o644); err != nil { + t.Fatalf("write nested file: %v", err) + } + if _, err := worktree.Add("data/nested/b.txt"); err != nil { + t.Fatalf("add nested file: %v", err) + } + if _, err := worktree.Commit("add nested file", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: directoryCommitAt}}); err != nil { + t.Fatalf("commit nested file: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{ + IncludeLastModified: true, + }) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if len(treeResponse.Entries) != 2 { + t.Fatalf("expected data dir and readme entries, got %+v", treeResponse.Entries) + } + + entriesByPath := make(map[string]GitTreeEntry, len(treeResponse.Entries)) + for _, entry := range treeResponse.Entries { + entriesByPath[entry.Path] = entry + } + assertTreeLastModified(t, entriesByPath["data"], directoryCommitAt) + assertTreeLastModified(t, entriesByPath["README.md"], initialCommitAt) + + lastModifiedByPath, err := lookupGitPathsLastModified(mirrorRepo, hash, treeResponse.Entries) + if err != nil { + t.Fatalf("lookup batched last modified: %v", err) + } + if got := lastModifiedByPath["data"]; !got.Equal(directoryCommitAt) { + t.Fatalf("expected batched data timestamp %s, got %s", directoryCommitAt, got) + } + if got := lastModifiedByPath["README.md"]; !got.Equal(initialCommitAt) { + t.Fatalf("expected batched readme timestamp %s, got %s", initialCommitAt, got) + } +} + +func assertTreeLastModified(t *testing.T, entry GitTreeEntry, expected time.Time) { + t.Helper() + if entry.LastModifiedAt == nil { + t.Fatalf("expected %s to have last modified timestamp", entry.Path) + } + if !entry.LastModifiedAt.Equal(expected) { + t.Fatalf("expected %s last modified %s, got %s", entry.Path, expected, *entry.LastModifiedAt) + } +} diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index a186225..4da1edb 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -3,6 +3,7 @@ package syfon import ( "context" "encoding/json" + "errors" "fmt" "io" "log" @@ -27,6 +28,30 @@ const refreshAuthzHeader = "X-Syfon-Refresh-Authz" const bulkStorageProbeBatchSize = 200 const bulkStorageProbeConcurrency = 4 +type HTTPError struct { + Method string + Path string + Status int + Body string +} + +func (err HTTPError) Error() string { + return fmt.Sprintf("syfon %s %s failed with status %d: %s", err.Method, err.Path, err.Status, strings.TrimSpace(err.Body)) +} + +func IsHTTPStatus(err error, statuses ...int) bool { + var httpErr *HTTPError + if !errors.As(err, &httpErr) { + return false + } + for _, status := range statuses { + if httpErr.Status == status { + return true + } + } + return false +} + type Manager struct { baseURL string client *http.Client @@ -52,6 +77,12 @@ type ProjectAccessMethod struct { Headers []string } +type ProjectMetricsSummary struct { + RecordCount int + RecordLatestUpdatedTime string + RecordRevision string +} + type BulkStorageProbeItem struct { ID string ObjectURL string @@ -240,6 +271,23 @@ func (manager *Manager) ListBucketScopes(ctx context.Context, authorizationHeade return out, nil } +func uniqueNonEmptyStrings(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + func (manager *Manager) CleanupProject(ctx context.Context, authorizationHeader string, organization string, project string) error { dataBaseURL, err := manager.dataAPIBaseURL() if err != nil { @@ -370,6 +418,28 @@ func (manager *Manager) ListProjectAuditRecords(ctx context.Context, authorizati return out, nil } +func (manager *Manager) GetProjectMetricsSummary(ctx context.Context, authorizationHeader string, organization string, project string) (*ProjectMetricsSummary, error) { + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) + var response struct { + RecordCount *int `json:"record_count"` + RecordLatestUpdatedTime string `json:"record_latest_updated_time"` + RecordRevision string `json:"record_revision"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, "/index/v1/metrics/summary", params, nil, &response); err != nil { + return nil, fmt.Errorf("get syfon metrics summary: %w", err) + } + if response.RecordCount == nil { + return nil, nil + } + return &ProjectMetricsSummary{ + RecordCount: *response.RecordCount, + RecordLatestUpdatedTime: strings.TrimSpace(response.RecordLatestUpdatedTime), + RecordRevision: strings.TrimSpace(response.RecordRevision), + }, nil +} + func (manager *Manager) ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { requestBody := struct { Organization string `json:"organization,omitempty"` @@ -499,6 +569,50 @@ func (manager *Manager) ListProjectFileUsage(ctx context.Context, authorizationH return out, nil } +func (manager *Manager) ListProjectFileUsageByObjectIDs(ctx context.Context, authorizationHeader string, organization string, project string, objectIDs []string, inactiveDays int) (map[string]FileUsage, error) { + out := make(map[string]FileUsage) + ids := uniqueNonEmptyStrings(objectIDs) + if len(ids) == 0 { + return out, nil + } + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) + request := struct { + ObjectIDs []string `json:"object_ids"` + InactiveDays *int `json:"inactive_days,omitempty"` + }{ + ObjectIDs: ids, + } + if inactiveDays > 0 { + request.InactiveDays = &inactiveDays + } + var response metricsapi.MetricsListResponse + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/index/v1/metrics/files/bulk", params, request, &response); err != nil { + return nil, fmt.Errorf("bulk list syfon metrics files: %w", err) + } + if response.Data == nil { + return out, nil + } + for _, item := range *response.Data { + objectID := strings.TrimSpace(stringValue(item.ObjectId)) + if objectID == "" { + continue + } + out[objectID] = FileUsage{ + ObjectID: objectID, + Name: stringValue(item.Name), + Size: int64Value(item.Size), + DownloadCount: int64Value(item.DownloadCount), + UploadCount: int64Value(item.UploadCount), + LastAccessTime: item.LastAccessTime, + LastDownloadTime: item.LastDownloadTime, + LastUploadTime: item.LastUploadTime, + } + } + return out, nil +} + func (manager *Manager) BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error { normalized := dedupeStrings(objectIDs) if len(normalized) == 0 { @@ -862,6 +976,14 @@ func (manager *Manager) ListProjectBucketSummary(ctx context.Context, authorizat } func (manager *Manager) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]ProjectBucketObject, error) { + return manager.listProjectBucketObjects(ctx, authorizationHeader, "/data/inspect/project-bucket", organization, project, pathPrefix) +} + +func (manager *Manager) ListProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]ProjectBucketObject, error) { + return manager.listProjectBucketObjects(ctx, authorizationHeader, "/data/inspect/project-bucket/inventory", organization, project, pathPrefix) +} + +func (manager *Manager) listProjectBucketObjects(ctx context.Context, authorizationHeader string, requestPath string, organization string, project string, pathPrefix string) ([]ProjectBucketObject, error) { requestBody := struct { Organization string `json:"organization,omitempty"` Project string `json:"project,omitempty"` @@ -886,7 +1008,7 @@ func (manager *Manager) ListProjectBucketObjects(ctx context.Context, authorizat LastModified string `json:"last_modified"` } `json:"items"` } - if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-bucket", nil, requestBody, &response); err != nil { + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, requestPath, nil, requestBody, &response); err != nil { return nil, fmt.Errorf("list syfon project bucket objects: %w", err) } out := make([]ProjectBucketObject, 0, len(response.Items)) @@ -1055,7 +1177,12 @@ func (manager *Manager) requestJSON(ctx context.Context, authorizationHeader str defer resp.Body.Close() if resp.StatusCode >= http.StatusBadRequest { bodyBytes, _ := io.ReadAll(resp.Body) - return fmt.Errorf("syfon %s %s failed with status %d: %s", method, requestPath, resp.StatusCode, strings.TrimSpace(string(bodyBytes))) + return &HTTPError{ + Method: method, + Path: requestPath, + Status: resp.StatusCode, + Body: strings.TrimSpace(string(bodyBytes)), + } } if out == nil { return nil diff --git a/internal/integrations/syfon/adapter_test.go b/internal/integrations/syfon/adapter_test.go index a7d63b6..a91b011 100644 --- a/internal/integrations/syfon/adapter_test.go +++ b/internal/integrations/syfon/adapter_test.go @@ -12,6 +12,123 @@ import ( "testing" ) +func TestGetProjectMetricsSummaryReadsRecordValidator(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodGet || r.URL.Path != "/index/v1/metrics/summary" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + if got := r.URL.Query().Get("organization"); got != "org" { + t.Fatalf("expected organization query, got %q", got) + } + if got := r.URL.Query().Get("project"); got != "proj" { + t.Fatalf("expected project query, got %q", got) + } + body := []byte(`{"total_files":99,"record_count":42,"record_latest_updated_time":"2026-07-02T00:00:00Z","record_revision":"rev-1"}`) + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + summary, err := manager.GetProjectMetricsSummary(context.Background(), "Bearer token", "org", "proj") + if err != nil { + t.Fatalf("GetProjectMetricsSummary returned error: %v", err) + } + if summary == nil { + t.Fatal("expected metrics summary") + } + if summary.RecordCount != 42 || summary.RecordLatestUpdatedTime != "2026-07-02T00:00:00Z" || summary.RecordRevision != "rev-1" { + t.Fatalf("unexpected metrics summary: %+v", summary) + } +} + +func TestListProjectFileUsageByObjectIDsUsesBulkEndpoint(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/index/v1/metrics/files/bulk" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + if got := r.URL.Query().Get("organization"); got != "org" { + t.Fatalf("expected organization query, got %q", got) + } + if got := r.URL.Query().Get("project"); got != "proj" { + t.Fatalf("expected project query, got %q", got) + } + var req struct { + ObjectIDs []string `json:"object_ids"` + InactiveDays *int `json:"inactive_days"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if !reflect.DeepEqual(req.ObjectIDs, []string{"obj-a", "obj-b"}) { + t.Fatalf("unexpected object ids: %+v", req.ObjectIDs) + } + if req.InactiveDays == nil || *req.InactiveDays != 30 { + t.Fatalf("unexpected inactive days: %+v", req.InactiveDays) + } + body := []byte(`{"data":[{"object_id":"obj-a","name":"a.txt","size":100,"download_count":3,"upload_count":1}]}`) + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + usage, err := manager.ListProjectFileUsageByObjectIDs(context.Background(), "Bearer token", "org", "proj", []string{"obj-a", "obj-b", "obj-a", ""}, 30) + if err != nil { + t.Fatalf("ListProjectFileUsageByObjectIDs returned error: %v", err) + } + if len(usage) != 1 || usage["obj-a"].DownloadCount != 3 || usage["obj-a"].UploadCount != 1 { + t.Fatalf("unexpected usage response: %+v", usage) + } +} + +func TestListProjectBucketInventoryUsesInventoryEndpoint(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/project-bucket/inventory" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Organization string `json:"organization"` + Project string `json:"project"` + Mode string `json:"mode"` + PathPrefix string `json:"path_prefix"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if req.Organization != "org" || req.Project != "proj" || req.Mode != "items" || req.PathPrefix != "CONFIG" { + t.Fatalf("unexpected request body: %+v", req) + } + body, err := json.Marshal(map[string]any{"items": []map[string]any{{ + "object_url": "s3://bucket/root/CONFIG/a.bin", + "provider": "s3", + "bucket": "bucket", + "key": "root/CONFIG/a.bin", + "path": "CONFIG/a.bin", + "size_bytes": 123, + "etag": "etag-1", + "last_modified": "2026-07-07T18:00:00Z", + }}}) + if err != nil { + t.Fatalf("marshal response: %v", err) + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: make(http.Header)}, nil + })} + manager := NewManager("http://syfon", client) + + items, err := manager.ListProjectBucketInventory(context.Background(), "Bearer token", "org", "proj", "CONFIG") + if err != nil { + t.Fatalf("ListProjectBucketInventory returned error: %v", err) + } + if len(items) != 1 || items[0].ObjectURL != "s3://bucket/root/CONFIG/a.bin" || items[0].SizeBytes != 123 { + t.Fatalf("unexpected inventory items: %+v", items) + } +} + func TestBulkGetProjectRecordsByChecksumReadsResultsMap(t *testing.T) { t.Helper() diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index 231a0e7..88773d1 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -16,7 +16,6 @@ import ( "github.com/gofiber/fiber/v3" ) -const defaultStorageChainFindingLimit = 500 const defaultStorageChildrenLimit = 100 const maxStorageChildrenLimit = 1000 @@ -261,11 +260,8 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err bucketMode = gitcore.StorageChainBucketModeItems } findingLimit := requestBody.FindingLimit - if findingLimit == 0 { - findingLimit = defaultStorageChainFindingLimit - } if findingLimit < -1 { - response := httputil.NewError("invalid_request", "finding_limit must be -1 for all findings or a positive integer", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response := httputil.NewError("invalid_request", "finding_limit must be -1, 0, or a positive integer", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) response.WriteLog(handler.logger) return response.Write(ctx) } @@ -308,6 +304,14 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err timings.StageStart("json_response") writeErr := httputil.JSON(response, http.StatusOK).Write(ctx) timings.Record("json_response", time.Since(writeStart)) + timings.RecordMemory( + "json_response", + "total_findings", response.Summary.TotalFindings, + "returned_findings", response.Summary.ReturnedFindings, + "bucket_objects", response.Summary.BucketObjectCount, + "syfon_records", response.Summary.SyfonRecordCount, + "git_files", response.Summary.GitTrackedFileCount, + ) handler.logStorageChainAuditTimings(projectCtx, gitSubpath, probeMode, bucketMode, response, timings) return writeErr } From aac2c294f7ce920afe7c2c50e8990b25d12bb726 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 7 Jul 2026 15:09:29 -0700 Subject: [PATCH 22/36] fix false positives issues in gecko related to innaccurate bucket canonicalized paths --- internal/git/setup.go | 8 +- internal/git/storage_analytics.go | 75 ++++++++++++++--- internal/git/storage_analytics_pipeline.go | 2 +- internal/git/storage_analytics_test.go | 97 +++++++++++++++++++++- internal/git/types.go | 3 +- internal/httpclient/client.go | 27 ++++++ internal/integrations/fence/client.go | 4 +- internal/integrations/github/client.go | 4 +- internal/integrations/syfon/adapter.go | 3 +- internal/server/http/register.go | 3 +- internal/server/http/shared/handler.go | 3 +- internal/server/middleware/middleware.go | 4 +- 12 files changed, 205 insertions(+), 28 deletions(-) create mode 100644 internal/httpclient/client.go diff --git a/internal/git/setup.go b/internal/git/setup.go index 3df2140..cd65941 100644 --- a/internal/git/setup.go +++ b/internal/git/setup.go @@ -9,9 +9,11 @@ import ( "net/http" "net/url" "strings" + "time" "github.com/calypr/gecko/config" geckodb "github.com/calypr/gecko/internal/db" + "github.com/calypr/gecko/internal/httpclient" "github.com/calypr/gecko/internal/integrations/syfon" servermw "github.com/calypr/gecko/internal/server/middleware" "github.com/golang-jwt/jwt/v5" @@ -25,6 +27,8 @@ type SetupService struct { accessChecks *servermw.FenceUserAccessHandler } +var arboristHTTPClient = httpclient.NewServiceClient(30 * time.Second) + func NewSetupService(db *sqlx.DB, gitService *GitService, storage *syfon.Manager, accessChecks *servermw.FenceUserAccessHandler) *SetupService { return &SetupService{ db: db, @@ -228,7 +232,7 @@ func createAuthzOwnedDescendant(ctx context.Context, authorizationHeader string, } req.Header.Set("Authorization", authorizationHeader) req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := arboristHTTPClient.Do(req) if err != nil { return fmt.Errorf("request arborist descendant create: %w", err) } @@ -250,7 +254,7 @@ func DeleteAuthzResource(ctx context.Context, authorizationHeader, resourcePath return fmt.Errorf("build arborist resource delete request: %w", err) } req.Header.Set("Authorization", authorizationHeader) - resp, err := http.DefaultClient.Do(req) + resp, err := arboristHTTPClient.Do(req) if err != nil { return fmt.Errorf("request arborist resource delete: %w", err) } diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index b2dda29..d4cc7fe 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -1237,7 +1237,7 @@ func applyScopedStorageMappings(recordsByChecksum map[string][]projectRecordStat states := make([]projectRecordState, 0, len(group)) for _, record := range group { clone := record - clone.CanonicalAccessURLs = canonicalizeRecordAccessURLs(record.AccessURLs, scopes) + clone.CanonicalAccessURLs = canonicalizeRecordAccessURLs(record.AccessURLs, scopes, record.Organization, record.Project) states = append(states, clone) } out[checksum] = states @@ -2451,7 +2451,10 @@ func accessURLsForStorage(record projectRecordState) []string { } func probeAccessURLsForRecord(record projectRecordState) []string { - return uniqueStrings(append(rawAccessURLsForRecord(record), accessURLsForStorage(record)...)) + if len(record.CanonicalAccessURLs) > 0 { + return record.CanonicalAccessURLs + } + return rawAccessURLsForRecord(record) } func rawAccessURLsForRecord(record projectRecordState) []string { @@ -2577,10 +2580,10 @@ func classifyRawAccessURLFindings(record projectRecordState) storageFindingKind return storageFindingNone } -func canonicalizeRecordAccessURLs(accessURLs []string, scopes []domain.StorageBucketScope) []string { +func canonicalizeRecordAccessURLs(accessURLs []string, scopes []domain.StorageBucketScope, organization string, project string) []string { out := make([]string, 0, len(accessURLs)) for _, accessURL := range accessURLs { - if objectURL := canonicalizeScopedStorageURL(accessURL, scopes); objectURL != "" { + if objectURL := canonicalizeScopedStorageURL(accessURL, scopes, organization, project); objectURL != "" { out = append(out, objectURL) continue } @@ -2591,7 +2594,7 @@ func canonicalizeRecordAccessURLs(accessURLs []string, scopes []domain.StorageBu return uniqueStrings(out) } -func canonicalizeScopedStorageURL(accessURL string, scopes []domain.StorageBucketScope) string { +func canonicalizeScopedStorageURL(accessURL string, scopes []domain.StorageBucketScope, organization string, project string) string { if len(scopes) == 0 { return "" } @@ -2602,15 +2605,18 @@ func canonicalizeScopedStorageURL(accessURL string, scopes []domain.StorageBucke targetBucket := "" prefixes := make([]string, 0, len(scopes)) for _, scope := range scopes { - bucket, prefix, ok := parseStorageScopePath(scope.Path) + if !storageScopeApplies(scope, organization, project) { + continue + } + scopeBucket, scopePrefix, ok := parseStorageScopePath(scope) if !ok { continue } - if strings.TrimSpace(bucket) != "" { - targetBucket = strings.TrimSpace(bucket) + if strings.TrimSpace(scopeBucket) != "" { + targetBucket = strings.TrimSpace(scopeBucket) } - if strings.TrimSpace(prefix) != "" { - prefixes = append(prefixes, strings.Trim(strings.TrimSpace(prefix), "/")) + if strings.TrimSpace(scopePrefix) != "" { + prefixes = append(prefixes, strings.Trim(strings.TrimSpace(scopePrefix), "/")) } } if targetBucket == "" { @@ -2623,8 +2629,32 @@ func canonicalizeScopedStorageURL(accessURL string, scopes []domain.StorageBucke return canonicalStorageURL(targetBucket, normalizedKey, "") } -func parseStorageScopePath(raw string) (string, string, bool) { - return parseStorageURL(raw) +func storageScopeApplies(scope domain.StorageBucketScope, organization string, project string) bool { + scopeOrg := strings.TrimSpace(scope.Organization) + if scopeOrg != "" && !strings.EqualFold(scopeOrg, strings.TrimSpace(organization)) { + return false + } + scopeProject := strings.TrimSpace(scope.ProjectID) + return scopeProject == "" || strings.EqualFold(scopeProject, strings.TrimSpace(project)) +} + +func parseStorageScopePath(scope domain.StorageBucketScope) (string, string, bool) { + bucket := strings.TrimSpace(scope.Bucket) + pathValue := strings.TrimSpace(scope.Path) + if pathValue == "" { + return bucket, "", bucket != "" + } + if strings.HasPrefix(strings.ToLower(pathValue), "s3://") { + parsedBucket, parsedPrefix, ok := parseStorageURLAllowRoot(pathValue) + if !ok { + return bucket, "", bucket != "" + } + if bucket == "" { + bucket = parsedBucket + } + return bucket, parsedPrefix, bucket != "" + } + return bucket, strings.Trim(pathValue, "/"), bucket != "" } func parseStorageURL(raw string) (string, string, bool) { @@ -2649,6 +2679,27 @@ func parseStorageURL(raw string) (string, string, bool) { return bucket, key, true } +func parseStorageURLAllowRoot(raw string) (string, string, bool) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "", "", false + } + if !strings.HasPrefix(strings.ToLower(trimmed), "s3://") { + return "", "", false + } + rest := strings.TrimPrefix(trimmed, "s3://") + rest = strings.TrimLeft(rest, "/") + parts := strings.SplitN(rest, "/", 2) + bucket := strings.TrimSpace(parts[0]) + if bucket == "" { + return "", "", false + } + if len(parts) == 1 { + return bucket, "", true + } + return bucket, strings.Trim(strings.TrimSpace(parts[1]), "/"), true +} + func normalizeScopedStorageKeyForGecko(key string, prefixes []string) string { key = strings.Trim(strings.TrimSpace(key), "/") normalizedPrefixes := normalizedScopePrefixesForGecko(prefixes) diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index c21d154..bed3cc0 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -487,7 +487,7 @@ func projectAuditRecordPathURLs(record gintegrationsyfon.ProjectRecord, scopes [ out = append(out, trimmed) } } - out = append(out, canonicalizeRecordAccessURLs(out, scopes)...) + out = append(out, canonicalizeRecordAccessURLs(out, scopes, record.Organization, record.Project)...) return uniqueStrings(out) } diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 09c649b..b9744c9 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -2447,11 +2447,100 @@ func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) t.Fatalf("did not expect stale raw URL to produce a chain finding once mapped object exists: %+v", finding) } } - if len(backend.probeItems) != 2 { - t.Fatalf("expected raw and canonical probes, got %+v", backend.probeItems) + if len(backend.probeItems) != 1 { + t.Fatalf("expected only the canonical scoped probe, got %+v", backend.probeItems) } - if !containsProbeTarget(backend.probeItems, "s3://bforepc-prod/OHSU/slide.ome.tiff") || !containsProbeTarget(backend.probeItems, "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff") { - t.Fatalf("expected probe targets to include raw and canonical URLs, got %+v", backend.probeItems) + if containsProbeTarget(backend.probeItems, "s3://bforepc-prod/OHSU/slide.ome.tiff") { + t.Fatalf("did not expect raw alias URL to be probed once scoped URL resolves: %+v", backend.probeItems) + } + if !containsProbeTarget(backend.probeItems, "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff") { + t.Fatalf("expected probe targets to include canonical scoped URL, got %+v", backend.probeItems) + } +} + +func TestCanonicalizeScopedStorageURLMatchesSyfonDownloadScopeResolution(t *testing.T) { + tests := []struct { + name string + accessURL string + scopes []domain.StorageBucketScope + org string + project string + want string + }{ + { + name: "project scope rewrites alias bucket to physical bucket", + accessURL: "s3://bforepc-prod/OHSU/koei_chin/visium_hd/barcodes.tsv.gz", + scopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + org: "HTAN_INT", + project: "BForePC", + want: "s3://bforepc/bforepc-prod/OHSU/koei_chin/visium_hd/barcodes.tsv.gz", + }, + { + name: "root scope does not require slash in path", + accessURL: "s3://alias/path/file.txt", + scopes: []domain.StorageBucketScope{{ + Bucket: "physical", + Organization: "org", + ProjectID: "proj", + Path: "s3://physical", + }}, + org: "org", + project: "proj", + want: "s3://physical/path/file.txt", + }, + { + name: "explicit bucket with prefix path", + accessURL: "s3://alias/path/file.txt", + scopes: []domain.StorageBucketScope{{ + Bucket: "physical", + Organization: "org", + ProjectID: "proj", + Path: "root/prefix", + }}, + org: "org", + project: "proj", + want: "s3://physical/root/prefix/path/file.txt", + }, + { + name: "already scoped url is not double prefixed", + accessURL: "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", + scopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + org: "HTAN_INT", + project: "BForePC", + want: "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", + }, + { + name: "unrelated scope is ignored", + accessURL: "s3://alias/path/file.txt", + scopes: []domain.StorageBucketScope{{ + Bucket: "physical", + Organization: "other", + ProjectID: "proj", + Path: "s3://physical/root", + }}, + org: "org", + project: "proj", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := canonicalizeScopedStorageURL(tt.accessURL, tt.scopes, tt.org, tt.project) + if got != tt.want { + t.Fatalf("unexpected scoped URL: got %q want %q", got, tt.want) + } + }) } } diff --git a/internal/git/types.go b/internal/git/types.go index 0c34de5..17ab650 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -9,6 +9,7 @@ import ( appconfig "github.com/calypr/gecko/config" geckodb "github.com/calypr/gecko/internal/db" "github.com/calypr/gecko/internal/git/domain" + "github.com/calypr/gecko/internal/httpclient" "github.com/calypr/gecko/internal/integrations/fence" gitapi "github.com/calypr/gecko/internal/integrations/github" "github.com/jmoiron/sqlx" @@ -666,7 +667,7 @@ func NewGitService(config GitServiceConfig) *GitService { } client := config.HTTPClient if client == nil { - client = &http.Client{Timeout: 20 * time.Second} + client = httpclient.NewServiceClient(20 * time.Second) } return &GitService{ config: config, diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go new file mode 100644 index 0000000..441eb0b --- /dev/null +++ b/internal/httpclient/client.go @@ -0,0 +1,27 @@ +package httpclient + +import ( + "crypto/tls" + "net" + "net/http" + "time" +) + +// NewServiceClient returns a default outbound service client with HTTP/2 +// disabled. Gecko mostly talks to in-cluster services over HTTP/1.1, and this +// avoids process-fatal panics in Go's HTTP/2 HPACK encoder under concurrent +// service traffic. +func NewServiceClient(timeout time.Duration) *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ForceAttemptHTTP2 = false + transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} + transport.DialContext = (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext + + return &http.Client{ + Timeout: timeout, + Transport: transport, + } +} diff --git a/internal/integrations/fence/client.go b/internal/integrations/fence/client.go index 4e9b699..5ea859a 100644 --- a/internal/integrations/fence/client.go +++ b/internal/integrations/fence/client.go @@ -8,8 +8,10 @@ import ( "io" "net/http" "strings" + "time" "github.com/calypr/gecko/internal/git/domain" + "github.com/calypr/gecko/internal/httpclient" servermw "github.com/calypr/gecko/internal/server/middleware" ) @@ -24,7 +26,7 @@ type Client struct { func NewClient(client *http.Client, config Config) *Client { if client == nil { - client = http.DefaultClient + client = httpclient.NewServiceClient(30 * time.Second) } return &Client{client: client, config: config} } diff --git a/internal/integrations/github/client.go b/internal/integrations/github/client.go index e434851..2838222 100644 --- a/internal/integrations/github/client.go +++ b/internal/integrations/github/client.go @@ -5,8 +5,10 @@ import ( "fmt" "net/http" "strings" + "time" "github.com/calypr/gecko/internal/git/domain" + "github.com/calypr/gecko/internal/httpclient" google_github "github.com/google/go-github/v87/github" ) @@ -26,7 +28,7 @@ type GitHubRepositoryMetadata struct { func NewClient(client *http.Client, config Config) *Client { if client == nil { - client = http.DefaultClient + client = httpclient.NewServiceClient(30 * time.Second) } if config.APIBase == "" { config.APIBase = "https://api.github.com" diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index 4da1edb..019b983 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -17,6 +17,7 @@ import ( "time" "github.com/calypr/gecko/internal/git/domain" + "github.com/calypr/gecko/internal/httpclient" "github.com/calypr/syfon/apigen/client/bucketapi" drsapi "github.com/calypr/syfon/apigen/client/drs" internalapi "github.com/calypr/syfon/apigen/client/internalapi" @@ -158,7 +159,7 @@ type ProjectBucketDeleteResult struct { func NewManager(baseURL string, client *http.Client) *Manager { httpClient := client if httpClient == nil { - httpClient = http.DefaultClient + httpClient = httpclient.NewServiceClient(5 * time.Minute) } return &Manager{ baseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"), diff --git a/internal/server/http/register.go b/internal/server/http/register.go index b0b1c1b..9a89aaa 100644 --- a/internal/server/http/register.go +++ b/internal/server/http/register.go @@ -1,7 +1,6 @@ package httpapi import ( - "net/http" "strings" "github.com/calypr/gecko/internal/httputil" @@ -18,7 +17,7 @@ type Dependencies = shared.Dependencies func Register(app *fiber.App, deps Dependencies) { handler := shared.NewHandler(deps) - authzHandler := servermw.NewFenceUserAccessHandler(http.DefaultClient) + authzHandler := servermw.NewFenceUserAccessHandler(nil) app.Get("/swagger/doc.json", func(ctx fiber.Ctx) error { return ctx.SendFile("./docs/swagger.json") diff --git a/internal/server/http/shared/handler.go b/internal/server/http/shared/handler.go index 90cca05..a2214b3 100644 --- a/internal/server/http/shared/handler.go +++ b/internal/server/http/shared/handler.go @@ -1,7 +1,6 @@ package shared import ( - "net/http" "os" "strings" @@ -48,7 +47,7 @@ func NewHandler(deps Dependencies) *Handler { var projectSync *git.ReconcileService var storageManager *gintegrationsyfon.Manager if deps.GitService != nil { - storageManager = gintegrationsyfon.NewManager(strings.TrimSpace(os.Getenv("SYFON_DATA_API_BASE_URL")), http.DefaultClient) + storageManager = gintegrationsyfon.NewManager(strings.TrimSpace(os.Getenv("SYFON_DATA_API_BASE_URL")), nil) projectSetup = git.NewSetupService(deps.DB, deps.GitService, storageManager, servermw.NewFenceUserAccessHandler(nil)) projectSync = git.NewReconcileService( deps.DB, diff --git a/internal/server/middleware/middleware.go b/internal/server/middleware/middleware.go index 75cd8c8..d974c56 100644 --- a/internal/server/middleware/middleware.go +++ b/internal/server/middleware/middleware.go @@ -7,8 +7,10 @@ import ( "io" "net/http" "strings" + "time" "github.com/calypr/gecko/apierror" + "github.com/calypr/gecko/internal/httpclient" "github.com/calypr/gecko/internal/httputil" "github.com/golang-jwt/jwt/v5" ) @@ -35,7 +37,7 @@ type FenceUserAccessHandler struct { func NewFenceUserAccessHandler(client *http.Client) *FenceUserAccessHandler { if client == nil { - client = http.DefaultClient + client = httpclient.NewServiceClient(30 * time.Second) } return &FenceUserAccessHandler{client: client} } From 9d95340ef0a3cca87db19653876ca8518c8990ee Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 7 Jul 2026 16:29:56 -0700 Subject: [PATCH 23/36] fix bugs --- internal/git/storage_analytics.go | 18 ++- internal/git/storage_analytics_pipeline.go | 93 ++++++++--- internal/git/storage_analytics_test.go | 176 +++++++++++++++++++++ internal/integrations/syfon/adapter.go | 12 +- 4 files changed, 266 insertions(+), 33 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index d4cc7fe..70ccb02 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -1230,14 +1230,14 @@ func (service *StorageAnalyticsService) loadProjectStorageScopes(ctx context.Con return scopes, nil } -func applyScopedStorageMappings(recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, scopes []domain.StorageBucketScope) (map[string][]projectRecordState, map[string][]projectRecordState) { +func applyScopedStorageMappings(recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, scopes []domain.StorageBucketScope, organization string, project string) (map[string][]projectRecordState, map[string][]projectRecordState) { attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { out := make(map[string][]projectRecordState, len(input)) for checksum, group := range input { states := make([]projectRecordState, 0, len(group)) for _, record := range group { clone := record - clone.CanonicalAccessURLs = canonicalizeRecordAccessURLs(record.AccessURLs, scopes, record.Organization, record.Project) + clone.CanonicalAccessURLs = canonicalizeRecordAccessURLs(record.AccessURLs, scopes, organization, project) states = append(states, clone) } out[checksum] = states @@ -2594,6 +2594,20 @@ func canonicalizeRecordAccessURLs(accessURLs []string, scopes []domain.StorageBu return uniqueStrings(out) } +func canonicalizeRecordAccessURLsForProjectInventory(accessURLs []string, scopes []domain.StorageBucketScope, organization string, project string) ([]string, error) { + out := make([]string, 0, len(accessURLs)) + for _, accessURL := range accessURLs { + if objectURL := canonicalizeScopedStorageURL(accessURL, scopes, organization, project); objectURL != "" { + out = append(out, objectURL) + continue + } + if _, _, ok := parseStorageURL(accessURL); ok { + return nil, fmt.Errorf("storage access URL %q could not be mapped into bucket scopes for project %s/%s", strings.TrimSpace(accessURL), strings.TrimSpace(organization), strings.TrimSpace(project)) + } + } + return uniqueStrings(out), nil +} + func canonicalizeScopedStorageURL(accessURL string, scopes []domain.StorageBucketScope, organization string, project string) string { if len(scopes) == 0 { return "" diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index bed3cc0..19401b2 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -304,6 +304,12 @@ func (service *StorageAnalyticsService) loadProjectChainScopeMappings(ctx contex if err != nil { return nil, fmt.Errorf("list syfon project scopes: %w", err) } + if len(scopes) == 0 { + scopes, err = service.loadProjectStorageScopes(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + } sort.SliceStable(scopes, func(i, j int) bool { iProject := strings.TrimSpace(scopes[i].ProjectID) jProject := strings.TrimSpace(scopes[j].ProjectID) @@ -321,11 +327,11 @@ func (service *StorageAnalyticsService) loadProjectChainScopeMappings(ctx contex return scopes, nil } -func applyScopeCanonicalization(recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope) *storageAuditRecordSet { +func applyScopeCanonicalization(recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, organization string, project string) *storageAuditRecordSet { if recordSet == nil { return nil } - recordsByChecksum, allProjectRecords := applyScopedStorageMappings(recordSet.recordsByChecksum, recordSet.allProjectRecords, scopes) + recordsByChecksum, allProjectRecords := applyScopedStorageMappings(recordSet.recordsByChecksum, recordSet.allProjectRecords, scopes, organization, project) return &storageAuditRecordSet{ recordsByChecksum: recordsByChecksum, allProjectRecords: allProjectRecords, @@ -443,14 +449,14 @@ func projectAuditRecordValidatorFromSummary(summary gintegrationsyfon.ProjectMet } } -func filterProjectAuditRecordSet(recordSet *storageAuditRecordSet, pathPrefix string, scopes []domain.StorageBucketScope) *storageAuditRecordSet { +func filterProjectAuditRecordSet(recordSet *storageAuditRecordSet, pathPrefix string, scopes []domain.StorageBucketScope, organization string, project string) *storageAuditRecordSet { if recordSet == nil || normalizeRepoSubpath(pathPrefix) == "" { return recordSet } filtered := make(map[string][]projectRecordState, len(recordSet.allProjectRecords)) for checksum, group := range recordSet.allProjectRecords { for _, record := range group { - if projectAuditRecordMatchesPathPrefix(record.ProjectRecord, pathPrefix, scopes) { + if projectAuditRecordMatchesPathPrefix(record.ProjectRecord, pathPrefix, scopes, organization, project) { filtered[checksum] = append(filtered[checksum], record) } } @@ -461,12 +467,12 @@ func filterProjectAuditRecordSet(recordSet *storageAuditRecordSet, pathPrefix st } } -func projectAuditRecordMatchesPathPrefix(record gintegrationsyfon.ProjectRecord, pathPrefix string, scopes []domain.StorageBucketScope) bool { +func projectAuditRecordMatchesPathPrefix(record gintegrationsyfon.ProjectRecord, pathPrefix string, scopes []domain.StorageBucketScope, organization string, project string) bool { normalizedPrefix := normalizeRepoSubpath(pathPrefix) if normalizedPrefix == "" { return true } - for _, accessURL := range projectAuditRecordPathURLs(record, scopes) { + for _, accessURL := range projectAuditRecordPathURLs(record, scopes, organization, project) { _, key, ok := parseStorageURL(accessURL) if !ok { continue @@ -479,7 +485,7 @@ func projectAuditRecordMatchesPathPrefix(record gintegrationsyfon.ProjectRecord, return false } -func projectAuditRecordPathURLs(record gintegrationsyfon.ProjectRecord, scopes []domain.StorageBucketScope) []string { +func projectAuditRecordPathURLs(record gintegrationsyfon.ProjectRecord, scopes []domain.StorageBucketScope, organization string, project string) []string { out := make([]string, 0, len(record.AccessURLs)+len(record.AccessMethods)*2) out = append(out, record.AccessURLs...) for _, method := range record.AccessMethods { @@ -487,7 +493,7 @@ func projectAuditRecordPathURLs(record gintegrationsyfon.ProjectRecord, scopes [ out = append(out, trimmed) } } - out = append(out, canonicalizeRecordAccessURLs(out, scopes, record.Organization, record.Project)...) + out = append(out, canonicalizeRecordAccessURLs(out, scopes, organization, project)...) return uniqueStrings(out) } @@ -708,37 +714,60 @@ func (service *StorageAnalyticsService) attachProjectStorageListValidations(ctx }, nil } -func attachProjectStorageInventoryValidations(recordSet *storageAuditRecordSet, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) *storageAuditRecordSet { +func attachProjectStorageInventoryValidations(recordSet *storageAuditRecordSet, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, scopes []domain.StorageBucketScope, organization string, project string) (*storageAuditRecordSet, error) { if recordSet == nil { - return nil + return nil, nil } - attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { + inventoryBuckets := projectInventoryBuckets(bucketObjectsByURL) + attach := func(input map[string][]projectRecordState) (map[string][]projectRecordState, error) { out := make(map[string][]projectRecordState, len(input)) for checksum, group := range input { states := make([]projectRecordState, 0, len(group)) for _, record := range group { clone := record - clone.AccessProbes = inventoryValidationProbesForRecord(record, bucketObjectsByURL) + probes, err := inventoryValidationProbesForRecord(record, bucketObjectsByURL, inventoryBuckets, scopes, organization, project) + if err != nil { + return nil, err + } + clone.AccessProbes = probes states = append(states, clone) } out[checksum] = states } - return out + return out, nil } - return &storageAuditRecordSet{ - recordsByChecksum: attach(recordSet.recordsByChecksum), - allProjectRecords: attach(recordSet.allProjectRecords), + recordsByChecksum, err := attach(recordSet.recordsByChecksum) + if err != nil { + return nil, err } + allProjectRecords, err := attach(recordSet.allProjectRecords) + if err != nil { + return nil, err + } + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: allProjectRecords, + }, nil } -func inventoryValidationProbesForRecord(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []gintegrationsyfon.BulkStorageProbeResult { +func inventoryValidationProbesForRecord(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, inventoryBuckets map[string]struct{}, scopes []domain.StorageBucketScope, organization string, project string) ([]gintegrationsyfon.BulkStorageProbeResult, error) { probes := make([]gintegrationsyfon.BulkStorageProbeResult, 0) seen := make(map[string]struct{}) - for _, accessURL := range probeAccessURLsForRecord(record) { + accessURLs, err := canonicalizeRecordAccessURLsForProjectInventory(record.AccessURLs, scopes, organization, project) + if err != nil { + return nil, err + } + for _, accessURL := range accessURLs { objectURL := canonicalStorageURL("", "", accessURL) if objectURL == "" { continue } + bucket, _, _ := parseStorageURL(objectURL) + if len(inventoryBuckets) > 0 { + if _, ok := inventoryBuckets[bucket]; !ok { + return nil, fmt.Errorf("storage access URL mapped to %q, but bucket %q is not present in project bucket inventory for project %s/%s", objectURL, bucket, strings.TrimSpace(organization), strings.TrimSpace(project)) + } + } if _, ok := seen[objectURL]; ok { continue } @@ -750,7 +779,22 @@ func inventoryValidationProbesForRecord(record projectRecordState, bucketObjects } probes = append(probes, inventoryMissingProbe(record, objectURL, expectedName)) } - return probes + return probes, nil +} + +func projectInventoryBuckets(bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) map[string]struct{} { + out := make(map[string]struct{}) + for objectURL, item := range bucketObjectsByURL { + if bucket := strings.TrimSpace(item.Bucket); bucket != "" { + out[bucket] = struct{}{} + continue + } + bucket, _, ok := parseStorageURL(objectURL) + if ok { + out[bucket] = struct{}{} + } + } + return out } func inventoryPresentProbe(record projectRecordState, objectURL string, expectedName string, item gintegrationsyfon.ProjectBucketObject) gintegrationsyfon.BulkStorageProbeResult { @@ -820,7 +864,7 @@ func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context if err != nil { return nil, err } - recordSet = applyScopeCanonicalization(recordSet, scopes) + recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) view := &storageAuditStorageView{ scopes: scopes, recordsByChecksum: recordSet.recordsByChecksum, @@ -862,7 +906,7 @@ func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context } func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { - recordSet = applyScopeCanonicalization(recordSet, scopes) + recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) view := &storageAuditStorageView{ scopes: append([]domain.StorageBucketScope(nil), scopes...), recordsByChecksum: recordSet.recordsByChecksum, @@ -889,7 +933,10 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex } view.bucketObjects, view.bucketObjectsByURL = cloneBucketInventory(validationObjects, validationObjectsByURL) validateStart := time.Now() - probedRecordSet := attachProjectStorageInventoryValidations(recordSet, view.bucketObjectsByURL) + probedRecordSet, err := attachProjectStorageInventoryValidations(recordSet, view.bucketObjectsByURL, scopes, organization, project) + if err != nil { + return nil, err + } timings.Record("inventory_list_validation", time.Since(validateStart)) timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL)) view.recordsByChecksum = probedRecordSet.recordsByChecksum @@ -956,7 +1003,7 @@ func (service *StorageAnalyticsService) loadStorageAuditStorageView(ctx context. if err != nil { return nil, err } - recordSet = applyScopeCanonicalization(recordSet, scopes) + recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) if includeProbes { recordSet, err = service.attachProjectStorageProbes(ctx, authorizationHeader, recordSet) if err != nil { diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index b9744c9..08e2bdd 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -2458,6 +2458,182 @@ func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) } } +func TestBuildStorageChainAuditCanonicalizesStaleRecordBucketUsingAuditProjectScopes(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/file.bin": lfsPointer("dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + Organization: "Ellrott_Lab", + Project: "hla2vec", + Size: 100, + AccessURLs: []string{ + "s3://EllrottLab/calypr/ff4b54c2-fc50-5850-9fc3-9d662e1f9e44/dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + }, + }, + }, + buckets: map[string]domain.StorageBucket{ + "gdc-mirror-bucket": {Bucket: "gdc-mirror-bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "gdc-mirror-bucket": {{ + Bucket: "gdc-mirror-bucket", + Organization: "gdc_mirror", + ProjectID: "gdc_mirror", + Path: "s3://gdc-mirror-bucket/gdc_mirror", + }}, + }, + projectScopes: []domain.StorageBucketScope{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://gdc-mirror-bucket/gdc_mirror/calypr/ff4b54c2-fc50-5850-9fc3-9d662e1f9e44/dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + Bucket: "gdc-mirror-bucket", + Key: "gdc_mirror/calypr/ff4b54c2-fc50-5850-9fc3-9d662e1f9e44/dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + Path: "dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + SizeBytes: 100, + Provider: "s3", + MetaSHA256: "dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "gdc_mirror", + "gdc_mirror", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if backend.listBucketScopesCalls == 0 { + t.Fatalf("expected audit to fall back to bucket scope enumeration") + } + if got := chain.Summary.CountsByKind["syfon_missing_bucket_object"]; got != 0 { + t.Fatalf("expected stale EllrottLab URL to be canonicalized into the audit project bucket, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected current project bucket object to complete the chain, got summary %+v", chain.Summary) + } + findings := loadAllChainFindings(t, service, "gdc_mirror", "gdc_mirror", chain) + for _, finding := range findings { + if strings.Contains(finding.BucketObjectURL, "EllrottLab") || strings.Contains(finding.NormalizedPath, "EllrottLab") { + t.Fatalf("did not expect stale EllrottLab bucket to appear in findings: %+v", finding) + } + } +} + +func TestBuildStorageChainAuditErrorsWhenRecordBucketCannotMapToAuditProjectScope(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/file.bin": lfsPointer("dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + Organization: "Ellrott_Lab", + Project: "hla2vec", + Size: 100, + AccessURLs: []string{"s3://EllrottLab/calypr/file.bin"}, + }, + }, + projectScopes: []domain.StorageBucketScope{}, + } + service := NewStorageAnalyticsService(backend) + + _, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "gdc_mirror", + "gdc_mirror", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err == nil { + t.Fatalf("expected unmapped S3 access URL to fail audit") + } + if !strings.Contains(err.Error(), `storage access URL "s3://EllrottLab/calypr/file.bin" could not be mapped into bucket scopes for project gdc_mirror/gdc_mirror`) { + t.Fatalf("expected explicit unmapped URL error, got %v", err) + } +} + +func TestBuildStorageChainAuditErrorsWhenProjectScopeBucketDisagreesWithInventory(t *testing.T) { + checksum := "ba49c9465b911b950abd60fb333e76032db677b29194ae2268a2a6797402cb4f" + did := "0005c5a4-d3cb-57fc-ad83-78a52c950851" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/file.tsv": lfsPointer(checksum, 4247846), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: did, + Checksum: checksum, + Organization: "gdc_mirror", + Project: "gdc_mirror", + Size: 4247846, + Name: "186b06a9-0b6c-48fe-83fa-451514703728.rna_seq.augmented_star_gene_counts.tsv", + AccessURLs: []string{"s3://gdcdata/" + did + "/" + checksum}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + { + Bucket: "EllrottLab", + Organization: "gdc_mirror", + ProjectID: "gdc_mirror", + Path: "s3://EllrottLab/calypr", + }, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://gdcdata/" + did + "/" + checksum, + Bucket: "gdcdata", + Key: did + "/" + checksum, + Path: "186b06a9-0b6c-48fe-83fa-451514703728.rna_seq.augmented_star_gene_counts.tsv", + SizeBytes: 4247846, + Provider: "s3", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + _, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "gdc_mirror", + "gdc_mirror", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err == nil { + t.Fatalf("expected scope bucket mismatch to fail audit") + } + if !strings.Contains(err.Error(), `storage access URL mapped to "s3://EllrottLab/calypr/0005c5a4-d3cb-57fc-ad83-78a52c950851/ba49c9465b911b950abd60fb333e76032db677b29194ae2268a2a6797402cb4f"`) { + t.Fatalf("expected mapped EllrottLab URL in error, got %v", err) + } + if !strings.Contains(err.Error(), `bucket "EllrottLab" is not present in project bucket inventory for project gdc_mirror/gdc_mirror`) { + t.Fatalf("expected inventory bucket mismatch error, got %v", err) + } +} + func TestCanonicalizeScopedStorageURLMatchesSyfonDownloadScopeResolution(t *testing.T) { tests := []struct { name string diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index 019b983..dcad5e0 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -442,13 +442,9 @@ func (manager *Manager) GetProjectMetricsSummary(ctx context.Context, authorizat } func (manager *Manager) ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { - requestBody := struct { - Organization string `json:"organization,omitempty"` - Project string `json:"project,omitempty"` - }{ - Organization: strings.TrimSpace(organization), - Project: strings.TrimSpace(project), - } + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) var response struct { Items []struct { Bucket string `json:"bucket"` @@ -457,7 +453,7 @@ func (manager *Manager) ListProjectScopes(ctx context.Context, authorizationHead Path string `json:"path"` } `json:"items"` } - if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-scopes", nil, requestBody, &response); err != nil { + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, "/data/inspect/project-scopes", params, nil, &response); err != nil { return nil, fmt.Errorf("list syfon project scopes: %w", err) } out := make([]domain.StorageBucketScope, 0, len(response.Items)) From 304906577bce13d5efd4d6a1299c6030d9f7b263 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 8 Jul 2026 08:18:48 -0700 Subject: [PATCH 24/36] remove false positvies --- internal/git/storage_analytics.go | 188 +++++++++++++++++++- internal/git/storage_analytics_pipeline.go | 6 +- internal/git/storage_analytics_test.go | 133 +++++++++++--- internal/integrations/syfon/adapter.go | 8 +- internal/integrations/syfon/adapter_test.go | 41 +++++ 5 files changed, 347 insertions(+), 29 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 70ccb02..fab6e52 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -153,9 +153,10 @@ type RepoInventoryFile struct { type projectRecordState struct { gintegrationsyfon.ProjectRecord - CanonicalAccessURLs []string - Usage gintegrationsyfon.FileUsage - AccessProbes []gintegrationsyfon.BulkStorageProbeResult + CanonicalAccessURLs []string + CanonicalAccessURLByRaw map[string]string + Usage gintegrationsyfon.FileUsage + AccessProbes []gintegrationsyfon.BulkStorageProbeResult } type storageAggregate struct { @@ -794,7 +795,7 @@ func remainingAccessMethodsAfterBrokenRemoval(record GitStorageCleanupRecordAudi } remaining = append(remaining, method) } - return remaining + return appendReplacementAccessMethods(remaining, record.AccessProbes, brokenURLs) } func brokenAccessURLsForRecord(record GitStorageCleanupRecordAudit) map[string]struct{} { @@ -819,6 +820,55 @@ func brokenAccessURLsForRecord(record GitStorageCleanupRecordAudit) map[string]s return broken } +func appendReplacementAccessMethods(existing []gintegrationsyfon.ProjectAccessMethod, probes []GitStorageCleanupAccessProbe, brokenURLs map[string]struct{}) []gintegrationsyfon.ProjectAccessMethod { + seen := make(map[string]struct{}, len(existing)+len(probes)) + out := make([]gintegrationsyfon.ProjectAccessMethod, 0, len(existing)+len(probes)) + for _, method := range existing { + url := strings.TrimSpace(method.URL) + if url == "" { + continue + } + key := normalizeCleanupSelectionKey(url) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, method) + } + for _, probe := range probes { + if strings.TrimSpace(probe.Status) != "present" || !probeValidationMatched(probe.ValidationStatus) { + continue + } + replacementURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.URL) + if replacementURL == "" { + continue + } + key := normalizeCleanupSelectionKey(replacementURL) + if _, broken := brokenURLs[key]; broken { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, gintegrationsyfon.ProjectAccessMethod{ + AccessID: "s3", + Type: "s3", + URL: replacementURL, + }) + } + return out +} + +func probeValidationMatched(status string) bool { + switch strings.TrimSpace(status) { + case "", "not_requested", "matched": + return true + default: + return false + } +} + func recordStatusMeansBrokenAccess(status string) bool { switch strings.TrimSpace(status) { case "missing", "error": @@ -830,7 +880,7 @@ func recordStatusMeansBrokenAccess(status string) bool { func accessProbeIsBroken(probe GitStorageCleanupAccessProbe) bool { switch strings.TrimSpace(probe.ErrorKind) { - case "missing_access_url", "credential_missing": + case "missing_access_url", "credential_missing", "stale_scope_mapping": return true } switch strings.TrimSpace(probe.Status) { @@ -1238,6 +1288,7 @@ func applyScopedStorageMappings(recordsByChecksum map[string][]projectRecordStat for _, record := range group { clone := record clone.CanonicalAccessURLs = canonicalizeRecordAccessURLs(record.AccessURLs, scopes, organization, project) + clone.CanonicalAccessURLByRaw = canonicalizeRecordAccessURLMappings(record.AccessURLs, scopes, organization, project) states = append(states, clone) } out[checksum] = states @@ -1977,6 +2028,9 @@ func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[st if hasExactPathBucketMismatch(record, bucketMatches, bucketObjectsByURL) { return storageFindingBrokenBucketMap } + if len(repairableBrokenAccessProbes(record)) > 0 { + return storageFindingBrokenBucketMap + } if recordHasMatchedCanonicalAccessProbe(record) { return storageFindingNone } @@ -2504,6 +2558,8 @@ func repairBrokenBucketMappingRecord(record projectRecordState) ([]gintegrations if !removedAny { return nil, false, false } + auditRecord := cleanupRecordAuditForRecord("", "access_url", record) + remaining = appendReplacementAccessMethods(remaining, auditRecord.AccessProbes, brokenAccessURLsForRecord(auditRecord)) if len(remaining) == 0 { return nil, true, true } @@ -2520,13 +2576,115 @@ func accessURLHasBrokenBucketMapping(accessURL string, probesByURL map[string][] if strings.TrimSpace(probe.Status) == "present" { return false } - if strings.TrimSpace(probe.ErrorKind) == "credential_missing" { + if syfonProbeIsBrokenAccess(probe) { hasBrokenBucketMapping = true } } return hasBrokenBucketMapping } +func repairableBrokenAccessRecord(record projectRecordState) projectRecordState { + clone := record + clone.AccessProbes = repairableBrokenAccessProbes(record) + return clone +} + +func repairableBrokenAccessProbes(record projectRecordState) []gintegrationsyfon.BulkStorageProbeResult { + if len(record.AccessProbes) == 0 { + return nil + } + probesByURL := make(map[string][]gintegrationsyfon.BulkStorageProbeResult, len(record.AccessProbes)) + for _, probe := range record.AccessProbes { + objectURL := syfonProbeObjectURL(probe) + if objectURL == "" { + continue + } + probesByURL[objectURL] = append(probesByURL[objectURL], probe) + } + out := make([]gintegrationsyfon.BulkStorageProbeResult, 0) + for _, accessURL := range rawAccessURLsForRecord(record) { + rawURL := strings.TrimSpace(accessURL) + if rawURL == "" { + continue + } + if accessURLHasPresentProbe(rawURL, probesByURL) { + continue + } + if mappedURL := strings.TrimSpace(record.CanonicalAccessURLByRaw[rawURL]); mappedURL != "" && mappedURL != rawURL && accessURLHasPresentProbe(mappedURL, probesByURL) { + out = append(out, staleScopeMappingProbe(rawURL, mappedURL)) + out = append(out, presentAccessProbes(mappedURL, probesByURL)...) + continue + } + for _, probe := range probesByURL[rawURL] { + if syfonProbeIsBrokenAccess(probe) { + out = append(out, probe) + } + } + } + return out +} + +func staleScopeMappingProbe(rawURL string, mappedURL string) gintegrationsyfon.BulkStorageProbeResult { + bucket, key, _ := parseStorageURL(mappedURL) + return gintegrationsyfon.BulkStorageProbeResult{ + ObjectURL: strings.TrimSpace(rawURL), + Operation: StorageChainValidationModeList, + Bucket: bucket, + Key: key, + Status: "error", + Exists: false, + ErrorKind: "stale_scope_mapping", + Error: fmt.Sprintf("access URL should be updated to current project scope URL %q", strings.TrimSpace(mappedURL)), + ValidationStatus: "unverifiable", + } +} + +func presentAccessProbes(accessURL string, probesByURL map[string][]gintegrationsyfon.BulkStorageProbeResult) []gintegrationsyfon.BulkStorageProbeResult { + out := make([]gintegrationsyfon.BulkStorageProbeResult, 0) + for _, probe := range probesByURL[strings.TrimSpace(accessURL)] { + if strings.TrimSpace(probe.Status) != "present" { + continue + } + switch strings.TrimSpace(probe.ValidationStatus) { + case "", "not_requested", "matched": + out = append(out, probe) + } + } + return out +} + +func accessURLHasPresentProbe(accessURL string, probesByURL map[string][]gintegrationsyfon.BulkStorageProbeResult) bool { + for _, probe := range probesByURL[strings.TrimSpace(accessURL)] { + if strings.TrimSpace(probe.Status) != "present" { + continue + } + switch strings.TrimSpace(probe.ValidationStatus) { + case "", "not_requested", "matched": + return true + } + } + return false +} + +func syfonProbeObjectURL(probe gintegrationsyfon.BulkStorageProbeResult) string { + if objectURL := strings.TrimSpace(probe.ObjectURL); objectURL != "" { + return objectURL + } + return canonicalStorageURL(probe.Bucket, probe.Key, "") +} + +func syfonProbeIsBrokenAccess(probe gintegrationsyfon.BulkStorageProbeResult) bool { + switch strings.TrimSpace(probe.ErrorKind) { + case "missing_access_url", "credential_missing", "stale_scope_mapping": + return true + } + switch strings.TrimSpace(probe.Status) { + case "missing", "forbidden", "unsupported", "invalid", "error": + return true + } + return false +} + func classifyRawAccessURLFindings(record projectRecordState) storageFindingKind { if len(record.AccessProbes) == 0 { return storageFindingNone @@ -2594,6 +2752,24 @@ func canonicalizeRecordAccessURLs(accessURLs []string, scopes []domain.StorageBu return uniqueStrings(out) } +func canonicalizeRecordAccessURLMappings(accessURLs []string, scopes []domain.StorageBucketScope, organization string, project string) map[string]string { + out := make(map[string]string, len(accessURLs)) + for _, accessURL := range accessURLs { + rawURL := strings.TrimSpace(accessURL) + if rawURL == "" { + continue + } + if objectURL := canonicalizeScopedStorageURL(rawURL, scopes, organization, project); objectURL != "" { + out[rawURL] = objectURL + continue + } + if objectURL := canonicalStorageURL("", "", rawURL); objectURL != "" { + out[rawURL] = objectURL + } + } + return out +} + func canonicalizeRecordAccessURLsForProjectInventory(accessURLs []string, scopes []domain.StorageBucketScope, organization string, project string) ([]string, error) { out := make([]string, 0, len(accessURLs)) for _, accessURL := range accessURLs { diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 19401b2..2221113 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -1270,7 +1270,11 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum bucketMatches := matchedBucketObjectURLs(record, index.bucketObjectsByURL) switch classifyStorageFinding(record, index.bucketObjectsByURL) { case storageFindingBrokenBucketMap: - findings := buildChainRecordFindingsWithOptions("syfon_broken_bucket_mapping", record, gitPaths, bucketMatches, "Syfon access URL did not resolve through a configured bucket mapping.", true) + findingRecord := repairableBrokenAccessRecord(record) + if len(findingRecord.AccessProbes) == 0 { + findingRecord = record + } + findings := buildChainRecordFindingsWithOptions("syfon_broken_bucket_mapping", findingRecord, gitPaths, bucketMatches, "Syfon access URL did not resolve through a configured bucket mapping.", true) acc.findings = append(acc.findings, findings...) acc.addCount("syfon_broken_bucket_mapping", len(findings)) case storageFindingObjectMissing: diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 08e2bdd..1e7ed38 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -2435,17 +2435,16 @@ func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) if err != nil { t.Fatalf("build chain audit: %v", err) } - if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { - t.Fatalf("expected mapped bucket match to suppress stale raw URL misclassification, got %+v", chain.Summary) + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { + t.Fatalf("expected mapped bucket match to surface stale access URL repair, got %+v", chain.Summary) } - if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { - t.Fatalf("expected mapped bucket match to preserve clean-chain count, got %+v", chain.Summary) + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 0 { + t.Fatalf("expected stale access URL repair to block clean-chain count, got %+v", chain.Summary) } findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) - for _, finding := range findings { - if finding.NormalizedPath == "data/slide.ome.tiff" { - t.Fatalf("did not expect stale raw URL to produce a chain finding once mapped object exists: %+v", finding) - } + finding := assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "data/slide.ome.tiff") + if finding.DefaultAction != storageActionRemoveBrokenAccessURLs { + t.Fatalf("expected stale URL finding to default to access method repair, got %+v", finding) } if len(backend.probeItems) != 1 { t.Fatalf("expected only the canonical scoped probe, got %+v", backend.probeItems) @@ -2522,14 +2521,13 @@ func TestBuildStorageChainAuditCanonicalizesStaleRecordBucketUsingAuditProjectSc if got := chain.Summary.CountsByKind["syfon_missing_bucket_object"]; got != 0 { t.Fatalf("expected stale EllrottLab URL to be canonicalized into the audit project bucket, got summary %+v", chain.Summary) } - if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { - t.Fatalf("expected current project bucket object to complete the chain, got summary %+v", chain.Summary) + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { + t.Fatalf("expected current project bucket object to surface stale access URL repair, got summary %+v", chain.Summary) } findings := loadAllChainFindings(t, service, "gdc_mirror", "gdc_mirror", chain) - for _, finding := range findings { - if strings.Contains(finding.BucketObjectURL, "EllrottLab") || strings.Contains(finding.NormalizedPath, "EllrottLab") { - t.Fatalf("did not expect stale EllrottLab bucket to appear in findings: %+v", finding) - } + finding := assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "data/file.bin") + if strings.Contains(finding.BucketObjectURL, "EllrottLab") || strings.Contains(finding.NormalizedPath, "EllrottLab") { + t.Fatalf("did not expect stale EllrottLab bucket to appear as resolved target: %+v", finding) } } @@ -2800,7 +2798,7 @@ func TestBuildStorageChainAuditFlagsExactPathMismatchWhenHashExistsElsewhereInBu assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "CONFIG/cbds-BForePC.json") } -func TestClassifyStorageFindingSuppressesRawURLFailuresWhenScopedProbeMatches(t *testing.T) { +func TestClassifyStorageFindingSurfacesStaleRawURLWhenScopedProbeMatches(t *testing.T) { record := projectRecordState{ ProjectRecord: gintegrationsyfon.ProjectRecord{ ObjectID: "obj-a", @@ -2809,6 +2807,9 @@ func TestClassifyStorageFindingSuppressesRawURLFailuresWhenScopedProbeMatches(t AccessURLs: []string{"s3://bforepc-prod/JHU/slide.ome.tiff"}, }, CanonicalAccessURLs: []string{"s3://bforepc/bforepc-prod/JHU/slide.ome.tiff"}, + CanonicalAccessURLByRaw: map[string]string{ + "s3://bforepc-prod/JHU/slide.ome.tiff": "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + }, AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ { ObjectURL: "s3://bforepc-prod/JHU/slide.ome.tiff", @@ -2828,12 +2829,19 @@ func TestClassifyStorageFindingSuppressesRawURLFailuresWhenScopedProbeMatches(t }, } - if got := classifyStorageFinding(record, nil); got != storageFindingNone { - t.Fatalf("expected scoped matched probe to suppress raw URL credential miss, got %s", got) + if got := classifyStorageFinding(record, nil); got != storageFindingBrokenBucketMap { + t.Fatalf("expected scoped matched probe to surface repairable stale URL, got %s", got) + } + repairRecord := repairableBrokenAccessRecord(record) + if len(repairRecord.AccessProbes) != 2 { + t.Fatalf("expected stale raw and scoped replacement probes, got %+v", repairRecord.AccessProbes) + } + if repairRecord.AccessProbes[0].ErrorKind != "stale_scope_mapping" || repairRecord.AccessProbes[1].ObjectURL != "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff" { + t.Fatalf("unexpected repair probes: %+v", repairRecord.AccessProbes) } } -func TestClassifyStorageFindingSuppressesRawURLMismatchWhenScopedProbeMatches(t *testing.T) { +func TestClassifyStorageFindingSurfacesRawURLMismatchWhenScopedProbeMatches(t *testing.T) { record := projectRecordState{ ProjectRecord: gintegrationsyfon.ProjectRecord{ ObjectID: "obj-a", @@ -2842,6 +2850,9 @@ func TestClassifyStorageFindingSuppressesRawURLMismatchWhenScopedProbeMatches(t AccessURLs: []string{"s3://bforepc-prod/JHU/slide.ome.tiff"}, }, CanonicalAccessURLs: []string{"s3://bforepc/bforepc-prod/JHU/slide.ome.tiff"}, + CanonicalAccessURLByRaw: map[string]string{ + "s3://bforepc-prod/JHU/slide.ome.tiff": "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + }, AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ { ObjectURL: "s3://bforepc-prod/JHU/slide.ome.tiff", @@ -2861,8 +2872,90 @@ func TestClassifyStorageFindingSuppressesRawURLMismatchWhenScopedProbeMatches(t }, } - if got := classifyStorageFinding(record, nil); got != storageFindingNone { - t.Fatalf("expected scoped matched probe to suppress raw URL mismatch, got %s", got) + if got := classifyStorageFinding(record, nil); got != storageFindingBrokenBucketMap { + t.Fatalf("expected scoped matched probe to surface repairable stale URL, got %s", got) + } +} + +func TestClassifyStorageFindingSurfacesRepairableBrokenAccessMethod(t *testing.T) { + record := projectRecordState{ + ProjectRecord: gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Size: 100, + AccessURLs: []string{ + "s3://retired-bucket/JHU/slide.ome.tiff", + "s3://bforepc-prod/JHU/slide.ome.tiff", + }, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{ + {AccessID: "s3", Type: "s3", URL: "s3://retired-bucket/JHU/slide.ome.tiff"}, + {AccessID: "s3", Type: "s3", URL: "s3://bforepc-prod/JHU/slide.ome.tiff"}, + }, + }, + CanonicalAccessURLs: []string{"s3://retired-bucket/JHU/slide.ome.tiff", "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff"}, + CanonicalAccessURLByRaw: map[string]string{ + "s3://retired-bucket/JHU/slide.ome.tiff": "s3://retired-bucket/JHU/slide.ome.tiff", + "s3://bforepc-prod/JHU/slide.ome.tiff": "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + }, + AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ + { + ObjectURL: "s3://retired-bucket/JHU/slide.ome.tiff", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + ValidationStatus: "unverifiable", + }, + { + ObjectURL: "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + Bucket: "bforepc", + Key: "bforepc-prod/JHU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + + if got := classifyStorageFinding(record, nil); got != storageFindingBrokenBucketMap { + t.Fatalf("expected repairable broken access method finding, got %s", got) + } + broken := repairableBrokenAccessRecord(record) + if len(broken.AccessProbes) != 3 || broken.AccessProbes[0].ObjectURL != "s3://retired-bucket/JHU/slide.ome.tiff" || broken.AccessProbes[1].ErrorKind != "stale_scope_mapping" { + t.Fatalf("expected retired, stale, and replacement probes in repair record, got %+v", broken.AccessProbes) + } + remaining, shouldDelete, ok := repairBrokenBucketMappingRecord(broken) + if !ok || shouldDelete { + t.Fatalf("expected repair update, got ok=%v shouldDelete=%v remaining=%+v", ok, shouldDelete, remaining) + } + if len(remaining) != 1 || remaining[0].URL != "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff" { + t.Fatalf("expected working access method to be replaced with scoped URL, got %+v", remaining) + } +} + +func TestRemainingAccessMethodsReplacesStaleScopeMapping(t *testing.T) { + record := GitStorageCleanupRecordAudit{ + ObjectID: "obj-a", + AccessURLs: []string{"s3://bforepc-prod/JHU/slide.ome.tiff"}, + AccessMethods: []GitStorageCleanupAccessMethod{{AccessID: "s3", Type: "s3", URL: "s3://bforepc-prod/JHU/slide.ome.tiff"}}, + AccessProbes: []GitStorageCleanupAccessProbe{ + { + URL: "s3://bforepc-prod/JHU/slide.ome.tiff", + Status: "error", + ErrorKind: "stale_scope_mapping", + }, + { + URL: "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + Bucket: "bforepc", + Key: "bforepc-prod/JHU/slide.ome.tiff", + Status: "present", + ValidationStatus: "matched", + }, + }, + } + + remaining := remainingAccessMethodsAfterBrokenRemoval(record) + if len(remaining) != 1 || remaining[0].URL != "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff" { + t.Fatalf("expected stale URL replaced with scoped URL, got %+v", remaining) } } diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index dcad5e0..c048ff5 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -396,12 +396,16 @@ func (manager *Manager) ListProjectAuditRecords(ctx context.Context, authorizati } accessMethods := make([]ProjectAccessMethod, 0, len(item.AccessMethods)) for _, method := range item.AccessMethods { + methodURL := strings.TrimSpace(method.URL) accessMethods = append(accessMethods, ProjectAccessMethod{ AccessID: strings.TrimSpace(method.AccessID), Type: strings.TrimSpace(method.Type), - URL: strings.TrimSpace(method.URL), + URL: methodURL, Headers: append([]string(nil), method.Headers...), }) + if methodURL != "" { + accessURLs = append(accessURLs, methodURL) + } } out = append(out, ProjectRecord{ ObjectID: strings.TrimSpace(item.ObjectID), @@ -412,7 +416,7 @@ func (manager *Manager) ListProjectAuditRecords(ctx context.Context, authorizati Size: item.Size, CreatedAt: parseOptionalTime(optionalString(item.CreatedTime)), UpdatedAt: parseOptionalTime(optionalString(item.UpdatedTime)), - AccessURLs: accessURLs, + AccessURLs: uniqueNonEmptyStrings(accessURLs), AccessMethods: accessMethods, }) } diff --git a/internal/integrations/syfon/adapter_test.go b/internal/integrations/syfon/adapter_test.go index a91b011..13c2401 100644 --- a/internal/integrations/syfon/adapter_test.go +++ b/internal/integrations/syfon/adapter_test.go @@ -545,6 +545,47 @@ func TestListProjectAuditRecordsIncludesPathPrefix(t *testing.T) { } } +func TestListProjectAuditRecordsFoldsAccessMethodURLsIntoAccessURLs(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/project-records" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + body, err := json.Marshal(map[string]any{"items": []map[string]any{{ + "object_id": "obj-1", + "name": "file.txt", + "checksum": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "size": 10, + "access_methods": []map[string]any{ + {"access_id": "s3", "type": "s3", "url": "s3://bucket/path/file.txt"}, + {"access_id": "s3", "type": "s3", "url": "s3://bucket/path/file.txt"}, + }, + }}}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + records, err := manager.ListProjectAuditRecords(context.Background(), "Bearer token", "org", "proj", "") + if err != nil { + t.Fatalf("ListProjectAuditRecords returned error: %v", err) + } + if len(records) != 1 { + t.Fatalf("expected one record, got %+v", records) + } + if !reflect.DeepEqual(records[0].AccessURLs, []string{"s3://bucket/path/file.txt"}) { + t.Fatalf("expected access method URL folded into access URLs, got %+v", records[0].AccessURLs) + } + if len(records[0].AccessMethods) != 2 { + t.Fatalf("expected access methods preserved, got %+v", records[0].AccessMethods) + } +} + type roundTripFunc func(*http.Request) (*http.Response, error) func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { From 18ed0c9b97bae217c03df53f54420b3e9c7e10c7 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 8 Jul 2026 09:25:30 -0700 Subject: [PATCH 25/36] remove false positives, update apply route --- internal/git/storage_analytics.go | 297 +++++++++----- internal/git/storage_analytics_pipeline.go | 9 +- internal/git/storage_analytics_test.go | 368 ++++++++++++++---- internal/git/types.go | 3 + internal/server/http/git/storage_analytics.go | 1 + 5 files changed, 511 insertions(+), 167 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index fab6e52..5520037 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -60,6 +60,7 @@ func storageRepairPolicyForKind(kind string) storageRepairPolicy { return storageRepairPolicy{ actionability: storageActionabilityManualChoice, actions: []string{ + storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionDeleteBoth, @@ -610,6 +611,9 @@ func resolveStorageCleanupApplyAction(finding GitStorageCleanupApplyFinding, act return "", fmt.Errorf("unsupported cleanup finding kind %q", kind) } action := cleanupActionForApplyFinding(actionSelection, finding) + if action == "" { + action = strings.TrimSpace(finding.SuggestedAction) + } if action == "" { action = legacyDefaultStorageCleanupAction(kind, deleteRepoOrphans, deleteStaleDuplicates, deleteBucketOnlyObjects, repairBrokenBucketMappings) } @@ -880,7 +884,10 @@ func recordStatusMeansBrokenAccess(status string) bool { func accessProbeIsBroken(probe GitStorageCleanupAccessProbe) bool { switch strings.TrimSpace(probe.ErrorKind) { - case "missing_access_url", "credential_missing", "stale_scope_mapping": + case "missing_access_url", "credential_missing": + return true + } + if accessProbeHasMismatch(probe, "name_mismatch") { return true } switch strings.TrimSpace(probe.Status) { @@ -956,6 +963,7 @@ func (service *StorageAnalyticsService) executeStorageCleanupApplyPlan(ctx conte } if len(plan.UpdateAccessMethods) > 0 || len(toDelete) > 0 { service.evictProjectJoinCache(organization, project) + service.evictProjectAuditRecordCache(organization, project) } for _, objectID := range toDelete { success := true @@ -1117,44 +1125,26 @@ func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context service.projectJoinMu.Unlock() }() - checksums := make([]string, 0, len(inventory)) - seenChecksums := make(map[string]struct{}, len(inventory)) - for _, item := range inventory { - checksum := strings.TrimSpace(item.Checksum) - if checksum == "" { - continue - } - if _, ok := seenChecksums[checksum]; ok { - continue - } - seenChecksums[checksum] = struct{}{} - checksums = append(checksums, checksum) - } - recordsByChecksumRaw, err := service.storage.BulkGetProjectRecordsByChecksum(ctx, authorizationHeader, organization, project, checksums) + recordStart := time.Now() + projectRecords, err := service.loadCachedProjectAuditRecords(ctx, authorizationHeader, organization, project) if err != nil { - inflight.err = fmt.Errorf("lookup syfon project records by checksum: %w", err) + inflight.err = err return nil, nil, inflight.err } - objectIDs := projectRecordObjectIDs(recordsByChecksumRaw) + checksums := inventoryChecksumSet(inventory) + matchedRecords := filterProjectRecordsByChecksum(projectRecords, checksums) + log.Printf("storage_folder_exact_syfon_local_join org=%s project=%s git_checksums=%d project_records=%d matched_records=%d duration_ms=%d", organization, project, len(checksums), len(projectRecords), len(matchedRecords), time.Since(recordStart).Milliseconds()) + + usageStart := time.Now() + objectIDs := projectRecordObjectIDs(matchedRecords) usageByObjectID, err := service.listProjectFileUsageByObjectIDs(ctx, authorizationHeader, organization, project, objectIDs, cleanupInactiveDays) if err != nil { inflight.err = fmt.Errorf("list syfon project file usage: %w", err) return nil, nil, inflight.err } - recordsByChecksum := make(map[string][]projectRecordState, len(recordsByChecksumRaw)) - for _, records := range recordsByChecksumRaw { - for _, record := range records { - normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) - if normalizedChecksum == "" { - continue - } - record.Checksum = normalizedChecksum - recordsByChecksum[normalizedChecksum] = append(recordsByChecksum[normalizedChecksum], projectRecordState{ - ProjectRecord: record, - Usage: usageByObjectID[record.ObjectID], - }) - } - } + log.Printf("storage_folder_exact_bulk_usage_lookup org=%s project=%s object_ids=%d usage_rows=%d duration_ms=%d", organization, project, len(objectIDs), len(usageByObjectID), time.Since(usageStart).Milliseconds()) + + recordsByChecksum := buildProjectJoinRecordsByChecksum(matchedRecords, usageByObjectID) service.projectJoinMu.Lock() service.projectJoinCache[cacheKey] = cachedProjectJoinState{ expiresAt: time.Now().Add(projectJoinCacheTTL), @@ -1167,16 +1157,58 @@ func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context return recordsByChecksum, usageByObjectID, nil } -func projectRecordObjectIDs(recordsByChecksum map[string][]gintegrationsyfon.ProjectRecord) []string { - objectIDs := make([]string, 0) - for _, records := range recordsByChecksum { - for _, record := range records { - objectIDs = append(objectIDs, record.ObjectID) +func inventoryChecksumSet(inventory []RepoInventoryFile) map[string]struct{} { + checksums := make(map[string]struct{}, len(inventory)) + for _, item := range inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" { + continue + } + checksums[checksum] = struct{}{} + } + return checksums +} + +func filterProjectRecordsByChecksum(records []gintegrationsyfon.ProjectRecord, checksums map[string]struct{}) []gintegrationsyfon.ProjectRecord { + out := make([]gintegrationsyfon.ProjectRecord, 0) + for _, record := range records { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue } + if _, ok := checksums[normalizedChecksum]; !ok { + continue + } + record.Checksum = normalizedChecksum + out = append(out, record) + } + return out +} + +func projectRecordObjectIDs(records []gintegrationsyfon.ProjectRecord) []string { + objectIDs := make([]string, 0, len(records)) + for _, record := range records { + objectIDs = append(objectIDs, record.ObjectID) } return uniqueStrings(objectIDs) } +func buildProjectJoinRecordsByChecksum(records []gintegrationsyfon.ProjectRecord, usageByObjectID map[string]gintegrationsyfon.FileUsage) map[string][]projectRecordState { + recordsByChecksum := make(map[string][]projectRecordState) + for _, record := range records { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + record.Checksum = normalizedChecksum + recordsByChecksum[normalizedChecksum] = append(recordsByChecksum[normalizedChecksum], projectRecordState{ + ProjectRecord: record, + Usage: usageByObjectID[record.ObjectID], + }) + } + return recordsByChecksum +} + func (service *StorageAnalyticsService) listProjectFileUsageByObjectIDs(ctx context.Context, authorizationHeader string, organization string, project string, objectIDs []string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) { usageByObjectID := make(map[string]gintegrationsyfon.FileUsage) if len(objectIDs) == 0 { @@ -1216,6 +1248,13 @@ func (service *StorageAnalyticsService) evictProjectJoinCache(organization strin } } +func (service *StorageAnalyticsService) evictProjectAuditRecordCache(organization string, project string) { + cacheKey := service.projectChainInputCacheKey(organization, project) + service.chainInputMu.Lock() + delete(service.projectAuditCache, cacheKey) + service.chainInputMu.Unlock() +} + func (service *StorageAnalyticsService) listProjectRecordStates(ctx context.Context, authorizationHeader string, organization string, project string, usageByObjectID map[string]gintegrationsyfon.FileUsage) (map[string][]projectRecordState, error) { projectRecords, err := service.storage.ListProjectRecords(ctx, authorizationHeader, organization, project) if err != nil { @@ -2131,13 +2170,26 @@ func hasExactPathBucketMismatch(record projectRecordState, bucketMatches []strin func recordHasValidationMismatchProbe(record projectRecordState) bool { for _, probe := range record.AccessProbes { - if strings.TrimSpace(probe.ValidationStatus) == "mismatched" { + if storageProbeValidationMismatchIsSignificant(record, probe) { return true } } return false } +func storageProbeValidationMismatchIsSignificant(record projectRecordState, probe gintegrationsyfon.BulkStorageProbeResult) bool { + if strings.TrimSpace(probe.ValidationStatus) != "mismatched" { + return false + } + if !syfonProbeHasMismatch(probe, "size_mismatch") || len(probe.ValidationMismatches) != 1 { + return true + } + if probe.SizeBytes == nil { + return true + } + return !storageSizesMatchForAudit(record.Size, *probe.SizeBytes) +} + func recordHasMatchedCanonicalAccessProbe(record projectRecordState) bool { rawURLs := make(map[string]struct{}, len(record.AccessURLs)) for _, accessURL := range record.AccessURLs { @@ -2177,25 +2229,29 @@ func inventoryHasValidationMismatch(record projectRecordState, bucketObjectURLs if len(bucketObjectURLs) == 0 { return false } - checksum := normalizeAnalyticsChecksum(record.Checksum) for _, objectURL := range bucketObjectURLs { item, ok := bucketObjectsByURL[objectURL] if !ok { continue } - if record.Size > 0 && item.SizeBytes > 0 && item.SizeBytes != record.Size { + if !storageSizesMatchForAudit(record.Size, item.SizeBytes) { return true } - if checksum != "" { - metaSHA := normalizeAnalyticsChecksum(item.MetaSHA256) - if metaSHA != "" && metaSHA != checksum { - return true - } - } } return false } +func storageSizesMatchForAudit(expectedSize int64, observedSize int64) bool { + if expectedSize <= 0 || observedSize <= 0 { + return true + } + diff := expectedSize - observedSize + if diff < 0 { + diff = -diff + } + return diff <= 1 +} + func accessProbesForRecord(record projectRecordState) []GitStorageCleanupAccessProbe { if len(record.AccessProbes) > 0 { probes := make([]GitStorageCleanupAccessProbe, 0, len(record.AccessProbes)) @@ -2363,6 +2419,12 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, } actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupport(kind) primaryProbe := selectChainProbe(record, bucketObjectURLs) + suggestedFix := suggestedFixForChainFinding(kind, record) + suggestedAction := suggestedActionForChainFinding(kind, record) + if suggestedAction != "" && !contains(availableActions, suggestedAction) { + availableActions = append([]string(nil), availableActions...) + availableActions = append(availableActions, suggestedAction) + } findings := make([]GitStorageChainFinding, 0, len(paths)) for _, path := range paths { findings = append(findings, GitStorageChainFinding{ @@ -2382,6 +2444,8 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, RecordCount: 1, SizeBytes: record.Size, RecommendedAction: action, + SuggestedFix: suggestedFix, + SuggestedAction: suggestedAction, Actionability: actionability, AvailableActions: availableActions, DefaultAction: defaultAction, @@ -2392,6 +2456,81 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, return findings } +func suggestedActionForChainFinding(kind string, record projectRecordState) string { + switch strings.TrimSpace(kind) { + case "git_syfon_metadata_mismatch", "syfon_broken_bucket_mapping": + default: + return "" + } + if recordHasAccessProbeMismatch(record, "name_mismatch") { + return storageActionRemoveBrokenAccessURLs + } + return "" +} + +func suggestedFixForChainFinding(kind string, record projectRecordState) string { + normalizedKind := strings.TrimSpace(kind) + switch normalizedKind { + case "git_syfon_metadata_mismatch": + case "syfon_broken_bucket_mapping": + if !recordHasAccessProbeMismatch(record, "name_mismatch") { + return "" + } + default: + return "" + } + mismatches := make([]string, 0, 2) + if probe, ok := firstAccessProbeMismatch(record, "size_mismatch"); ok { + bucketSize := int64(0) + if probe.SizeBytes != nil { + bucketSize = *probe.SizeBytes + } + mismatches = append(mismatches, fmt.Sprintf("Syfon/Git size is %s, bucket inventory reports %s", formatAuditSize(record.Size), formatAuditSize(bucketSize))) + } + if probe, ok := firstAccessProbeMismatch(record, "name_mismatch"); ok { + expectedName := path.Base(strings.Trim(strings.TrimSpace(record.Name), "/")) + observedName := path.Base(strings.Trim(strings.TrimSpace(probe.Key), "/")) + mismatches = append(mismatches, fmt.Sprintf("Syfon name is %q, bucket key basename is %q", expectedName, observedName)) + } + if len(mismatches) == 0 { + return "Bucket object exists, but its storage evidence does not match the Syfon/Git record. Review the record and bucket object before applying a destructive fix." + } + if recordHasAccessProbeMismatch(record, "name_mismatch") { + return strings.Join(mismatches, ". ") + ". Remove the broken access URL if another access URL works; otherwise delete and recreate or correct the Syfon record." + } + return strings.Join(mismatches, ". ") + ". Update the stale Syfon metadata or delete and recreate the record after confirming the bucket object is authoritative." +} + +func recordHasAccessProbeMismatch(record projectRecordState, mismatch string) bool { + _, ok := firstAccessProbeMismatch(record, mismatch) + return ok +} + +func firstAccessProbeMismatch(record projectRecordState, mismatch string) (GitStorageCleanupAccessProbe, bool) { + for _, probe := range accessProbesForRecord(record) { + if accessProbeHasMismatch(probe, mismatch) { + return probe, true + } + } + return GitStorageCleanupAccessProbe{}, false +} + +func accessProbeHasMismatch(probe GitStorageCleanupAccessProbe, mismatch string) bool { + for _, item := range probe.ValidationMismatches { + if strings.TrimSpace(item) == mismatch { + return true + } + } + return false +} + +func formatAuditSize(size int64) string { + if size <= 0 { + return "unknown" + } + return fmt.Sprintf("%d B", size) +} + type chainProbeSelection struct { bucketObjectURL string probe GitStorageCleanupAccessProbe @@ -2611,8 +2750,6 @@ func repairableBrokenAccessProbes(record projectRecordState) []gintegrationsyfon continue } if mappedURL := strings.TrimSpace(record.CanonicalAccessURLByRaw[rawURL]); mappedURL != "" && mappedURL != rawURL && accessURLHasPresentProbe(mappedURL, probesByURL) { - out = append(out, staleScopeMappingProbe(rawURL, mappedURL)) - out = append(out, presentAccessProbes(mappedURL, probesByURL)...) continue } for _, probe := range probesByURL[rawURL] { @@ -2624,35 +2761,6 @@ func repairableBrokenAccessProbes(record projectRecordState) []gintegrationsyfon return out } -func staleScopeMappingProbe(rawURL string, mappedURL string) gintegrationsyfon.BulkStorageProbeResult { - bucket, key, _ := parseStorageURL(mappedURL) - return gintegrationsyfon.BulkStorageProbeResult{ - ObjectURL: strings.TrimSpace(rawURL), - Operation: StorageChainValidationModeList, - Bucket: bucket, - Key: key, - Status: "error", - Exists: false, - ErrorKind: "stale_scope_mapping", - Error: fmt.Sprintf("access URL should be updated to current project scope URL %q", strings.TrimSpace(mappedURL)), - ValidationStatus: "unverifiable", - } -} - -func presentAccessProbes(accessURL string, probesByURL map[string][]gintegrationsyfon.BulkStorageProbeResult) []gintegrationsyfon.BulkStorageProbeResult { - out := make([]gintegrationsyfon.BulkStorageProbeResult, 0) - for _, probe := range probesByURL[strings.TrimSpace(accessURL)] { - if strings.TrimSpace(probe.Status) != "present" { - continue - } - switch strings.TrimSpace(probe.ValidationStatus) { - case "", "not_requested", "matched": - out = append(out, probe) - } - } - return out -} - func accessURLHasPresentProbe(accessURL string, probesByURL map[string][]gintegrationsyfon.BulkStorageProbeResult) bool { for _, probe := range probesByURL[strings.TrimSpace(accessURL)] { if strings.TrimSpace(probe.Status) != "present" { @@ -2675,7 +2783,10 @@ func syfonProbeObjectURL(probe gintegrationsyfon.BulkStorageProbeResult) string func syfonProbeIsBrokenAccess(probe gintegrationsyfon.BulkStorageProbeResult) bool { switch strings.TrimSpace(probe.ErrorKind) { - case "missing_access_url", "credential_missing", "stale_scope_mapping": + case "missing_access_url", "credential_missing": + return true + } + if syfonProbeHasMismatch(probe, "name_mismatch") { return true } switch strings.TrimSpace(probe.Status) { @@ -2685,6 +2796,15 @@ func syfonProbeIsBrokenAccess(probe gintegrationsyfon.BulkStorageProbeResult) bo return false } +func syfonProbeHasMismatch(probe gintegrationsyfon.BulkStorageProbeResult, mismatch string) bool { + for _, item := range probe.ValidationMismatches { + if strings.TrimSpace(item) == mismatch { + return true + } + } + return false +} + func classifyRawAccessURLFindings(record projectRecordState) storageFindingKind { if len(record.AccessProbes) == 0 { return storageFindingNone @@ -2706,7 +2826,7 @@ func classifyRawAccessURLFindings(record projectRecordState) storageFindingKind hasBrokenBucketMapping := false hasProbeError := false for _, probe := range probes { - if strings.TrimSpace(probe.ValidationStatus) == "mismatched" { + if storageProbeValidationMismatchIsSignificant(record, probe) { return storageFindingValidationMismatch } if strings.TrimSpace(probe.Status) == "present" { @@ -3210,30 +3330,9 @@ func expectedStorageObjectNameForListValidation(objectURL string, recordName str if expectedName == "." || expectedName == "/" || expectedName == "" { return "" } - _, key, ok := parseStorageURL(objectURL) - if !ok { - return expectedName - } - keyBase := path.Base(strings.Trim(key, "/")) - if isContentAddressedStorageBasename(keyBase) { - return "" - } return expectedName } -func isContentAddressedStorageBasename(value string) bool { - trimmed := strings.ToLower(strings.TrimSpace(value)) - if len(trimmed) < 32 { - return false - } - for _, r := range trimmed { - if (r < '0' || r > '9') && (r < 'a' || r > 'f') { - return false - } - } - return true -} - func sortStorageAggregates(items []storageAggregate, sortBy string, sortOrder string) { desc := !strings.EqualFold(strings.TrimSpace(sortOrder), "asc") switch strings.ToLower(strings.TrimSpace(sortBy)) { diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 2221113..eba234c 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -355,6 +355,7 @@ func (service *StorageAnalyticsService) loadCachedProjectAuditRecordSet(ctx cont } func (service *StorageAnalyticsService) loadCachedProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) { + start := time.Now() cacheKey := service.projectChainInputCacheKey(organization, project) summary, err := service.storage.GetProjectMetricsSummary(ctx, authorizationHeader, organization, project) if err != nil { @@ -366,6 +367,7 @@ func (service *StorageAnalyticsService) loadCachedProjectAuditRecords(ctx contex cached, ok := service.projectAuditCache[cacheKey] service.chainInputMu.RUnlock() if ok && cached.validator == validator && time.Since(cached.cachedAt) < chainProjectRecordCacheMaxAge { + log.Printf("syfon_project_record_cache_hit org=%s project=%s record_count=%d age_ms=%d duration_ms=%d", organization, project, len(cached.records), time.Since(cached.cachedAt).Milliseconds(), time.Since(start).Milliseconds()) return cached.records, nil } @@ -373,6 +375,7 @@ func (service *StorageAnalyticsService) loadCachedProjectAuditRecords(ctx contex service.chainInputMu.Lock() if cached, ok := service.projectAuditCache[cacheKey]; ok && cached.validator == validator && time.Since(cached.cachedAt) < chainProjectRecordCacheMaxAge { service.chainInputMu.Unlock() + log.Printf("syfon_project_record_cache_hit org=%s project=%s record_count=%d age_ms=%d duration_ms=%d", organization, project, len(cached.records), time.Since(cached.cachedAt).Milliseconds(), time.Since(start).Milliseconds()) return cached.records, nil } if inflight, ok := service.projectAuditWork[workKey]; ok { @@ -381,6 +384,7 @@ func (service *StorageAnalyticsService) loadCachedProjectAuditRecords(ctx contex if inflight.err != nil { return nil, inflight.err } + log.Printf("syfon_project_record_cache_wait org=%s project=%s record_count=%d duration_ms=%d", organization, project, len(inflight.records), time.Since(start).Milliseconds()) return inflight.records, nil } inflight := &inflightProjectAuditRecordState{ @@ -389,6 +393,7 @@ func (service *StorageAnalyticsService) loadCachedProjectAuditRecords(ctx contex } service.projectAuditWork[workKey] = inflight service.chainInputMu.Unlock() + log.Printf("syfon_project_record_cache_refresh org=%s project=%s validator_count=%d validator_latest=%q validator_revision=%q", organization, project, validator.RecordCount, validator.RecordLatestUpdatedTime, validator.RecordRevision) defer func() { service.chainInputMu.Lock() delete(service.projectAuditWork, workKey) @@ -410,12 +415,14 @@ func (service *StorageAnalyticsService) loadCachedProjectAuditRecords(ctx contex } service.chainInputMu.Unlock() inflight.records = copiedRecords + log.Printf("syfon_project_record_cache_refreshed org=%s project=%s record_count=%d duration_ms=%d", organization, project, len(copiedRecords), time.Since(start).Milliseconds()) return copiedRecords, nil } projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project, "") if err != nil { return nil, fmt.Errorf("list syfon project audit records: %w", err) } + log.Printf("syfon_project_record_cache_uncached org=%s project=%s record_count=%d duration_ms=%d", organization, project, len(projectRecords), time.Since(start).Milliseconds()) return projectRecords, nil } @@ -799,7 +806,7 @@ func projectInventoryBuckets(bucketObjectsByURL map[string]gintegrationsyfon.Pro func inventoryPresentProbe(record projectRecordState, objectURL string, expectedName string, item gintegrationsyfon.ProjectBucketObject) gintegrationsyfon.BulkStorageProbeResult { size := item.SizeBytes - sizeMatch := item.SizeBytes == record.Size + sizeMatch := storageSizesMatchForAudit(record.Size, item.SizeBytes) nameMatch := true if expectedName != "" { nameMatch = path.Base(strings.Trim(strings.TrimSpace(item.Key), "/")) == expectedName diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 1e7ed38..bb22b9e 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -342,6 +342,15 @@ func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { if backend.listProjectFileUsageByObjectIDsCalls != 1 { t.Fatalf("expected exact summary to use bulk file usage lookup, got %d", backend.listProjectFileUsageByObjectIDsCalls) } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected exact summary to load shared project audit records once, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to join cached project records locally, got %d checksum lookup calls", backend.bulkGetProjectRecordsCalls) + } + if contains(backend.listProjectFileUsageObjectIDs, "obj-orphan") { + t.Fatalf("expected exact summary usage lookup to include only Git-matched records, got %+v", backend.listProjectFileUsageObjectIDs) + } if backend.listProjectFileUsageCalls != 0 { t.Fatalf("expected exact summary to skip paginated project usage lookup, got %d", backend.listProjectFileUsageCalls) } @@ -1759,11 +1768,11 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { if chain.Summary.CountsByKind["syfon_missing_bucket_object"] != 1 { t.Fatalf("expected validate mode to report missing Syfon bucket object, got summary %+v", chain.Summary) } - if got := chain.Summary.CountsByKind["git_syfon_metadata_mismatch"]; got != 1 { - t.Fatalf("expected one LIST-derived metadata mismatch, got summary %+v", chain.Summary) + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { + t.Fatalf("expected one LIST-derived broken bucket mapping, got summary %+v", chain.Summary) } assertNoChainFinding(t, chain.Findings, "bucket_only_object") - assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/mismatch.txt") + assertHasChainFinding(t, chain.Findings, "syfon_broken_bucket_mapping", "data/mismatch.txt") assertHasChainFinding(t, chain.Findings, "syfon_missing_bucket_object", "s3://bucket/syfon-only.txt") } @@ -1829,15 +1838,149 @@ func TestBuildStorageChainAuditMetadataModeUsesMetadataValidation(t *testing.T) } } -func TestExpectedStorageObjectNameForListValidationSkipsContentAddressedKeys(t *testing.T) { - if got := expectedStorageObjectNameForListValidation("s3://cbds/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/ec4b068cb42b52449dd44052c3bfb2a459b00336a9cd42cd29c22ca1d1b26cb0", "CONFIG/cbds-BForePC.json"); got != "" { - t.Fatalf("expected content-addressed storage key to skip name validation, got %q", got) +func TestExpectedStorageObjectNameForListValidationUsesRecordNameForContentAddressedKeys(t *testing.T) { + if got := expectedStorageObjectNameForListValidation("s3://cbds/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/ec4b068cb42b52449dd44052c3bfb2a459b00336a9cd42cd29c22ca1d1b26cb0", "CONFIG/cbds-BForePC.json"); got != "cbds-BForePC.json" { + t.Fatalf("expected content-addressed storage key to validate basename, got %q", got) } if got := expectedStorageObjectNameForListValidation("s3://bucket/path/cbds-BForePC.json", "CONFIG/cbds-BForePC.json"); got != "cbds-BForePC.json" { t.Fatalf("expected filename-backed storage key to validate basename, got %q", got) } } +func TestBuildStorageChainAuditIgnoresOneByteListSizeDrift(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 44), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Name: "a.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 44, AccessURLs: []string{"s3://bucket/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/a.txt", Bucket: "bucket", Key: "data/a.txt", Path: "data/a.txt", SizeBytes: 43}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + ValidationMode: StorageChainValidationModeList, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["git_syfon_metadata_mismatch"]; got != 0 { + t.Fatalf("expected one-byte size drift to be ignored, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected one-byte size drift to count as complete, got summary %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditSuggestsFixForLargeSizeDrift(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 44), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Name: "a.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 44, AccessURLs: []string{"s3://bucket/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/a.txt", Bucket: "bucket", Key: "data/a.txt", Path: "data/a.txt", SizeBytes: 40}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + ValidationMode: StorageChainValidationModeList, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + finding := assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/a.txt") + if !strings.Contains(finding.SuggestedFix, "Syfon/Git size is 44 B, bucket inventory reports 40 B") { + t.Fatalf("expected size-drift suggested fix, got %+v", finding) + } + if finding.SuggestedAction != "" { + t.Fatalf("did not expect fake suggested action for size drift, got %+v", finding) + } +} + +func TestBuildStorageChainAuditSuggestsRealAccessURLRepairForNameMismatch(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/right.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Name: "right.txt", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + AccessURLs: []string{"s3://bucket/data/wrong.txt", "s3://bucket/data/right.txt"}, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{ + {AccessID: "bad", Type: "s3", URL: "s3://bucket/data/wrong.txt"}, + {AccessID: "good", Type: "s3", URL: "s3://bucket/data/right.txt"}, + }, + }, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/wrong.txt", Bucket: "bucket", Key: "data/wrong.txt", Path: "data/wrong.txt", SizeBytes: 100}, + {ObjectURL: "s3://bucket/data/right.txt", Bucket: "bucket", Key: "data/right.txt", Path: "data/right.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + ValidationMode: StorageChainValidationModeList, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + finding := assertHasChainFinding(t, chain.Findings, "syfon_broken_bucket_mapping", "data/right.txt") + if finding.SuggestedAction != storageActionRemoveBrokenAccessURLs { + t.Fatalf("expected name mismatch to suggest real access URL repair, got %+v", finding) + } + if !contains(finding.AvailableActions, storageActionRemoveBrokenAccessURLs) { + t.Fatalf("expected suggested action to be advertised, got %+v", finding.AvailableActions) + } + if !strings.Contains(finding.SuggestedFix, `Syfon name is "right.txt", bucket key basename is "wrong.txt"`) { + t.Fatalf("expected name mismatch suggested fix, got %+v", finding) + } + + applyResponse, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", nil, nil, []GitStorageCleanupApplyFinding{{ + Kind: finding.Kind, + NormalizedPath: finding.NormalizedPath, + ObjectIDs: finding.ObjectIDs, + Records: finding.Records, + AccessURLs: finding.AccessURLs, + AvailableActions: finding.AvailableActions, + SuggestedAction: finding.SuggestedAction, + }}, false, false, false, false, false) + if err != nil { + t.Fatalf("apply suggested repair: %v", err) + } + if len(applyResponse.UpdatedRecordIDs) != 1 || applyResponse.UpdatedRecordIDs[0] != "obj-a" { + t.Fatalf("expected access method update, got %+v", applyResponse) + } + methods := backend.updatedAccessMethods["obj-a"] + if len(methods) != 1 || methods[0].URL != "s3://bucket/data/right.txt" { + t.Fatalf("expected broken access URL removed through Syfon update, got %+v", methods) + } +} + func TestBuildStorageChainAuditCachesProjectChainInputs(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -1879,6 +2022,125 @@ func TestBuildStorageChainAuditCachesProjectChainInputs(t *testing.T) { } } +func TestStorageFolderExactReusesAuditProjectRecordCache(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-present": {ObjectID: "obj-present", DownloadCount: 3}, + }, + } + service := NewStorageAnalyticsService(backend) + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly}); err != nil { + t.Fatalf("build chain audit: %v", err) + } + summary, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build exact storage summary: %v", err) + } + if summary.RecordCount != 1 || summary.DownloadCount != 3 { + t.Fatalf("expected exact summary to join cached record and usage, got %+v", summary) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected audit and exact summary to share one project record enumeration, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.getProjectMetricsSummaryCalls != 2 { + t.Fatalf("expected one cheap metrics validation per request, got %d", backend.getProjectMetricsSummaryCalls) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to avoid checksum bulk record lookup, got %d", backend.bulkGetProjectRecordsCalls) + } +} + +func TestStorageFolderExactWarmsAuditProjectRecordCache(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + + if _, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash); err != nil { + t.Fatalf("build exact storage summary: %v", err) + } + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly}); err != nil { + t.Fatalf("build chain audit: %v", err) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected exact summary and audit to share one project record enumeration, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.getProjectMetricsSummaryCalls != 2 { + t.Fatalf("expected one cheap metrics validation per request, got %d", backend.getProjectMetricsSummaryCalls) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to avoid checksum bulk record lookup, got %d", backend.bulkGetProjectRecordsCalls) + } +} + +func TestStorageFolderExactAndAuditCoalesceProjectRecordRefresh(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + listProjectAuditRecordsDelay: 50 * time.Millisecond, + projectMetricsSummary: &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 1, + RecordLatestUpdatedTime: "2026-07-01T00:00:00Z", + }, + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + errs := make(chan error, 2) + + go func() { + _, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + errs <- err + }() + go func() { + _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly}) + errs <- err + }() + for i := 0; i < 2; i++ { + if err := <-errs; err != nil { + t.Fatalf("expected concurrent exact summary and audit to succeed, got %v", err) + } + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected concurrent exact summary and audit to coalesce one project record refresh, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to avoid checksum bulk record lookup, got %d", backend.bulkGetProjectRecordsCalls) + } +} + func TestBuildStorageChainAuditCachesProjectRecordsPerPathPrefix(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "CONFIG/a.json": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -2019,8 +2281,8 @@ func TestStorageChildrenSkipsSummaryJoinUsageWork(t *testing.T) { t.Fatalf("expected concurrent storage analytics request to succeed, got %v", err) } } - if backend.bulkGetProjectRecordsCalls != 2 { - t.Fatalf("expected summary and page-scoped children checksum lookups, got %d", backend.bulkGetProjectRecordsCalls) + if backend.bulkGetProjectRecordsCalls != 1 { + t.Fatalf("expected only page-scoped children to use checksum lookup, got %d", backend.bulkGetProjectRecordsCalls) } if backend.listProjectFileUsageCalls != 0 { t.Fatalf("expected paginated usage lookup to remain unused, got %d", backend.listProjectFileUsageCalls) @@ -2086,7 +2348,7 @@ func TestListProjectFileUsageByObjectIDsFallsBackWhenBulkEndpointMissing(t *test } } -func TestLoadProjectJoinCacheDeduplicatesChecksums(t *testing.T) { +func TestLoadProjectJoinCacheFiltersSharedRecordsByDeduplicatedGitChecksums(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), "data/b.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -2094,14 +2356,18 @@ func TestLoadProjectJoinCacheDeduplicatesChecksums(t *testing.T) { backend := &fakeStorageAnalyticsBackend{ projectRecords: []gintegrationsyfon.ProjectRecord{ {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-orphan", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/orphan"}}, }, } service := NewStorageAnalyticsService(backend) if _, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash); err != nil { t.Fatalf("build storage summary: %v", err) } - if len(backend.bulkChecksums) != 1 { - t.Fatalf("expected duplicate file checksums to be deduplicated before backend lookup, got %+v", backend.bulkChecksums) + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to avoid checksum lookup, got %d calls", backend.bulkGetProjectRecordsCalls) + } + if len(backend.listProjectFileUsageObjectIDs) != 1 || backend.listProjectFileUsageObjectIDs[0] != "obj-a" { + t.Fatalf("expected duplicate Git checksums to request one matched object usage row, got %+v", backend.listProjectFileUsageObjectIDs) } } @@ -2435,16 +2701,17 @@ func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) if err != nil { t.Fatalf("build chain audit: %v", err) } - if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { - t.Fatalf("expected mapped bucket match to surface stale access URL repair, got %+v", chain.Summary) + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected mapped bucket match to suppress stale raw URL misclassification, got %+v", chain.Summary) } - if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 0 { - t.Fatalf("expected stale access URL repair to block clean-chain count, got %+v", chain.Summary) + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected mapped bucket match to preserve clean-chain count, got %+v", chain.Summary) } findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) - finding := assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "data/slide.ome.tiff") - if finding.DefaultAction != storageActionRemoveBrokenAccessURLs { - t.Fatalf("expected stale URL finding to default to access method repair, got %+v", finding) + for _, finding := range findings { + if finding.NormalizedPath == "data/slide.ome.tiff" { + t.Fatalf("did not expect stale raw URL to produce a chain finding once mapped object exists: %+v", finding) + } } if len(backend.probeItems) != 1 { t.Fatalf("expected only the canonical scoped probe, got %+v", backend.probeItems) @@ -2521,13 +2788,14 @@ func TestBuildStorageChainAuditCanonicalizesStaleRecordBucketUsingAuditProjectSc if got := chain.Summary.CountsByKind["syfon_missing_bucket_object"]; got != 0 { t.Fatalf("expected stale EllrottLab URL to be canonicalized into the audit project bucket, got summary %+v", chain.Summary) } - if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { - t.Fatalf("expected current project bucket object to surface stale access URL repair, got summary %+v", chain.Summary) + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected current project bucket object to complete the chain, got summary %+v", chain.Summary) } findings := loadAllChainFindings(t, service, "gdc_mirror", "gdc_mirror", chain) - finding := assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "data/file.bin") - if strings.Contains(finding.BucketObjectURL, "EllrottLab") || strings.Contains(finding.NormalizedPath, "EllrottLab") { - t.Fatalf("did not expect stale EllrottLab bucket to appear as resolved target: %+v", finding) + for _, finding := range findings { + if strings.Contains(finding.BucketObjectURL, "EllrottLab") || strings.Contains(finding.NormalizedPath, "EllrottLab") { + t.Fatalf("did not expect stale EllrottLab bucket to appear in findings: %+v", finding) + } } } @@ -2798,7 +3066,7 @@ func TestBuildStorageChainAuditFlagsExactPathMismatchWhenHashExistsElsewhereInBu assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "CONFIG/cbds-BForePC.json") } -func TestClassifyStorageFindingSurfacesStaleRawURLWhenScopedProbeMatches(t *testing.T) { +func TestClassifyStorageFindingSuppressesRawURLFailuresWhenScopedProbeMatches(t *testing.T) { record := projectRecordState{ ProjectRecord: gintegrationsyfon.ProjectRecord{ ObjectID: "obj-a", @@ -2829,19 +3097,12 @@ func TestClassifyStorageFindingSurfacesStaleRawURLWhenScopedProbeMatches(t *test }, } - if got := classifyStorageFinding(record, nil); got != storageFindingBrokenBucketMap { - t.Fatalf("expected scoped matched probe to surface repairable stale URL, got %s", got) - } - repairRecord := repairableBrokenAccessRecord(record) - if len(repairRecord.AccessProbes) != 2 { - t.Fatalf("expected stale raw and scoped replacement probes, got %+v", repairRecord.AccessProbes) - } - if repairRecord.AccessProbes[0].ErrorKind != "stale_scope_mapping" || repairRecord.AccessProbes[1].ObjectURL != "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff" { - t.Fatalf("unexpected repair probes: %+v", repairRecord.AccessProbes) + if got := classifyStorageFinding(record, nil); got != storageFindingNone { + t.Fatalf("expected scoped matched probe to suppress raw URL credential miss, got %s", got) } } -func TestClassifyStorageFindingSurfacesRawURLMismatchWhenScopedProbeMatches(t *testing.T) { +func TestClassifyStorageFindingSuppressesRawURLMismatchWhenScopedProbeMatches(t *testing.T) { record := projectRecordState{ ProjectRecord: gintegrationsyfon.ProjectRecord{ ObjectID: "obj-a", @@ -2872,8 +3133,8 @@ func TestClassifyStorageFindingSurfacesRawURLMismatchWhenScopedProbeMatches(t *t }, } - if got := classifyStorageFinding(record, nil); got != storageFindingBrokenBucketMap { - t.Fatalf("expected scoped matched probe to surface repairable stale URL, got %s", got) + if got := classifyStorageFinding(record, nil); got != storageFindingNone { + t.Fatalf("expected scoped matched probe to suppress raw URL mismatch, got %s", got) } } @@ -2920,42 +3181,15 @@ func TestClassifyStorageFindingSurfacesRepairableBrokenAccessMethod(t *testing.T t.Fatalf("expected repairable broken access method finding, got %s", got) } broken := repairableBrokenAccessRecord(record) - if len(broken.AccessProbes) != 3 || broken.AccessProbes[0].ObjectURL != "s3://retired-bucket/JHU/slide.ome.tiff" || broken.AccessProbes[1].ErrorKind != "stale_scope_mapping" { - t.Fatalf("expected retired, stale, and replacement probes in repair record, got %+v", broken.AccessProbes) + if len(broken.AccessProbes) != 1 || broken.AccessProbes[0].ObjectURL != "s3://retired-bucket/JHU/slide.ome.tiff" { + t.Fatalf("expected only broken access probe in repair record, got %+v", broken.AccessProbes) } remaining, shouldDelete, ok := repairBrokenBucketMappingRecord(broken) if !ok || shouldDelete { t.Fatalf("expected repair update, got ok=%v shouldDelete=%v remaining=%+v", ok, shouldDelete, remaining) } - if len(remaining) != 1 || remaining[0].URL != "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff" { - t.Fatalf("expected working access method to be replaced with scoped URL, got %+v", remaining) - } -} - -func TestRemainingAccessMethodsReplacesStaleScopeMapping(t *testing.T) { - record := GitStorageCleanupRecordAudit{ - ObjectID: "obj-a", - AccessURLs: []string{"s3://bforepc-prod/JHU/slide.ome.tiff"}, - AccessMethods: []GitStorageCleanupAccessMethod{{AccessID: "s3", Type: "s3", URL: "s3://bforepc-prod/JHU/slide.ome.tiff"}}, - AccessProbes: []GitStorageCleanupAccessProbe{ - { - URL: "s3://bforepc-prod/JHU/slide.ome.tiff", - Status: "error", - ErrorKind: "stale_scope_mapping", - }, - { - URL: "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", - Bucket: "bforepc", - Key: "bforepc-prod/JHU/slide.ome.tiff", - Status: "present", - ValidationStatus: "matched", - }, - }, - } - - remaining := remainingAccessMethodsAfterBrokenRemoval(record) - if len(remaining) != 1 || remaining[0].URL != "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff" { - t.Fatalf("expected stale URL replaced with scoped URL, got %+v", remaining) + if len(remaining) != 1 || remaining[0].URL != "s3://bforepc-prod/JHU/slide.ome.tiff" { + t.Fatalf("expected working access method to remain, got %+v", remaining) } } diff --git a/internal/git/types.go b/internal/git/types.go index 17ab650..2b4afcd 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -428,6 +428,8 @@ type GitStorageChainFinding struct { RecordCount int `json:"record_count"` SizeBytes int64 `json:"size_bytes,omitempty"` RecommendedAction string `json:"recommended_action"` + SuggestedFix string `json:"suggested_fix,omitempty"` + SuggestedAction string `json:"suggested_action,omitempty"` Actionability string `json:"actionability,omitempty"` AvailableActions []string `json:"available_actions,omitempty"` DefaultAction string `json:"default_action,omitempty"` @@ -550,6 +552,7 @@ type GitStorageCleanupApplyFinding struct { AccessURLs []string `json:"access_urls,omitempty"` AvailableActions []string `json:"available_actions,omitempty"` DefaultAction string `json:"default_action,omitempty"` + SuggestedAction string `json:"suggested_action,omitempty"` Evidence *GitAuditEvidence `json:"evidence,omitempty"` } diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index 88773d1..b207990 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -354,6 +354,7 @@ func (handler *Handler) handleGitProjectStorageCleanupApplyPOST(ctx fiber.Ctx) e requestBody.Findings[i].AccessURLs = normalizeAnalyticsPathList(requestBody.Findings[i].AccessURLs) requestBody.Findings[i].AvailableActions = normalizeStringList(requestBody.Findings[i].AvailableActions) requestBody.Findings[i].DefaultAction = strings.TrimSpace(requestBody.Findings[i].DefaultAction) + requestBody.Findings[i].SuggestedAction = strings.TrimSpace(requestBody.Findings[i].SuggestedAction) if requestBody.Findings[i].Evidence != nil { requestBody.Findings[i].Evidence.ObjectIDs = normalizeStringList(requestBody.Findings[i].Evidence.ObjectIDs) requestBody.Findings[i].Evidence.AccessURLs = normalizeAnalyticsPathList(requestBody.Findings[i].Evidence.AccessURLs) From 97e425f08bc13aa5b85b4b59337777d5e30a11d3 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 8 Jul 2026 09:30:51 -0700 Subject: [PATCH 26/36] fix build errs --- internal/git/storage_analytics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 5520037..5e66a8c 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -2421,7 +2421,7 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, primaryProbe := selectChainProbe(record, bucketObjectURLs) suggestedFix := suggestedFixForChainFinding(kind, record) suggestedAction := suggestedActionForChainFinding(kind, record) - if suggestedAction != "" && !contains(availableActions, suggestedAction) { + if suggestedAction != "" && !stringSliceContains(availableActions, suggestedAction) { availableActions = append([]string(nil), availableActions...) availableActions = append(availableActions, suggestedAction) } From c452d06b61150c10e7d79bbad5859a21783f418b Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 8 Jul 2026 11:35:31 -0700 Subject: [PATCH 27/36] re-enable http2 --- internal/git/github_file_contents_test.go | 114 ++++++++++++++ internal/git/github_logging.go | 20 +++ internal/git/response.go | 45 +++++- internal/git/service.go | 11 +- internal/httpclient/client.go | 9 +- internal/integrations/github/client.go | 157 +++++++++++++++++++- internal/integrations/github/client_test.go | 96 ++++++++++++ 7 files changed, 441 insertions(+), 11 deletions(-) create mode 100644 internal/git/github_file_contents_test.go create mode 100644 internal/git/github_logging.go create mode 100644 internal/integrations/github/client_test.go diff --git a/internal/git/github_file_contents_test.go b/internal/git/github_file_contents_test.go new file mode 100644 index 0000000..6c7e159 --- /dev/null +++ b/internal/git/github_file_contents_test.go @@ -0,0 +1,114 @@ +package git + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/google/go-github/v87/github" +) + +type gitRoundTripFunc func(*http.Request) (*http.Response, error) + +func (fn gitRoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + +func TestGetGitHubFileContentsUsesInstallationToken(t *testing.T) { + attempts := 0 + client, err := github.NewClient( + github.WithHTTPClient(&http.Client{ + Transport: gitRoundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + if req.Header.Get("Authorization") != "Bearer install-token" { + t.Fatalf("expected installation token auth header, got %q", req.Header.Get("Authorization")) + } + return gitHubJSONResponse(http.StatusOK, `{"type":"file","name":"Practitioner.ndjson","path":"META/Practitioner.ndjson","sha":"abc","size":43,"content":"dmVyc2lvbg==","encoding":"base64"}`), nil + }), + }), + github.WithAuthToken("install-token"), + ) + if err != nil { + t.Fatalf("create github client: %v", err) + } + + metadata, _, err := getGitHubFileContents(context.Background(), client, GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }, "META/Practitioner.ndjson", &github.RepositoryContentGetOptions{Ref: "main"}, "https://api.github.com", true, githubAccessTokenFingerprint("install-token"), githubAccessTokenLength("install-token")) + if err != nil { + t.Fatalf("get github file contents: %v", err) + } + if attempts != 1 { + t.Fatalf("expected one github file contents request, got %d attempts", attempts) + } + if metadata.GetPath() != "META/Practitioner.ndjson" { + t.Fatalf("unexpected metadata path %q", metadata.GetPath()) + } +} + +func TestGetGitHubFileContentsDoesNotRetryEOF(t *testing.T) { + attempts := 0 + client, err := github.NewClient( + github.WithHTTPClient(&http.Client{ + Transport: gitRoundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + return nil, io.EOF + }), + }), + github.WithAuthToken("install-token"), + ) + if err != nil { + t.Fatalf("create github client: %v", err) + } + + _, _, err = getGitHubFileContents(context.Background(), client, GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }, "META/Practitioner.ndjson", &github.RepositoryContentGetOptions{Ref: "main"}, "https://api.github.com", true, githubAccessTokenFingerprint("install-token"), githubAccessTokenLength("install-token")) + if err == nil { + t.Fatal("expected EOF error") + } + if attempts != 1 { + t.Fatalf("expected no retry after EOF, got %d attempts", attempts) + } +} + +func TestGetGitHubFileContentsDoesNotRetryUnauthorized(t *testing.T) { + attempts := 0 + client, err := github.NewClient( + github.WithHTTPClient(&http.Client{ + Transport: gitRoundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + return gitHubJSONResponse(http.StatusUnauthorized, `{"message":"Bad credentials"}`), nil + }), + }), + github.WithAuthToken("bad-token"), + ) + if err != nil { + t.Fatalf("create github client: %v", err) + } + + _, _, err = getGitHubFileContents(context.Background(), client, GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }, "META/Practitioner.ndjson", &github.RepositoryContentGetOptions{Ref: "main"}, "https://api.github.com", true, githubAccessTokenFingerprint("bad-token"), githubAccessTokenLength("bad-token")) + if err == nil { + t.Fatal("expected unauthorized error") + } + if attempts != 1 { + t.Fatalf("expected no retry for unauthorized response, got %d attempts", attempts) + } +} + +func gitHubJSONResponse(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(body)), + } +} diff --git a/internal/git/github_logging.go b/internal/git/github_logging.go new file mode 100644 index 0000000..ddf7e30 --- /dev/null +++ b/internal/git/github_logging.go @@ -0,0 +1,20 @@ +package git + +import ( + "crypto/sha256" + "encoding/hex" + "strings" +) + +func githubAccessTokenFingerprint(accessToken string) string { + accessToken = strings.TrimSpace(accessToken) + if accessToken == "" { + return "" + } + sum := sha256.Sum256([]byte(accessToken)) + return hex.EncodeToString(sum[:])[:16] +} + +func githubAccessTokenLength(accessToken string) int { + return len(strings.TrimSpace(accessToken)) +} diff --git a/internal/git/response.go b/internal/git/response.go index e965a6d..2df659e 100644 --- a/internal/git/response.go +++ b/internal/git/response.go @@ -6,10 +6,12 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" "path/filepath" "sort" "strings" + "time" servermw "github.com/calypr/gecko/internal/server/middleware" gogit "github.com/go-git/go-git/v5" @@ -362,7 +364,7 @@ func (service *GitService) GetGitHubFileMetadata(ctx context.Context, authorizat if strings.TrimSpace(ref) != "" { opts.Ref = strings.TrimSpace(ref) } - metadata, _, response, err := client.Repositories.GetContents(ctx, identity.Owner, identity.Repo, path, opts) + metadata, response, err := getGitHubFileContents(ctx, client, identity, path, opts, strings.TrimRight(service.config.GitHubAPIBase, "/"), strings.TrimSpace(accessToken) != "", githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken)) if err != nil { statusCode := http.StatusBadGateway if response != nil && response.StatusCode > 0 { @@ -397,3 +399,44 @@ func (service *GitService) GetGitHubFileMetadata(ctx context.Context, authorizat } return metadata, []byte(contentString), nil } + +func getGitHubFileContents(ctx context.Context, client *github.Client, identity GitRepositoryIdentity, path string, opts *github.RepositoryContentGetOptions, apiBase string, authConfigured bool, tokenFingerprint string, tokenLength int) (*github.RepositoryContent, *github.Response, error) { + ref := "" + if opts != nil { + ref = strings.TrimSpace(opts.Ref) + } + started := time.Now() + requestURL := githubContentsRequestURL(apiBase, identity, path, ref) + log.Printf("INFO: github_file_contents_request_start owner=%s repo=%s path=%q ref=%q request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d", identity.Owner, identity.Repo, path, ref, requestURL, authConfigured, tokenFingerprint, tokenLength) + metadata, _, response, err := client.Repositories.GetContents(ctx, identity.Owner, identity.Repo, path, opts) + statusCode := 0 + rateLimitRemaining := -1 + rateLimitReset := "" + if response != nil { + rateLimitRemaining = response.Rate.Remaining + if !response.Rate.Reset.Time.IsZero() { + rateLimitReset = response.Rate.Reset.Time.UTC().Format(time.RFC3339) + } + if response.Response != nil { + statusCode = response.Response.StatusCode + } + } + if err == nil { + log.Printf("INFO: github_file_contents_request_done owner=%s repo=%s path=%q ref=%q request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%d rate_limit_reset=%q duration_ms=%d", identity.Owner, identity.Repo, path, ref, requestURL, authConfigured, tokenFingerprint, tokenLength, statusCode, rateLimitRemaining, rateLimitReset, time.Since(started).Milliseconds()) + return metadata, response, nil + } + log.Printf("INFO: github_file_contents_request_done owner=%s repo=%s path=%q ref=%q request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%d rate_limit_reset=%q duration_ms=%d error_type=%T error=%q", identity.Owner, identity.Repo, path, ref, requestURL, authConfigured, tokenFingerprint, tokenLength, statusCode, rateLimitRemaining, rateLimitReset, time.Since(started).Milliseconds(), err, err.Error()) + return nil, response, err +} + +func githubContentsRequestURL(apiBase string, identity GitRepositoryIdentity, path string, ref string) string { + if strings.TrimSpace(apiBase) == "" { + apiBase = "https://api.github.com" + } + base := strings.TrimRight(apiBase, "/") + requestURL := fmt.Sprintf("%s/repos/%s/%s/contents/%s", base, identity.Owner, identity.Repo, strings.TrimLeft(path, "/")) + if ref != "" { + requestURL += "?ref=" + ref + } + return requestURL +} diff --git a/internal/git/service.go b/internal/git/service.go index d284813..ed79b20 100644 --- a/internal/git/service.go +++ b/internal/git/service.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "log" "os" "path/filepath" "strings" @@ -234,5 +235,13 @@ func (service *GitService) RequestInstallationToken(ctx context.Context, authori if service.fenceAPI == nil { return "", fmt.Errorf("fence client is not initialized") } - return service.fenceAPI.RequestInstallationToken(ctx, authorizationHeader, organization, project, identity, access) + started := time.Now() + log.Printf("INFO: github_installation_token_request_start organization=%s project=%s owner=%s repo=%s access=%s", organization, project, identity.Owner, identity.Repo, access) + token, err := service.fenceAPI.RequestInstallationToken(ctx, authorizationHeader, organization, project, identity, access) + if err != nil { + log.Printf("INFO: github_installation_token_request_done organization=%s project=%s owner=%s repo=%s access=%s token_received=false duration_ms=%d error=%q", organization, project, identity.Owner, identity.Repo, access, time.Since(started).Milliseconds(), err.Error()) + return "", err + } + log.Printf("INFO: github_installation_token_request_done organization=%s project=%s owner=%s repo=%s access=%s token_received=%t token_fingerprint=%s token_length=%d duration_ms=%d", organization, project, identity.Owner, identity.Repo, access, strings.TrimSpace(token) != "", githubAccessTokenFingerprint(token), githubAccessTokenLength(token), time.Since(started).Milliseconds()) + return token, nil } diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go index 441eb0b..0b5f954 100644 --- a/internal/httpclient/client.go +++ b/internal/httpclient/client.go @@ -1,20 +1,15 @@ package httpclient import ( - "crypto/tls" "net" "net/http" "time" ) -// NewServiceClient returns a default outbound service client with HTTP/2 -// disabled. Gecko mostly talks to in-cluster services over HTTP/1.1, and this -// avoids process-fatal panics in Go's HTTP/2 HPACK encoder under concurrent -// service traffic. +// NewServiceClient returns a default outbound service client with normal Go +// transport behavior, including HTTP/2 when the peer supports it. func NewServiceClient(timeout time.Duration) *http.Client { transport := http.DefaultTransport.(*http.Transport).Clone() - transport.ForceAttemptHTTP2 = false - transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} transport.DialContext = (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, diff --git a/internal/integrations/github/client.go b/internal/integrations/github/client.go index 2838222..47f6c62 100644 --- a/internal/integrations/github/client.go +++ b/internal/integrations/github/client.go @@ -2,8 +2,15 @@ package github import ( "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" "fmt" + "io" + "log" + "net" "net/http" + "net/url" "strings" "time" @@ -13,7 +20,8 @@ import ( ) type Config struct { - APIBase string + APIBase string + DisableTransportDiagnostics bool } type Client struct { @@ -41,7 +49,7 @@ func (c *Client) FetchRepositoryMetadata(ctx context.Context, accessToken string if err != nil { return nil, err } - repo, _, err := githubClient.Repositories.Get(ctx, identity.Owner, identity.Repo) + repo, err := c.fetchRepositoryMetadata(ctx, githubClient, identity, accessToken) if err != nil { return nil, fmt.Errorf("github repository metadata lookup failed for %s/%s: %w", identity.Owner, identity.Repo, err) } @@ -53,6 +61,151 @@ func (c *Client) FetchRepositoryMetadata(ctx context.Context, accessToken string return &domain.GitHubRepositoryMetadata{DefaultBranch: defaultBranch, HTMLURL: htmlURL}, nil } +func (c *Client) fetchRepositoryMetadata(ctx context.Context, client *google_github.Client, identity domain.GitRepositoryIdentity, accessToken string) (*google_github.Repository, error) { + started := time.Now() + requestURL := c.repositoryMetadataRequestURL(identity) + authConfigured := strings.TrimSpace(accessToken) != "" + tokenFingerprint := githubAccessTokenFingerprint(accessToken) + tokenLength := githubAccessTokenLength(accessToken) + log.Printf("INFO: github_repository_metadata_request_start owner=%s repo=%s request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d", identity.Owner, identity.Repo, requestURL, authConfigured, tokenFingerprint, tokenLength) + repo, response, err := client.Repositories.Get(ctx, identity.Owner, identity.Repo) + statusCode := 0 + rateLimitRemaining := -1 + rateLimitReset := "" + if response != nil { + rateLimitRemaining = response.Rate.Remaining + if !response.Rate.Reset.Time.IsZero() { + rateLimitReset = response.Rate.Reset.Time.UTC().Format(time.RFC3339) + } + if response.Response != nil { + statusCode = response.Response.StatusCode + } + } + if err == nil { + log.Printf("INFO: github_repository_metadata_request_done owner=%s repo=%s request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%d rate_limit_reset=%q duration_ms=%d", identity.Owner, identity.Repo, requestURL, authConfigured, tokenFingerprint, tokenLength, statusCode, rateLimitRemaining, rateLimitReset, time.Since(started).Milliseconds()) + return repo, nil + } + log.Printf("INFO: github_repository_metadata_request_done owner=%s repo=%s request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%d rate_limit_reset=%q duration_ms=%d error_type=%T error=%q", identity.Owner, identity.Repo, requestURL, authConfigured, tokenFingerprint, tokenLength, statusCode, rateLimitRemaining, rateLimitReset, time.Since(started).Milliseconds(), err, err.Error()) + if response == nil { + c.logTransportDiagnostic(ctx, accessToken, err) + } + return nil, err +} + +func (c *Client) repositoryMetadataRequestURL(identity domain.GitRepositoryIdentity) string { + apiBase := strings.TrimRight(c.config.APIBase, "/") + if apiBase == "" { + apiBase = "https://api.github.com" + } + return fmt.Sprintf("%s/repos/%s/%s", apiBase, identity.Owner, identity.Repo) +} + +func githubAccessTokenFingerprint(accessToken string) string { + accessToken = strings.TrimSpace(accessToken) + if accessToken == "" { + return "" + } + sum := sha256.Sum256([]byte(accessToken)) + return hex.EncodeToString(sum[:])[:16] +} + +func githubAccessTokenLength(accessToken string) int { + return len(strings.TrimSpace(accessToken)) +} + +func (c *Client) logTransportDiagnostic(parent context.Context, accessToken string, originalErr error) { + if c.config.DisableTransportDiagnostics { + return + } + apiBase := strings.TrimRight(c.config.APIBase, "/") + if apiBase == "" { + apiBase = "https://api.github.com" + } + parsed, err := url.Parse(apiBase) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_done api_base=%q token_fingerprint=%s token_length=%d original_error_type=%T original_error=%q diagnostic_error=%q", apiBase, githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), originalErr, originalErr.Error(), err.Error()) + return + } + host := parsed.Hostname() + port := parsed.Port() + if port == "" { + if parsed.Scheme == "http" { + port = "80" + } else { + port = "443" + } + } + address := net.JoinHostPort(host, port) + log.Printf("INFO: github_transport_diagnostic_start api_base=%q host=%s port=%s scheme=%s token_fingerprint=%s token_length=%d original_error_type=%T original_error=%q", apiBase, host, port, parsed.Scheme, githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), originalErr, originalErr.Error()) + + diagCtx, cancel := context.WithTimeout(parent, 15*time.Second) + defer cancel() + c.logDNSDiagnostic(diagCtx, host) + c.logTCPDiagnostic(diagCtx, address) + if parsed.Scheme == "https" { + c.logTLSDiagnostic(host, address) + } + c.logHTTPDiagnostic(diagCtx, apiBase, accessToken) +} + +func (c *Client) logDNSDiagnostic(ctx context.Context, host string) { + started := time.Now() + ips, err := net.DefaultResolver.LookupHost(ctx, host) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=dns host=%s duration_ms=%d error_type=%T error=%q", host, time.Since(started).Milliseconds(), err, err.Error()) + return + } + log.Printf("INFO: github_transport_diagnostic_stage stage=dns host=%s duration_ms=%d ips=%q", host, time.Since(started).Milliseconds(), strings.Join(ips, ",")) +} + +func (c *Client) logTCPDiagnostic(ctx context.Context, address string) { + started := time.Now() + conn, err := (&net.Dialer{Timeout: 5 * time.Second}).DialContext(ctx, "tcp", address) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=tcp address=%s duration_ms=%d error_type=%T error=%q", address, time.Since(started).Milliseconds(), err, err.Error()) + return + } + remote := conn.RemoteAddr().String() + local := conn.LocalAddr().String() + _ = conn.Close() + log.Printf("INFO: github_transport_diagnostic_stage stage=tcp address=%s local_addr=%s remote_addr=%s duration_ms=%d", address, local, remote, time.Since(started).Milliseconds()) +} + +func (c *Client) logTLSDiagnostic(host string, address string) { + started := time.Now() + dialer := &net.Dialer{Timeout: 5 * time.Second} + conn, err := tls.DialWithDialer(dialer, "tcp", address, &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12}) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=tls host=%s address=%s duration_ms=%d error_type=%T error=%q", host, address, time.Since(started).Milliseconds(), err, err.Error()) + return + } + state := conn.ConnectionState() + _ = conn.Close() + log.Printf("INFO: github_transport_diagnostic_stage stage=tls host=%s address=%s version=%x cipher_suite=%x server_name=%s duration_ms=%d", host, address, state.Version, state.CipherSuite, state.ServerName, time.Since(started).Milliseconds()) +} + +func (c *Client) logHTTPDiagnostic(ctx context.Context, apiBase string, accessToken string) { + started := time.Now() + requestURL := strings.TrimRight(apiBase, "/") + "/rate_limit" + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=http request_url=%q token_fingerprint=%s token_length=%d duration_ms=%d error_type=%T error=%q", requestURL, githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), time.Since(started).Milliseconds(), err, err.Error()) + return + } + if strings.TrimSpace(accessToken) != "" { + request.Header.Set("Authorization", "Bearer "+strings.TrimSpace(accessToken)) + } + request.Header.Set("Accept", "application/vnd.github+json") + response, err := c.client.Do(request) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=http request_url=%q auth_configured=%t token_fingerprint=%s token_length=%d duration_ms=%d error_type=%T error=%q", requestURL, strings.TrimSpace(accessToken) != "", githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), time.Since(started).Milliseconds(), err, err.Error()) + return + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1024)) + log.Printf("INFO: github_transport_diagnostic_stage stage=http request_url=%q auth_configured=%t token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%q rate_limit_reset=%q duration_ms=%d", requestURL, strings.TrimSpace(accessToken) != "", githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), response.StatusCode, response.Header.Get("X-RateLimit-Remaining"), response.Header.Get("X-RateLimit-Reset"), time.Since(started).Milliseconds()) +} + func (c *Client) githubClient(accessToken string) (*google_github.Client, error) { options := []google_github.ClientOptionsFunc{ google_github.WithAuthToken(accessToken), diff --git a/internal/integrations/github/client_test.go b/internal/integrations/github/client_test.go new file mode 100644 index 0000000..0812ab0 --- /dev/null +++ b/internal/integrations/github/client_test.go @@ -0,0 +1,96 @@ +package github + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/calypr/gecko/internal/git/domain" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + +func TestFetchRepositoryMetadataUsesInstallationToken(t *testing.T) { + attempts := 0 + client := NewClient(&http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + if req.Header.Get("Authorization") != "Bearer install-token" { + t.Fatalf("expected installation token auth header, got %q", req.Header.Get("Authorization")) + } + return githubJSONResponse(http.StatusOK, `{"default_branch":"main","html_url":"https://github.com/BForePC/BForePC"}`), nil + }), + }, Config{}) + + metadata, err := client.FetchRepositoryMetadata(context.Background(), "install-token", domain.GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }) + if err != nil { + t.Fatalf("fetch repository metadata: %v", err) + } + if attempts != 1 { + t.Fatalf("expected one metadata request, got %d attempts", attempts) + } + if metadata.DefaultBranch != "main" || metadata.HTMLURL != "https://github.com/BForePC/BForePC" { + t.Fatalf("unexpected metadata: %+v", metadata) + } +} + +func TestFetchRepositoryMetadataDoesNotRetryEOF(t *testing.T) { + attempts := 0 + client := NewClient(&http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + return nil, io.EOF + }), + }, Config{DisableTransportDiagnostics: true}) + + _, err := client.FetchRepositoryMetadata(context.Background(), "install-token", domain.GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }) + if err == nil { + t.Fatal("expected EOF error") + } + if attempts != 1 { + t.Fatalf("expected no retry after EOF, got %d attempts", attempts) + } +} + +func TestFetchRepositoryMetadataDoesNotRetryUnauthorized(t *testing.T) { + attempts := 0 + client := NewClient(&http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + return githubJSONResponse(http.StatusUnauthorized, `{"message":"Bad credentials"}`), nil + }), + }, Config{}) + + _, err := client.FetchRepositoryMetadata(context.Background(), "bad-token", domain.GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }) + if err == nil { + t.Fatal("expected unauthorized error") + } + if attempts != 1 { + t.Fatalf("expected no retry for unauthorized response, got %d attempts", attempts) + } +} + +func githubJSONResponse(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(body)), + } +} From 0f34f556334741fccb56e3cba0ca6cea72070786 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 8 Jul 2026 15:59:26 -0700 Subject: [PATCH 28/36] support redis caching for audits --- go.mod | 3 + go.sum | 4 + internal/git/storage_analytics.go | 421 +++++++++++++++--- internal/git/storage_analytics_pipeline.go | 21 +- internal/git/storage_analytics_test.go | 228 +++++++--- internal/git/storage_analytics_timing.go | 1 + internal/git/storage_chain_audit_cache.go | 364 +++++++++++++++ .../git/storage_chain_audit_cache_test.go | 19 + internal/git/types.go | 8 + internal/server/http/git/handler.go | 1 + internal/server/http/git/storage_analytics.go | 14 +- 11 files changed, 966 insertions(+), 118 deletions(-) create mode 100644 internal/git/storage_chain_audit_cache.go create mode 100644 internal/git/storage_chain_audit_cache_test.go diff --git a/go.mod b/go.mod index 4a2b032..676e177 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 github.com/qdrant/go-client v1.18.1 + github.com/redis/go-redis/v9 v9.17.1 github.com/stretchr/testify v1.11.1 github.com/swaggo/swag v1.16.6 github.com/uc-cdis/arborist v0.0.0-20260324212054-708d91019bea @@ -45,10 +46,12 @@ require ( github.com/bytedance/sonic v1.15.1 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/calypr/syfon v0.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/cloudwego/base64x v0.1.7 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect diff --git a/go.sum b/go.sum index 6d6c2f6..8ff836d 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -215,6 +217,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/qdrant/go-client v1.18.1 h1:o/dDmSl6ONAlaAFtjdlzztcs3NH0tJY3l5C/z/Uu0bE= github.com/qdrant/go-client v1.18.1/go.mod h1:Xkfp+r89uNOgSbvilVAhCZ3wKI4G+hB/r9Zr2m4zifI= +github.com/redis/go-redis/v9 v9.17.1 h1:7tl732FjYPRT9H9aNfyTwKg9iTETjWjGKEJ2t/5iWTs= +github.com/redis/go-redis/v9 v9.17.1/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 5e66a8c..e1130e3 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -109,14 +109,16 @@ type storageAnalyticsBackend interface { } type StorageAnalyticsService struct { - storage storageAnalyticsBackend - projectJoinMu sync.RWMutex - projectJoinCache map[string]cachedProjectJoinState - projectJoinWork map[string]*inflightProjectJoinState - chainInputMu sync.RWMutex - chainInputCache map[string]cachedChainInputState - projectAuditCache map[string]cachedProjectAuditRecordState - projectAuditWork map[string]*inflightProjectAuditRecordState + storage storageAnalyticsBackend + projectJoinMu sync.RWMutex + projectJoinCache map[string]cachedProjectJoinState + projectJoinWork map[string]*inflightProjectJoinState + chainInputMu sync.RWMutex + chainInputCache map[string]cachedChainInputState + projectAuditCache map[string]cachedProjectAuditRecordState + projectAuditWork map[string]*inflightProjectAuditRecordState + chainAuditResponseCache storageChainAuditResponseCache + exactProjectJoinCache storageExactProjectJoinCache } type StorageFolderTimings struct { @@ -145,6 +147,14 @@ func NewStorageAnalyticsService(storage storageAnalyticsBackend) *StorageAnalyti } } +func (service *StorageAnalyticsService) EnableStorageChainAuditResponseCacheFromEnv() { + if service == nil { + return + } + service.chainAuditResponseCache = NewStorageChainAuditResponseCacheFromEnv() + service.exactProjectJoinCache = NewStorageExactProjectJoinCacheFromEnv() +} + type RepoInventoryFile struct { RepoPath string Name string @@ -264,7 +274,7 @@ func BuildGitRepoInventory(ref string, gitSubpath string, repo *gogit.Repository } func (service *StorageAnalyticsService) BuildStorageSummary(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitStorageSummaryResponse, error) { - index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, false) if err != nil { return nil, err } @@ -313,10 +323,10 @@ func (service *StorageAnalyticsService) BuildStorageChildren(ctx context.Context }, nil } -func (service *StorageAnalyticsService) BuildStorageFolder(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string, cursor string, summaryMode string, timings *StorageFolderTimings) (*GitStorageFolderResponse, error) { +func (service *StorageAnalyticsService) BuildStorageFolder(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string, cursor string, summaryMode string, forceRefresh bool, timings *StorageFolderTimings) (*GitStorageFolderResponse, error) { if strings.EqualFold(strings.TrimSpace(summaryMode), StorageFolderSummaryModeExact) { exactStart := time.Now() - index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, forceRefresh) timings.Record("exact_join", time.Since(exactStart)) if err != nil { return nil, err @@ -394,7 +404,7 @@ func (service *StorageAnalyticsService) BuildStorageFolder(ctx context.Context, } func (service *StorageAnalyticsService) BuildProjectDiffAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitProjectDiffAuditResponse, error) { - _, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + _, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, false) if err != nil { return nil, err } @@ -438,13 +448,44 @@ func (service *StorageAnalyticsService) BuildStorageChainAudit(ctx context.Conte } func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, error) { + normalized, err := normalizeStorageChainAuditOptions(options) + if err != nil { + return nil, err + } + if service.chainAuditResponseCache != nil && storageChainAuditResponseCacheAllowed(gitSubpath, normalized) { + return service.buildStorageChainAuditWithResponseCache(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, normalized) + } + if service.chainAuditResponseCache != nil && storageChainAuditRootResponseProjectionAllowed(gitSubpath, normalized) { + response, ok, err := service.projectStorageChainAuditFromRootResponseCache(ctx, organization, project, ref, gitSubpath, mirrorPath, repo, hash, normalized) + if err != nil { + return nil, err + } + if ok { + return response, nil + } + } + if service.chainAuditResponseCache != nil && normalized.Timings != nil { + normalized.Timings.Record("audit_response_cache_bypass_non_root", 0) + } + return service.buildStorageChainAuditFresh(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, normalized) +} + +func storageChainAuditResponseCacheAllowed(gitSubpath string, options StorageChainAuditOptions) bool { + return normalizeRepoSubpath(gitSubpath) == "" && normalizeRepoSubpath(options.BucketPathPrefix) == "" +} + +func storageChainAuditRootResponseProjectionAllowed(gitSubpath string, options StorageChainAuditOptions) bool { + return normalizeRepoSubpath(gitSubpath) != "" && normalizeRepoSubpath(options.BucketPathPrefix) == "" && !options.ForceAuditRefresh +} + +func normalizeStorageChainAuditOptions(options StorageChainAuditOptions) (StorageChainAuditOptions, error) { probeMode, ok := NormalizeStorageChainProbeMode(options.ProbeMode) if !ok { - return nil, fmt.Errorf("invalid storage chain probe mode %q", options.ProbeMode) + return StorageChainAuditOptions{}, fmt.Errorf("invalid storage chain probe mode %q", options.ProbeMode) } bucketMode, ok := NormalizeStorageChainBucketInventoryMode(options.BucketInventoryMode) if !ok { - return nil, fmt.Errorf("invalid storage chain bucket inventory mode %q", options.BucketInventoryMode) + return StorageChainAuditOptions{}, fmt.Errorf("invalid storage chain bucket inventory mode %q", options.BucketInventoryMode) } validationMode := strings.TrimSpace(options.ValidationMode) if validationMode == "" { @@ -456,9 +497,101 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co } validationMode, ok = NormalizeStorageChainValidationMode(validationMode) if !ok { - return nil, fmt.Errorf("invalid storage chain validation mode %q", options.ValidationMode) + return StorageChainAuditOptions{}, fmt.Errorf("invalid storage chain validation mode %q", options.ValidationMode) + } + options.ProbeMode = probeMode + options.ValidationMode = validationMode + options.BucketInventoryMode = bucketMode + options.BucketPathPrefix = normalizeRepoSubpath(options.BucketPathPrefix) + options.FindingKind = strings.TrimSpace(options.FindingKind) + return options, nil +} + +func (service *StorageAnalyticsService) buildStorageChainAuditWithResponseCache(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, error) { + cacheKey := storageChainAuditResponseCacheKey(organization, project, ref, gitSubpath, options.ProbeMode, options.ValidationMode, options.BucketInventoryMode, options.BucketPathPrefix, hash.String()) + cache := service.chainAuditResponseCache + if !options.ForceAuditRefresh { + start := time.Now() + cached, ok, err := cache.Get(ctx, cacheKey) + options.Timings.Record("audit_response_cache_lookup", time.Since(start)) + if err != nil { + logStorageChainAuditCacheError(options.Timings, cache.Source(), "get", err) + } + if ok { + response := projectStorageChainAuditResponse(cached.Response, options.FindingKind, options.FindingLimit) + applyStorageChainAuditCacheMetadata(response, true, cached.CachedAt, cache.Source(), "") + options.Timings.Record("audit_response_cache_hit", 0) + options.Timings.RecordMemory( + "audit_response_cache_hit", + "total_findings", response.Summary.TotalFindings, + "returned_findings", response.Summary.ReturnedFindings, + "finding_limit", options.FindingLimit, + ) + return response, nil + } + options.Timings.Record("audit_response_cache_miss", 0) + } else { + options.Timings.Record("audit_response_cache_force_refresh", 0) + } + + buildOptions := options + buildOptions.FindingKind = "" + buildOptions.FindingLimit = -1 + response, err := service.buildStorageChainAuditFresh(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, buildOptions) + if err != nil { + return nil, err + } + cachedAt := time.Now() + cacheStart := time.Now() + setErr := cache.Set(ctx, cacheKey, cachedStorageChainAuditResponse{CachedAt: cachedAt, Response: *response}, storageChainAuditCacheTTL()) + options.Timings.Record("audit_response_cache_store", time.Since(cacheStart)) + cacheError := "" + if setErr != nil { + cacheError = setErr.Error() + logStorageChainAuditCacheError(options.Timings, cache.Source(), "set", setErr) + } + projected := projectStorageChainAuditResponse(*response, options.FindingKind, options.FindingLimit) + applyStorageChainAuditCacheMetadata(projected, false, cachedAt, cache.Source(), cacheError) + return projected, nil +} + +func (service *StorageAnalyticsService) projectStorageChainAuditFromRootResponseCache(ctx context.Context, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, bool, error) { + cache := service.chainAuditResponseCache + cacheKey := storageChainAuditResponseCacheKey(organization, project, ref, "", options.ProbeMode, options.ValidationMode, options.BucketInventoryMode, "", hash.String()) + start := time.Now() + cached, ok, err := cache.Get(ctx, cacheKey) + options.Timings.Record("audit_response_root_cache_lookup", time.Since(start)) + if err != nil { + logStorageChainAuditCacheError(options.Timings, cache.Source(), "get_root", err) + return nil, false, nil + } + if !ok { + options.Timings.Record("audit_response_root_cache_miss", 0) + return nil, false, nil + } + inventoryStart := time.Now() + inventory, err := service.loadStorageChainInventory(ctx, ref, gitSubpath, mirrorPath, repo, hash) + options.Timings.Record("audit_response_root_cache_project_inventory", time.Since(inventoryStart)) + if err != nil { + return nil, false, err } - bucketPathPrefix := normalizeRepoSubpath(options.BucketPathPrefix) + response := projectStorageChainAuditResponseForSubpath(cached.Response, gitSubpath, inventory, options.FindingKind, options.FindingLimit) + applyStorageChainAuditCacheMetadata(response, true, cached.CachedAt, cache.Source()+":root", "") + options.Timings.Record("audit_response_root_cache_hit", 0) + options.Timings.RecordMemory( + "audit_response_root_cache_hit", + "git_subpath", normalizeRepoSubpath(gitSubpath), + "git_files", response.Summary.GitTrackedFileCount, + "total_findings", response.Summary.TotalFindings, + "returned_findings", response.Summary.ReturnedFindings, + ) + return response, true, nil +} + +func (service *StorageAnalyticsService) buildStorageChainAuditFresh(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, error) { + bucketMode := options.BucketInventoryMode + validationMode := options.ValidationMode + bucketPathPrefix := options.BucketPathPrefix start := time.Now() options.Timings.StageStart("chain_setup_total") inputs, err := service.loadStorageChainInputs(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, bucketMode, bucketPathPrefix, options.Timings) @@ -476,7 +609,7 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co } storageViewStart := time.Now() options.Timings.StageStart("storage_view") - storageView, err := service.buildStorageChainView(ctx, authorizationHeader, organization, project, inputs.recordSet, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.Timings) + storageView, err := service.buildStorageChainView(ctx, authorizationHeader, organization, project, inputs.recordSet, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.ForceAuditRefresh, options.Timings) options.Timings.Record("storage_view", time.Since(storageViewStart)) if storageView != nil { options.Timings.RecordMemory( @@ -531,6 +664,158 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co }, nil } +func projectStorageChainAuditResponseForSubpath(base GitStorageChainAuditResponse, gitSubpath string, inventory []RepoInventoryFile, findingKind string, findingLimit int) *GitStorageChainAuditResponse { + pathPrefix := normalizeRepoSubpath(gitSubpath) + filteredFindings := make([]GitStorageChainFinding, 0, len(base.Findings)) + for _, finding := range base.Findings { + if storageChainFindingMatchesSubpath(finding, pathPrefix) { + filteredFindings = append(filteredFindings, finding) + } + } + + summary := base.Summary + countsByKind := make(map[string]int, len(summary.CountsByKind)) + for kind := range summary.CountsByKind { + countsByKind[kind] = 0 + } + issueGitPaths := make(map[string]struct{}) + objectIDs := make(map[string]struct{}) + bucketObjects := make(map[string]struct{}) + for _, finding := range filteredFindings { + countsByKind[finding.Kind]++ + if storageChainFindingHasGitPath(finding.Kind) { + for _, sourcePath := range finding.SourcePaths { + if storagePathMatchesRepoSubpath(sourcePath, pathPrefix) && !strings.Contains(sourcePath, "://") { + issueGitPaths[normalizeRepoSubpath(sourcePath)] = struct{}{} + } + } + if len(finding.SourcePaths) == 0 && storagePathMatchesRepoSubpath(finding.NormalizedPath, pathPrefix) && !strings.Contains(finding.NormalizedPath, "://") { + issueGitPaths[normalizeRepoSubpath(finding.NormalizedPath)] = struct{}{} + } + } + for _, objectID := range finding.ObjectIDs { + if trimmed := strings.TrimSpace(objectID); trimmed != "" { + objectIDs[trimmed] = struct{}{} + } + } + if trimmed := strings.TrimSpace(finding.BucketObjectURL); trimmed != "" { + bucketObjects[trimmed] = struct{}{} + } + for _, accessURL := range finding.AccessURLs { + if trimmed := strings.TrimSpace(accessURL); trimmed != "" { + bucketObjects[trimmed] = struct{}{} + } + } + } + gitTrackedFileCount := len(inventory) + completeCount := gitTrackedFileCount - len(issueGitPaths) + if completeCount < 0 { + completeCount = 0 + } + countsByKind["bucket_syfon_git_complete"] = completeCount + summary.CountsByKind = countsByKind + summary.TotalFindings = len(filteredFindings) + summary.GitTrackedFileCount = gitTrackedFileCount + summary.SyfonRecordCount = completeCount + len(objectIDs) + summary.BucketObjectCount = completeCount + len(bucketObjects) + + projected := base + projected.Findings = filteredFindings + projected.Groups = summarizeChainIssueGroups(filteredFindings) + projected.Summary = summary + projected.PathPrefix = pathPrefix + projected.BucketPathPrefix = "" + return projectStorageChainAuditResponse(projected, findingKind, findingLimit) +} + +func storageChainFindingHasGitPath(kind string) bool { + switch strings.TrimSpace(kind) { + case "git_only_no_syfon", "syfon_git_no_bucket", "git_syfon_metadata_mismatch", "probe_error": + return true + } + return false +} + +func projectStorageChainAuditResponse(base GitStorageChainAuditResponse, findingKind string, findingLimit int) *GitStorageChainAuditResponse { + response := cloneStorageChainAuditResponse(base) + filteredFindings := filterStorageChainFindingsByKind(response.Findings, findingKind) + summary := filterStorageChainSummary(response.Summary, filteredFindings) + findings := limitStorageChainFindings(filteredFindings, findingLimit, findingKind) + summary.ReturnedFindings = len(findings) + summary.FindingLimit = findingLimit + summary.FindingsTruncated = len(findings) < len(filteredFindings) + response.Findings = findings + response.Groups = summarizeChainIssueGroups(filteredFindings) + response.Summary = summary + return &response +} + +func storageChainFindingMatchesSubpath(finding GitStorageChainFinding, pathPrefix string) bool { + if pathPrefix == "" { + return true + } + candidates := []string{finding.NormalizedPath, finding.BucketObjectURL, finding.ResolvedKey} + candidates = append(candidates, finding.SourcePaths...) + candidates = append(candidates, finding.AccessURLs...) + for _, record := range finding.Records { + candidates = append(candidates, record.NormalizedPath) + candidates = append(candidates, record.AccessURLs...) + for _, accessMethod := range record.AccessMethods { + candidates = append(candidates, accessMethod.URL) + } + for _, probe := range record.AccessProbes { + candidates = append(candidates, probe.URL, probe.Path, probe.Key) + } + } + if finding.Evidence != nil { + candidates = append(candidates, finding.Evidence.SourcePaths...) + candidates = append(candidates, finding.Evidence.AccessURLs...) + candidates = append(candidates, finding.Evidence.BucketObjectURLs...) + candidates = append(candidates, finding.Evidence.Keys...) + } + for _, candidate := range candidates { + if storagePathMatchesRepoSubpath(candidate, pathPrefix) { + return true + } + } + return false +} + +func storagePathMatchesRepoSubpath(raw string, pathPrefix string) bool { + pathPrefix = normalizeRepoSubpath(pathPrefix) + if pathPrefix == "" { + return true + } + value := strings.TrimSpace(raw) + if value == "" { + return false + } + if _, key, ok := parseStorageURL(value); ok { + key = normalizeRepoSubpath(key) + return key == pathPrefix || strings.HasPrefix(key, pathPrefix+"/") || strings.Contains("/"+key, "/"+pathPrefix+"/") + } + normalized := normalizeRepoSubpath(value) + return normalized == pathPrefix || strings.HasPrefix(normalized, pathPrefix+"/") || strings.Contains("/"+normalized, "/"+pathPrefix+"/") +} + +func logStorageChainAuditCacheError(timings *StorageChainAuditTimings, source string, operation string, err error) { + if timings == nil || timings.Logf == nil || err == nil { + return + } + timings.Logf("storage_chain_audit_cache_error %s source=%s operation=%s error=%q", strings.TrimSpace(timings.DebugPrefix), strings.TrimSpace(source), strings.TrimSpace(operation), err.Error()) +} + +func applyStorageChainAuditCacheMetadata(response *GitStorageChainAuditResponse, hit bool, cachedAt time.Time, source string, cacheError string) { + if response == nil || cachedAt.IsZero() { + return + } + response.Summary.AuditCacheHit = hit + response.Summary.AuditCachedAt = cachedAt.UTC().Format(time.RFC3339Nano) + response.Summary.AuditCacheAgeSeconds = int64(time.Since(cachedAt).Seconds()) + response.Summary.AuditCacheSource = strings.TrimSpace(source) + response.Summary.AuditCacheError = strings.TrimSpace(cacheError) +} + func filterStorageChainFindingsByKind(findings []GitStorageChainFinding, kind string) []GitStorageChainFinding { kind = strings.TrimSpace(kind) if kind == "" { @@ -887,9 +1172,6 @@ func accessProbeIsBroken(probe GitStorageCleanupAccessProbe) bool { case "missing_access_url", "credential_missing": return true } - if accessProbeHasMismatch(probe, "name_mismatch") { - return true - } switch strings.TrimSpace(probe.Status) { case "missing", "forbidden", "unsupported", "invalid", "error": return true @@ -1087,7 +1369,7 @@ func cleanupActionForFinding(index map[string]string, finding GitStorageCleanupF return strings.TrimSpace(index[cleanupActionKey("", path)]) } -func (service *StorageAnalyticsService) loadJoinState(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, []RepoInventoryFile, map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { +func (service *StorageAnalyticsService) loadJoinState(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, forceRefresh bool) (*repoAnalyticsIndex, []RepoInventoryFile, map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) if err != nil { return nil, nil, nil, nil, err @@ -1096,24 +1378,28 @@ func (service *StorageAnalyticsService) loadJoinState(ctx context.Context, autho if err != nil { return nil, nil, nil, nil, err } - recordsByChecksum, usageByObjectID, err := service.loadProjectJoinCache(ctx, authorizationHeader, organization, project, hash, index.sidecar.Files) + recordsByChecksum, usageByObjectID, err := service.loadProjectJoinCache(ctx, authorizationHeader, organization, project, hash, index.sidecar.Files, forceRefresh) if err != nil { return nil, nil, nil, nil, err } return index, inventory, recordsByChecksum, usageByObjectID, nil } -func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context, authorizationHeader string, organization string, project string, hash plumbing.Hash, inventory []RepoInventoryFile) (map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { +func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context, authorizationHeader string, organization string, project string, hash plumbing.Hash, inventory []RepoInventoryFile, forceRefresh bool) (map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { cacheKey := service.projectJoinCacheKey(organization, project, hash.String()) + redisKey := storageExactProjectJoinCacheKey(organization, project, hash.String()) service.projectJoinMu.Lock() - if cached, ok := service.projectJoinCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) { - service.projectJoinMu.Unlock() - return cached.recordsByChecksum, cached.usageByObjectID, nil + if !forceRefresh { + if cached, ok := service.projectJoinCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) { + service.projectJoinMu.Unlock() + log.Printf("storage_folder_exact_join_cache_hit org=%s project=%s source=memory records_by_checksum=%d usage_rows=%d", organization, project, len(cached.recordsByChecksum), len(cached.usageByObjectID)) + return cloneRecordStateMap(cached.recordsByChecksum), cloneFileUsageMap(cached.usageByObjectID), nil + } } if inflight, ok := service.projectJoinWork[cacheKey]; ok { service.projectJoinMu.Unlock() <-inflight.done - return inflight.recordsByChecksum, inflight.usageByObjectID, inflight.err + return cloneRecordStateMap(inflight.recordsByChecksum), cloneFileUsageMap(inflight.usageByObjectID), inflight.err } inflight := &inflightProjectJoinState{done: make(chan struct{})} service.projectJoinWork[cacheKey] = inflight @@ -1125,6 +1411,29 @@ func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context service.projectJoinMu.Unlock() }() + if !forceRefresh && service.exactProjectJoinCache != nil { + redisStart := time.Now() + cached, ok, err := service.exactProjectJoinCache.Get(ctx, redisKey) + if err != nil { + log.Printf("storage_folder_exact_join_cache_error org=%s project=%s source=%s operation=get error=%q", organization, project, service.exactProjectJoinCache.Source(), err.Error()) + } + if ok { + state := cachedProjectJoinState{ + expiresAt: time.Now().Add(projectJoinCacheTTL), + recordsByChecksum: cloneRecordStateMap(cached.RecordsByChecksum), + usageByObjectID: cloneFileUsageMap(cached.UsageByObjectID), + } + service.projectJoinMu.Lock() + service.projectJoinCache[cacheKey] = state + service.projectJoinMu.Unlock() + log.Printf("storage_folder_exact_join_cache_hit org=%s project=%s source=%s age_seconds=%d records_by_checksum=%d usage_rows=%d duration_ms=%d", organization, project, service.exactProjectJoinCache.Source(), int64(time.Since(cached.CachedAt).Seconds()), len(cached.RecordsByChecksum), len(cached.UsageByObjectID), time.Since(redisStart).Milliseconds()) + inflight.recordsByChecksum = cloneRecordStateMap(cached.RecordsByChecksum) + inflight.usageByObjectID = cloneFileUsageMap(cached.UsageByObjectID) + return cloneRecordStateMap(cached.RecordsByChecksum), cloneFileUsageMap(cached.UsageByObjectID), nil + } + log.Printf("storage_folder_exact_join_cache_miss org=%s project=%s source=%s duration_ms=%d", organization, project, service.exactProjectJoinCache.Source(), time.Since(redisStart).Milliseconds()) + } + recordStart := time.Now() projectRecords, err := service.loadCachedProjectAuditRecords(ctx, authorizationHeader, organization, project) if err != nil { @@ -1148,12 +1457,25 @@ func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context service.projectJoinMu.Lock() service.projectJoinCache[cacheKey] = cachedProjectJoinState{ expiresAt: time.Now().Add(projectJoinCacheTTL), - recordsByChecksum: recordsByChecksum, - usageByObjectID: usageByObjectID, + recordsByChecksum: cloneRecordStateMap(recordsByChecksum), + usageByObjectID: cloneFileUsageMap(usageByObjectID), } service.projectJoinMu.Unlock() - inflight.recordsByChecksum = recordsByChecksum - inflight.usageByObjectID = usageByObjectID + if service.exactProjectJoinCache != nil { + setStart := time.Now() + err := service.exactProjectJoinCache.Set(ctx, redisKey, cachedExactProjectJoinState{ + CachedAt: time.Now(), + RecordsByChecksum: recordsByChecksum, + UsageByObjectID: usageByObjectID, + }, storageChainAuditCacheTTL()) + if err != nil { + log.Printf("storage_folder_exact_join_cache_error org=%s project=%s source=%s operation=set error=%q", organization, project, service.exactProjectJoinCache.Source(), err.Error()) + } else { + log.Printf("storage_folder_exact_join_cache_store org=%s project=%s source=%s records_by_checksum=%d usage_rows=%d duration_ms=%d", organization, project, service.exactProjectJoinCache.Source(), len(recordsByChecksum), len(usageByObjectID), time.Since(setStart).Milliseconds()) + } + } + inflight.recordsByChecksum = cloneRecordStateMap(recordsByChecksum) + inflight.usageByObjectID = cloneFileUsageMap(usageByObjectID) return recordsByChecksum, usageByObjectID, nil } @@ -2181,7 +2503,19 @@ func storageProbeValidationMismatchIsSignificant(record projectRecordState, prob if strings.TrimSpace(probe.ValidationStatus) != "mismatched" { return false } - if !syfonProbeHasMismatch(probe, "size_mismatch") || len(probe.ValidationMismatches) != 1 { + significantMismatches := 0 + for _, mismatch := range probe.ValidationMismatches { + switch strings.TrimSpace(mismatch) { + case "", "name_mismatch": + continue + default: + significantMismatches++ + } + } + if significantMismatches == 0 { + return false + } + if !syfonProbeHasMismatch(probe, "size_mismatch") || significantMismatches != 1 { return true } if probe.SizeBytes == nil { @@ -2457,14 +2791,6 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, } func suggestedActionForChainFinding(kind string, record projectRecordState) string { - switch strings.TrimSpace(kind) { - case "git_syfon_metadata_mismatch", "syfon_broken_bucket_mapping": - default: - return "" - } - if recordHasAccessProbeMismatch(record, "name_mismatch") { - return storageActionRemoveBrokenAccessURLs - } return "" } @@ -2473,9 +2799,7 @@ func suggestedFixForChainFinding(kind string, record projectRecordState) string switch normalizedKind { case "git_syfon_metadata_mismatch": case "syfon_broken_bucket_mapping": - if !recordHasAccessProbeMismatch(record, "name_mismatch") { - return "" - } + return "" default: return "" } @@ -2487,17 +2811,9 @@ func suggestedFixForChainFinding(kind string, record projectRecordState) string } mismatches = append(mismatches, fmt.Sprintf("Syfon/Git size is %s, bucket inventory reports %s", formatAuditSize(record.Size), formatAuditSize(bucketSize))) } - if probe, ok := firstAccessProbeMismatch(record, "name_mismatch"); ok { - expectedName := path.Base(strings.Trim(strings.TrimSpace(record.Name), "/")) - observedName := path.Base(strings.Trim(strings.TrimSpace(probe.Key), "/")) - mismatches = append(mismatches, fmt.Sprintf("Syfon name is %q, bucket key basename is %q", expectedName, observedName)) - } if len(mismatches) == 0 { return "Bucket object exists, but its storage evidence does not match the Syfon/Git record. Review the record and bucket object before applying a destructive fix." } - if recordHasAccessProbeMismatch(record, "name_mismatch") { - return strings.Join(mismatches, ". ") + ". Remove the broken access URL if another access URL works; otherwise delete and recreate or correct the Syfon record." - } return strings.Join(mismatches, ". ") + ". Update the stale Syfon metadata or delete and recreate the record after confirming the bucket object is authoritative." } @@ -2786,9 +3102,6 @@ func syfonProbeIsBrokenAccess(probe gintegrationsyfon.BulkStorageProbeResult) bo case "missing_access_url", "credential_missing": return true } - if syfonProbeHasMismatch(probe, "name_mismatch") { - return true - } switch strings.TrimSpace(probe.Status) { case "missing", "forbidden", "unsupported", "invalid", "error": return true diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index eba234c..e2b59c8 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -272,7 +272,7 @@ func summarizeChainIssueGroups(findings []GitStorageChainFinding) []GitStorageCh } func (service *StorageAnalyticsService) loadStorageAuditBaseInputs(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*storageAuditBaseInputs, error) { - index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, false) if err != nil { return nil, err } @@ -808,16 +808,10 @@ func inventoryPresentProbe(record projectRecordState, objectURL string, expected size := item.SizeBytes sizeMatch := storageSizesMatchForAudit(record.Size, item.SizeBytes) nameMatch := true - if expectedName != "" { - nameMatch = path.Base(strings.Trim(strings.TrimSpace(item.Key), "/")) == expectedName - } - mismatches := make([]string, 0, 2) + mismatches := make([]string, 0, 1) if !sizeMatch { mismatches = append(mismatches, "size_mismatch") } - if !nameMatch { - mismatches = append(mismatches, "name_mismatch") - } validationStatus := "matched" if len(mismatches) > 0 { validationStatus = "mismatched" @@ -912,7 +906,7 @@ func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context return view, nil } -func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { +func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, forceRefresh bool, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) view := &storageAuditStorageView{ scopes: append([]domain.StorageBucketScope(nil), scopes...), @@ -932,7 +926,14 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex } if validationMode == StorageChainValidationModeList { inventoryStart := time.Now() - validationObjects, validationObjectsByURL, err := service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + var validationObjects []gintegrationsyfon.ProjectBucketObject + var validationObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + var err error + if forceRefresh { + validationObjects, validationObjectsByURL, err = service.loadProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + } else { + validationObjects, validationObjectsByURL, err = service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + } timings.Record("syfon_bucket_validation_inventory", time.Since(inventoryStart)) timings.RecordMemory("syfon_bucket_validation_inventory", "bucket_objects", len(validationObjects), "bucket_lookup", len(validationObjectsByURL)) if err != nil { diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index bb22b9e..5cd052e 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -547,7 +547,7 @@ func TestBuildStorageFolderCombinesSummaryAndChildren(t *testing.T) { if err != nil { t.Fatalf("build storage children: %v", err) } - folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "", StorageFolderSummaryModeExact, nil) + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "", StorageFolderSummaryModeExact, false, nil) if err != nil { t.Fatalf("build storage folder: %v", err) } @@ -565,7 +565,7 @@ func TestBuildStorageFolderCombinesSummaryAndChildren(t *testing.T) { } } - next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", folder.Children.NextCursor, StorageFolderSummaryModeExact, nil) + next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", folder.Children.NextCursor, StorageFolderSummaryModeExact, false, nil) if err != nil { t.Fatalf("build storage folder next page: %v", err) } @@ -597,7 +597,7 @@ func TestBuildStorageFolderDefaultModeUsesGitIndexOnly(t *testing.T) { } service := NewStorageAnalyticsService(backend) - folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", nil) + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", false, nil) if err != nil { t.Fatalf("build storage folder: %v", err) } @@ -651,7 +651,7 @@ func TestBuildStorageFolderDefaultModeUsesPreSortedServingIndex(t *testing.T) { } for name, test := range tests { t.Run(name, func(t *testing.T) { - folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, test.sortBy, test.sortOrder, "", "", nil) + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, test.sortBy, test.sortOrder, "", "", false, nil) if err != nil { t.Fatalf("build storage folder: %v", err) } @@ -661,7 +661,7 @@ func TestBuildStorageFolderDefaultModeUsesPreSortedServingIndex(t *testing.T) { if !folder.Children.HasMore || folder.Children.NextCursor == "" { t.Fatalf("expected first page cursor, got %+v", folder.Children) } - next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, test.sortBy, test.sortOrder, folder.Children.NextCursor, "", nil) + next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, test.sortBy, test.sortOrder, folder.Children.NextCursor, "", false, nil) if err != nil { t.Fatalf("build storage folder next page: %v", err) } @@ -689,7 +689,7 @@ func TestBuildStorageFolderDefaultModeUsesSidecarServingIndex(t *testing.T) { repoAnalyticsIndexCache.mu.Unlock() service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) - folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", nil) + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", false, nil) if err != nil { t.Fatalf("build storage folder from sidecar: %v", err) } @@ -1762,17 +1762,17 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { if chain.Summary.BucketObjectCount != 2 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { t.Fatalf("expected validate mode to count project inventory without bucket-only findings, got summary %+v", chain.Summary) } - if chain.Summary.CountsByKind["bucket_syfon_git_complete"] != 1 { + if chain.Summary.CountsByKind["bucket_syfon_git_complete"] != 2 { t.Fatalf("expected validate mode to count clean Syfon/Git/bucket join, got summary %+v", chain.Summary) } if chain.Summary.CountsByKind["syfon_missing_bucket_object"] != 1 { t.Fatalf("expected validate mode to report missing Syfon bucket object, got summary %+v", chain.Summary) } - if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { - t.Fatalf("expected one LIST-derived broken bucket mapping, got summary %+v", chain.Summary) + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected basename mismatch not to be a broken bucket mapping, got summary %+v", chain.Summary) } assertNoChainFinding(t, chain.Findings, "bucket_only_object") - assertHasChainFinding(t, chain.Findings, "syfon_broken_bucket_mapping", "data/mismatch.txt") + assertNoChainFinding(t, chain.Findings, "syfon_broken_bucket_mapping") assertHasChainFinding(t, chain.Findings, "syfon_missing_bucket_object", "s3://bucket/syfon-only.txt") } @@ -1838,15 +1838,6 @@ func TestBuildStorageChainAuditMetadataModeUsesMetadataValidation(t *testing.T) } } -func TestExpectedStorageObjectNameForListValidationUsesRecordNameForContentAddressedKeys(t *testing.T) { - if got := expectedStorageObjectNameForListValidation("s3://cbds/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/ec4b068cb42b52449dd44052c3bfb2a459b00336a9cd42cd29c22ca1d1b26cb0", "CONFIG/cbds-BForePC.json"); got != "cbds-BForePC.json" { - t.Fatalf("expected content-addressed storage key to validate basename, got %q", got) - } - if got := expectedStorageObjectNameForListValidation("s3://bucket/path/cbds-BForePC.json", "CONFIG/cbds-BForePC.json"); got != "cbds-BForePC.json" { - t.Fatalf("expected filename-backed storage key to validate basename, got %q", got) - } -} - func TestBuildStorageChainAuditIgnoresOneByteListSizeDrift(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 44), @@ -1912,7 +1903,7 @@ func TestBuildStorageChainAuditSuggestsFixForLargeSizeDrift(t *testing.T) { } } -func TestBuildStorageChainAuditSuggestsRealAccessURLRepairForNameMismatch(t *testing.T) { +func TestBuildStorageChainAuditIgnoresBucketBasenameMismatch(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/right.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), }) @@ -1925,10 +1916,9 @@ func TestBuildStorageChainAuditSuggestsRealAccessURLRepairForNameMismatch(t *tes Organization: "org", Project: "proj", Size: 100, - AccessURLs: []string{"s3://bucket/data/wrong.txt", "s3://bucket/data/right.txt"}, + AccessURLs: []string{"s3://bucket/data/wrong.txt"}, AccessMethods: []gintegrationsyfon.ProjectAccessMethod{ - {AccessID: "bad", Type: "s3", URL: "s3://bucket/data/wrong.txt"}, - {AccessID: "good", Type: "s3", URL: "s3://bucket/data/right.txt"}, + {AccessID: "s3", Type: "s3", URL: "s3://bucket/data/wrong.txt"}, }, }, }, @@ -1937,7 +1927,6 @@ func TestBuildStorageChainAuditSuggestsRealAccessURLRepairForNameMismatch(t *tes }, bucketObjects: []gintegrationsyfon.ProjectBucketObject{ {ObjectURL: "s3://bucket/data/wrong.txt", Bucket: "bucket", Key: "data/wrong.txt", Path: "data/wrong.txt", SizeBytes: 100}, - {ObjectURL: "s3://bucket/data/right.txt", Bucket: "bucket", Key: "data/right.txt", Path: "data/right.txt", SizeBytes: 100}, }, } service := NewStorageAnalyticsService(backend) @@ -1949,35 +1938,14 @@ func TestBuildStorageChainAuditSuggestsRealAccessURLRepairForNameMismatch(t *tes if err != nil { t.Fatalf("build chain audit: %v", err) } - finding := assertHasChainFinding(t, chain.Findings, "syfon_broken_bucket_mapping", "data/right.txt") - if finding.SuggestedAction != storageActionRemoveBrokenAccessURLs { - t.Fatalf("expected name mismatch to suggest real access URL repair, got %+v", finding) - } - if !contains(finding.AvailableActions, storageActionRemoveBrokenAccessURLs) { - t.Fatalf("expected suggested action to be advertised, got %+v", finding.AvailableActions) - } - if !strings.Contains(finding.SuggestedFix, `Syfon name is "right.txt", bucket key basename is "wrong.txt"`) { - t.Fatalf("expected name mismatch suggested fix, got %+v", finding) - } - - applyResponse, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", nil, nil, []GitStorageCleanupApplyFinding{{ - Kind: finding.Kind, - NormalizedPath: finding.NormalizedPath, - ObjectIDs: finding.ObjectIDs, - Records: finding.Records, - AccessURLs: finding.AccessURLs, - AvailableActions: finding.AvailableActions, - SuggestedAction: finding.SuggestedAction, - }}, false, false, false, false, false) - if err != nil { - t.Fatalf("apply suggested repair: %v", err) + if got := chain.Summary.CountsByKind["git_syfon_metadata_mismatch"]; got != 0 { + t.Fatalf("expected basename-only mismatch to be ignored, got summary %+v", chain.Summary) } - if len(applyResponse.UpdatedRecordIDs) != 1 || applyResponse.UpdatedRecordIDs[0] != "obj-a" { - t.Fatalf("expected access method update, got %+v", applyResponse) + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected basename-only mismatch not to be a broken mapping, got summary %+v", chain.Summary) } - methods := backend.updatedAccessMethods["obj-a"] - if len(methods) != 1 || methods[0].URL != "s3://bucket/data/right.txt" { - t.Fatalf("expected broken access URL removed through Syfon update, got %+v", methods) + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected basename-only mismatch to count as complete, got summary %+v", chain.Summary) } } @@ -2022,6 +1990,160 @@ func TestBuildStorageChainAuditCachesProjectChainInputs(t *testing.T) { } } +func TestBuildStorageChainAuditCachesFullResponseAcrossFindingViews(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Name: "present.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + {ObjectID: "obj-syfon-only", Name: "syfon-only.txt", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, AccessURLs: []string{"s3://bucket/data/syfon-only.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + {ObjectURL: "s3://bucket/data/syfon-only.txt", Bucket: "bucket", Key: "data/syfon-only.txt", Path: "data/syfon-only.txt", SizeBytes: 200}, + }, + } + service := NewStorageAnalyticsService(backend) + service.chainAuditResponseCache = newMemoryStorageChainAuditResponseCache() + + first, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + FindingKind: "git_only_no_syfon", + FindingLimit: -1, + }) + if err != nil { + t.Fatalf("build first chain audit: %v", err) + } + if first.Summary.AuditCacheHit { + t.Fatalf("expected first response to build fresh, got %+v", first.Summary) + } + if first.Summary.ReturnedFindings != 0 || first.Summary.FindingsTruncated { + t.Fatalf("expected first response to apply request finding kind, got %+v", first.Summary) + } + + second, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + FindingKind: "bucket_syfon_no_git", + FindingLimit: -1, + }) + if err != nil { + t.Fatalf("build second chain audit: %v", err) + } + if !second.Summary.AuditCacheHit { + t.Fatalf("expected second response to come from audit response cache, got %+v", second.Summary) + } + if len(second.Findings) != 1 || second.Findings[0].Kind != "bucket_syfon_no_git" { + t.Fatalf("expected cached full response to serve issue-kind view, got %+v", second.Findings) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected cached full response to skip Syfon record reload, got %d calls", backend.listProjectAuditRecordsCalls) + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected cached full response to skip bucket inventory reload, got %d calls", backend.listProjectBucketInventoryCalls) + } +} + +func TestBuildStorageChainAuditProjectsGitSubpathFromRootResponseCache(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "other/clean.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 300), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Name: "present.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + {ObjectID: "obj-clean", Name: "clean.txt", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj", Size: 300, AccessURLs: []string{"s3://bucket/other/clean.txt"}}, + {ObjectID: "obj-syfon-only", Name: "syfon-only.txt", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, AccessURLs: []string{"s3://bucket/data/syfon-only.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + {ObjectURL: "s3://bucket/other/clean.txt", Bucket: "bucket", Key: "other/clean.txt", Path: "other/clean.txt", SizeBytes: 300}, + {ObjectURL: "s3://bucket/data/syfon-only.txt", Bucket: "bucket", Key: "data/syfon-only.txt", Path: "data/syfon-only.txt", SizeBytes: 200}, + }, + } + service := NewStorageAnalyticsService(backend) + cache := newMemoryStorageChainAuditResponseCache() + service.chainAuditResponseCache = cache + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + root, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options) + if err != nil { + t.Fatalf("build root chain audit: %v", err) + } + if root.Summary.AuditCacheHit { + t.Fatalf("expected root audit to build fresh, got %+v", root.Summary) + } + if backend.listProjectAuditRecordsCalls != 1 || backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected root audit to load Syfon records and bucket inventory once, got records=%d bucket=%d", backend.listProjectAuditRecordsCalls, backend.listProjectBucketInventoryCalls) + } + subpath, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options) + if err != nil { + t.Fatalf("build subpath chain audit: %v", err) + } + if !subpath.Summary.AuditCacheHit || subpath.Summary.AuditCacheSource != "memory:root" { + t.Fatalf("expected subpath audit to project from root response cache, got %+v", subpath.Summary) + } + if subpath.PathPrefix != "data" || subpath.Summary.GitTrackedFileCount != 1 { + t.Fatalf("expected data-scoped audit projection, got path=%q summary=%+v", subpath.PathPrefix, subpath.Summary) + } + if got := subpath.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected subpath complete count from Git inventory, got summary %+v", subpath.Summary) + } + if got := subpath.Summary.CountsByKind["bucket_syfon_no_git"]; got != 1 { + t.Fatalf("expected subpath Syfon-only finding from root cache, got summary %+v findings=%+v", subpath.Summary, subpath.Findings) + } + if backend.listProjectAuditRecordsCalls != 1 || backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected subpath projection to skip Syfon records and bucket inventory, got records=%d bucket=%d", backend.listProjectAuditRecordsCalls, backend.listProjectBucketInventoryCalls) + } + cache.mu.RLock() + cacheEntryCount := len(cache.entries) + cache.mu.RUnlock() + if cacheEntryCount != 1 { + t.Fatalf("expected only root response cache entry, got %d", cacheEntryCount) + } +} + +func TestBuildStorageChainAuditForceRefreshBypassesResponseCache(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Name: "present.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + service.chainAuditResponseCache = newMemoryStorageChainAuditResponseCache() + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build first chain audit: %v", err) + } + options.ForceAuditRefresh = true + refreshed, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options) + if err != nil { + t.Fatalf("build forced chain audit: %v", err) + } + if refreshed.Summary.AuditCacheHit { + t.Fatalf("expected forced response to rebuild fresh, got %+v", refreshed.Summary) + } + if backend.listProjectBucketInventoryCalls != 2 { + t.Fatalf("expected forced refresh to reload bucket inventory, got %d calls", backend.listProjectBucketInventoryCalls) + } +} + func TestStorageFolderExactReusesAuditProjectRecordCache(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -3606,14 +3728,14 @@ func BenchmarkBuildStorageFolderDefaultModeLargeDirectoryPage(b *testing.B) { }) repoAnalyticsIndexCache.put(mirrorPath, hash, index) service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) - if _, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", "main", "data", mirrorPath, nil, hash, 100, "bytes", "desc", "", "", nil); err != nil { + if _, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", "main", "data", mirrorPath, nil, hash, 100, "bytes", "desc", "", "", false, nil); err != nil { b.Fatalf("warm storage folder index: %v", err) } b.ReportAllocs() b.ResetTimer() for index := 0; index < b.N; index++ { - if _, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", "main", "data", mirrorPath, nil, hash, 100, "bytes", "desc", "", "", nil); err != nil { + if _, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", "main", "data", mirrorPath, nil, hash, 100, "bytes", "desc", "", "", false, nil); err != nil { b.Fatalf("build storage folder: %v", err) } } diff --git a/internal/git/storage_analytics_timing.go b/internal/git/storage_analytics_timing.go index 98f565b..3a36eb8 100644 --- a/internal/git/storage_analytics_timing.go +++ b/internal/git/storage_analytics_timing.go @@ -27,6 +27,7 @@ type StorageChainAuditOptions struct { BucketPathPrefix string FindingKind string FindingLimit int + ForceAuditRefresh bool Timings *StorageChainAuditTimings } diff --git a/internal/git/storage_chain_audit_cache.go b/internal/git/storage_chain_audit_cache.go new file mode 100644 index 0000000..7aa2c4f --- /dev/null +++ b/internal/git/storage_chain_audit_cache.go @@ -0,0 +1,364 @@ +package git + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "sync" + "time" + + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + "github.com/redis/go-redis/v9" +) + +const ( + defaultStorageChainAuditCacheTTL = 24 * time.Hour + storageChainAuditCacheKeyPrefix = "gecko:storage_chain_audit:v1:" + storageExactJoinCacheKeyPrefix = "gecko:storage_exact_join:v1:" +) + +type storageChainAuditResponseCache interface { + Get(ctx context.Context, key string) (cachedStorageChainAuditResponse, bool, error) + Set(ctx context.Context, key string, value cachedStorageChainAuditResponse, ttl time.Duration) error + Source() string +} + +type cachedStorageChainAuditResponse struct { + CachedAt time.Time `json:"cached_at"` + Response GitStorageChainAuditResponse `json:"response"` +} + +type storageExactProjectJoinCache interface { + Get(ctx context.Context, key string) (cachedExactProjectJoinState, bool, error) + Set(ctx context.Context, key string, value cachedExactProjectJoinState, ttl time.Duration) error + Source() string +} + +type cachedExactProjectJoinState struct { + CachedAt time.Time `json:"cached_at"` + RecordsByChecksum map[string][]projectRecordState `json:"records_by_checksum"` + UsageByObjectID map[string]gintegrationsyfon.FileUsage `json:"usage_by_object_id"` +} + +type memoryStorageChainAuditResponseCache struct { + mu sync.RWMutex + entries map[string]memoryStorageChainAuditResponseEntry +} + +type memoryStorageChainAuditResponseEntry struct { + value cachedStorageChainAuditResponse + expiresAt time.Time +} + +func newMemoryStorageChainAuditResponseCache() *memoryStorageChainAuditResponseCache { + return &memoryStorageChainAuditResponseCache{entries: map[string]memoryStorageChainAuditResponseEntry{}} +} + +func (cache *memoryStorageChainAuditResponseCache) Get(_ context.Context, key string) (cachedStorageChainAuditResponse, bool, error) { + cache.mu.RLock() + entry, ok := cache.entries[key] + cache.mu.RUnlock() + if !ok || time.Now().After(entry.expiresAt) { + if ok { + cache.mu.Lock() + delete(cache.entries, key) + cache.mu.Unlock() + } + return cachedStorageChainAuditResponse{}, false, nil + } + return cloneCachedStorageChainAuditResponse(entry.value), true, nil +} + +func (cache *memoryStorageChainAuditResponseCache) Set(_ context.Context, key string, value cachedStorageChainAuditResponse, ttl time.Duration) error { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.entries[key] = memoryStorageChainAuditResponseEntry{ + value: cloneCachedStorageChainAuditResponse(value), + expiresAt: time.Now().Add(ttl), + } + return nil +} + +func (cache *memoryStorageChainAuditResponseCache) Source() string { + return "memory" +} + +type memoryStorageExactProjectJoinCache struct { + mu sync.RWMutex + entries map[string]memoryStorageExactProjectJoinEntry +} + +type memoryStorageExactProjectJoinEntry struct { + value cachedExactProjectJoinState + expiresAt time.Time +} + +func newMemoryStorageExactProjectJoinCache() *memoryStorageExactProjectJoinCache { + return &memoryStorageExactProjectJoinCache{entries: map[string]memoryStorageExactProjectJoinEntry{}} +} + +func (cache *memoryStorageExactProjectJoinCache) Get(_ context.Context, key string) (cachedExactProjectJoinState, bool, error) { + cache.mu.RLock() + entry, ok := cache.entries[key] + cache.mu.RUnlock() + if !ok || time.Now().After(entry.expiresAt) { + if ok { + cache.mu.Lock() + delete(cache.entries, key) + cache.mu.Unlock() + } + return cachedExactProjectJoinState{}, false, nil + } + return cloneCachedExactProjectJoinState(entry.value), true, nil +} + +func (cache *memoryStorageExactProjectJoinCache) Set(_ context.Context, key string, value cachedExactProjectJoinState, ttl time.Duration) error { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.entries[key] = memoryStorageExactProjectJoinEntry{ + value: cloneCachedExactProjectJoinState(value), + expiresAt: time.Now().Add(ttl), + } + return nil +} + +func (cache *memoryStorageExactProjectJoinCache) Source() string { + return "memory" +} + +type redisStorageChainAuditResponseCache struct { + client *redis.Client +} + +type redisStorageExactProjectJoinCache struct { + client *redis.Client +} + +func newRedisStorageChainAuditResponseCache(redisURL string) (*redisStorageChainAuditResponseCache, error) { + options, err := redis.ParseURL(strings.TrimSpace(redisURL)) + if err != nil { + return nil, err + } + cache := &redisStorageChainAuditResponseCache{client: redis.NewClient(options)} + if err := cache.client.Ping(context.Background()).Err(); err != nil { + _ = cache.client.Close() + return nil, err + } + return cache, nil +} + +func newRedisStorageExactProjectJoinCache(redisURL string) (*redisStorageExactProjectJoinCache, error) { + options, err := redis.ParseURL(strings.TrimSpace(redisURL)) + if err != nil { + return nil, err + } + cache := &redisStorageExactProjectJoinCache{client: redis.NewClient(options)} + if err := cache.client.Ping(context.Background()).Err(); err != nil { + _ = cache.client.Close() + return nil, err + } + return cache, nil +} + +func (cache *redisStorageChainAuditResponseCache) Get(ctx context.Context, key string) (cachedStorageChainAuditResponse, bool, error) { + raw, err := cache.client.Get(ctx, key).Bytes() + if err == redis.Nil { + return cachedStorageChainAuditResponse{}, false, nil + } + if err != nil { + return cachedStorageChainAuditResponse{}, false, err + } + var value cachedStorageChainAuditResponse + if err := json.Unmarshal(raw, &value); err != nil { + return cachedStorageChainAuditResponse{}, false, err + } + if value.CachedAt.IsZero() { + value.CachedAt = time.Now() + } + return value, true, nil +} + +func (cache *redisStorageChainAuditResponseCache) Set(ctx context.Context, key string, value cachedStorageChainAuditResponse, ttl time.Duration) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + return cache.client.Set(ctx, key, raw, ttl).Err() +} + +func (cache *redisStorageChainAuditResponseCache) Source() string { + return "redis" +} + +func (cache *redisStorageExactProjectJoinCache) Get(ctx context.Context, key string) (cachedExactProjectJoinState, bool, error) { + raw, err := cache.client.Get(ctx, key).Bytes() + if err == redis.Nil { + return cachedExactProjectJoinState{}, false, nil + } + if err != nil { + return cachedExactProjectJoinState{}, false, err + } + var value cachedExactProjectJoinState + if err := json.Unmarshal(raw, &value); err != nil { + return cachedExactProjectJoinState{}, false, err + } + if value.CachedAt.IsZero() { + value.CachedAt = time.Now() + } + return value, true, nil +} + +func (cache *redisStorageExactProjectJoinCache) Set(ctx context.Context, key string, value cachedExactProjectJoinState, ttl time.Duration) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + return cache.client.Set(ctx, key, raw, ttl).Err() +} + +func (cache *redisStorageExactProjectJoinCache) Source() string { + return "redis" +} + +func NewStorageChainAuditResponseCacheFromEnv() storageChainAuditResponseCache { + if !storageChainAuditCacheEnabled() { + return nil + } + redisURL := storageChainAuditRedisURLFromEnv() + if redisURL != "" { + cache, err := newRedisStorageChainAuditResponseCache(redisURL) + if err == nil { + return cache + } + } + return newMemoryStorageChainAuditResponseCache() +} + +func NewStorageExactProjectJoinCacheFromEnv() storageExactProjectJoinCache { + if !storageChainAuditCacheEnabled() { + return nil + } + redisURL := storageChainAuditRedisURLFromEnv() + if redisURL != "" { + cache, err := newRedisStorageExactProjectJoinCache(redisURL) + if err == nil { + return cache + } + } + return newMemoryStorageExactProjectJoinCache() +} + +func storageChainAuditRedisURLFromEnv() string { + redisURL := strings.TrimSpace(os.Getenv("STORAGE_CHAIN_AUDIT_CACHE_REDIS_URL")) + if redisURL == "" { + redisURL = strings.TrimSpace(os.Getenv("REDIS_URL")) + } + return redisURLWithPassword(redisURL, strings.TrimSpace(os.Getenv("STORAGE_CHAIN_AUDIT_CACHE_REDIS_PASSWORD"))) +} + +func redisURLWithPassword(redisURL string, password string) string { + redisURL = strings.TrimSpace(redisURL) + password = strings.TrimSpace(password) + if redisURL == "" || password == "" || !strings.HasPrefix(redisURL, "redis://") || strings.Contains(redisURL, "@") { + return redisURL + } + return "redis://:" + password + "@" + strings.TrimPrefix(redisURL, "redis://") +} + +func storageChainAuditCacheEnabled() bool { + raw := strings.TrimSpace(os.Getenv("STORAGE_CHAIN_AUDIT_CACHE_ENABLED")) + if raw == "" { + return true + } + enabled, err := strconv.ParseBool(raw) + return err == nil && enabled +} + +func storageChainAuditCacheTTL() time.Duration { + raw := strings.TrimSpace(os.Getenv("STORAGE_CHAIN_AUDIT_CACHE_TTL_SECONDS")) + if raw == "" { + return defaultStorageChainAuditCacheTTL + } + seconds, err := strconv.Atoi(raw) + if err != nil || seconds < 60 { + return defaultStorageChainAuditCacheTTL + } + return time.Duration(seconds) * time.Second +} + +func storageChainAuditResponseCacheKey(organization string, project string, ref string, gitSubpath string, probeMode string, validationMode string, bucketMode string, bucketPathPrefix string, hash string) string { + body := fmt.Sprintf( + "org=%s\nproject=%s\nref=%s\nhash=%s\ngit_subpath=%s\nprobe_mode=%s\nvalidation_mode=%s\nbucket_mode=%s\nbucket_path_prefix=%s", + strings.TrimSpace(organization), + strings.TrimSpace(project), + strings.TrimSpace(ref), + strings.TrimSpace(hash), + normalizeRepoSubpath(gitSubpath), + strings.TrimSpace(probeMode), + strings.TrimSpace(validationMode), + strings.TrimSpace(bucketMode), + normalizeRepoSubpath(bucketPathPrefix), + ) + sum := sha256.Sum256([]byte(body)) + return storageChainAuditCacheKeyPrefix + hex.EncodeToString(sum[:]) +} + +func storageExactProjectJoinCacheKey(organization string, project string, hash string) string { + body := fmt.Sprintf( + "org=%s\nproject=%s\nhash=%s", + strings.TrimSpace(organization), + strings.TrimSpace(project), + strings.TrimSpace(hash), + ) + sum := sha256.Sum256([]byte(body)) + return storageExactJoinCacheKeyPrefix + hex.EncodeToString(sum[:]) +} + +func cloneCachedStorageChainAuditResponse(value cachedStorageChainAuditResponse) cachedStorageChainAuditResponse { + return cachedStorageChainAuditResponse{ + CachedAt: value.CachedAt, + Response: cloneStorageChainAuditResponse(value.Response), + } +} + +func cloneStorageChainAuditResponse(response GitStorageChainAuditResponse) GitStorageChainAuditResponse { + response.Findings = append([]GitStorageChainFinding(nil), response.Findings...) + response.Groups = append([]GitStorageChainIssueGroup(nil), response.Groups...) + response.Summary.CountsByKind = cloneStringIntMap(response.Summary.CountsByKind) + return response +} + +func cloneCachedExactProjectJoinState(value cachedExactProjectJoinState) cachedExactProjectJoinState { + return cachedExactProjectJoinState{ + CachedAt: value.CachedAt, + RecordsByChecksum: cloneRecordStateMap(value.RecordsByChecksum), + UsageByObjectID: cloneFileUsageMap(value.UsageByObjectID), + } +} + +func cloneFileUsageMap(input map[string]gintegrationsyfon.FileUsage) map[string]gintegrationsyfon.FileUsage { + if input == nil { + return nil + } + out := make(map[string]gintegrationsyfon.FileUsage, len(input)) + for key, value := range input { + out[key] = value + } + return out +} + +func cloneStringIntMap(input map[string]int) map[string]int { + if input == nil { + return nil + } + out := make(map[string]int, len(input)) + for key, value := range input { + out[key] = value + } + return out +} diff --git a/internal/git/storage_chain_audit_cache_test.go b/internal/git/storage_chain_audit_cache_test.go new file mode 100644 index 0000000..2dc1a0e --- /dev/null +++ b/internal/git/storage_chain_audit_cache_test.go @@ -0,0 +1,19 @@ +package git + +import "testing" + +func TestRedisURLWithPasswordMatchesFenceStyle(t *testing.T) { + got := redisURLWithPassword("redis://authz-cache-service:6379/0", "secret") + want := "redis://:secret@authz-cache-service:6379/0" + if got != want { + t.Fatalf("redisURLWithPassword() = %q, want %q", got, want) + } +} + +func TestRedisURLWithPasswordKeepsExistingCredential(t *testing.T) { + input := "redis://:existing@authz-cache-service:6379/0" + got := redisURLWithPassword(input, "secret") + if got != input { + t.Fatalf("redisURLWithPassword() = %q, want %q", got, input) + } +} diff --git a/internal/git/types.go b/internal/git/types.go index 2b4afcd..edb7fb9 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -409,6 +409,9 @@ type GitStorageChainAuditRequest struct { BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` FindingKind string `json:"finding_kind,omitempty"` FindingLimit int `json:"finding_limit,omitempty"` + ForceAuditRefresh bool `json:"force_audit_refresh,omitempty"` + // ForceBucketInventoryRefresh is accepted as a compatibility alias for older frontend branches. + ForceBucketInventoryRefresh bool `json:"force_bucket_inventory_refresh,omitempty"` } type GitStorageChainFinding struct { @@ -452,6 +455,11 @@ type GitStorageChainAuditSummary struct { BucketSummaryMode string `json:"bucket_summary_mode,omitempty"` BucketInventoryAvailable bool `json:"bucket_inventory_available"` BucketInventoryError string `json:"bucket_inventory_error,omitempty"` + AuditCacheHit bool `json:"audit_cache_hit,omitempty"` + AuditCachedAt string `json:"audit_cached_at,omitempty"` + AuditCacheAgeSeconds int64 `json:"audit_cache_age_seconds,omitempty"` + AuditCacheSource string `json:"audit_cache_source,omitempty"` + AuditCacheError string `json:"audit_cache_error,omitempty"` } type GitStorageChainIssueGroup struct { diff --git a/internal/server/http/git/handler.go b/internal/server/http/git/handler.go index 94eb969..f8ee83a 100644 --- a/internal/server/http/git/handler.go +++ b/internal/server/http/git/handler.go @@ -31,6 +31,7 @@ func NewHandler(sharedHandler *shared.Handler) *Handler { var storageAnalytics *git.StorageAnalyticsService if sharedHandler.GitService != nil && sharedHandler.SyfonManager != nil { storageAnalytics = git.NewStorageAnalyticsService(sharedHandler.SyfonManager) + storageAnalytics.EnableStorageChainAuditResponseCacheFromEnv() } return &Handler{ Handler: sharedHandler, diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index b207990..815d667 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -104,6 +104,7 @@ func (handler *Handler) handleGitProjectStorageFolderGET(ctx fiber.Ctx) error { Logf: handler.logger.Info, } timings.Record("resolve_git_context", resolveDuration) + forceRefresh := parseBoolQuery(ctx.Query("force_refresh")) || parseBoolQuery(ctx.Query("force_audit_refresh")) || parseBoolQuery(ctx.Query("force_bucket_inventory_refresh")) buildStart := time.Now() response, err := handler.storageAnalytics.BuildStorageFolder( ctx.Context(), @@ -120,6 +121,7 @@ func (handler *Handler) handleGitProjectStorageFolderGET(ctx fiber.Ctx) error { options.sortOrder, options.cursor, options.summaryMode, + forceRefresh, timings, ) timings.Record("build_folder", time.Since(buildStart)) @@ -147,6 +149,11 @@ func parseStorageChildrenRequestOptions(ctx fiber.Ctx, projectID string) (storag }, nil } +func parseBoolQuery(raw string) bool { + parsed, err := strconv.ParseBool(strings.TrimSpace(raw)) + return err == nil && parsed +} + func parseStorageChildrenLimit(rawLimit string) (int, error) { if rawLimit == "" { return defaultStorageChildrenLimit, nil @@ -276,6 +283,7 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err findingKind := strings.TrimSpace(requestBody.FindingKind) timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_prefix=%q finding_kind=%q finding_limit=%d", projectCtx.projectID, projectCtx.refName, gitSubpath, validationMode, probeMode, bucketMode, bucketPathPrefix, findingKind, findingLimit) handler.logger.Info("storage_chain_audit_request_start %s", timings.DebugPrefix) + forceAuditRefresh := requestBody.ForceAuditRefresh || requestBody.ForceBucketInventoryRefresh response, err := handler.storageAnalytics.BuildStorageChainAuditWithOptions( ctx.Context(), projectCtx.authorizationHeader, @@ -293,6 +301,7 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err BucketPathPrefix: bucketPathPrefix, FindingKind: findingKind, FindingLimit: findingLimit, + ForceAuditRefresh: forceAuditRefresh, Timings: timings, }, ) @@ -595,7 +604,7 @@ func (handler *Handler) logStorageChainAuditTimings(projectCtx *gitAnalyticsCont bucketPathExists = strconv.FormatBool(*response.Summary.BucketPathExists) } handler.logger.Info( - "storage_chain_audit project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_exists=%s bucket_summary_mode=%s bucket_inventory_available=%t findings=%d returned_findings=%d findings_truncated=%t finding_limit=%d bucket_objects=%d syfon_records=%d git_files=%d %s", + "storage_chain_audit project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_exists=%s bucket_summary_mode=%s bucket_inventory_available=%t audit_cache_hit=%t audit_cache_source=%s audit_cache_age_seconds=%d findings=%d returned_findings=%d findings_truncated=%t finding_limit=%d bucket_objects=%d syfon_records=%d git_files=%d %s", projectCtx.projectID, projectCtx.refName, gitSubpath, @@ -605,6 +614,9 @@ func (handler *Handler) logStorageChainAuditTimings(projectCtx *gitAnalyticsCont bucketPathExists, response.Summary.BucketSummaryMode, response.Summary.BucketInventoryAvailable, + response.Summary.AuditCacheHit, + response.Summary.AuditCacheSource, + response.Summary.AuditCacheAgeSeconds, response.Summary.TotalFindings, response.Summary.ReturnedFindings, response.Summary.FindingsTruncated, From 332ee47671dd9b78b056e10a5a3a6d06d8c2719f Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 8 Jul 2026 16:34:29 -0700 Subject: [PATCH 29/36] standardize path normalization in gecko audit route --- internal/git/storage_analytics.go | 282 ++++++++++----------- internal/git/storage_analytics_pipeline.go | 85 ++++++- internal/git/storage_analytics_test.go | 76 ++++++ 3 files changed, 283 insertions(+), 160 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index e1130e3..f118165 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -609,7 +609,7 @@ func (service *StorageAnalyticsService) buildStorageChainAuditFresh(ctx context. } storageViewStart := time.Now() options.Timings.StageStart("storage_view") - storageView, err := service.buildStorageChainView(ctx, authorizationHeader, organization, project, inputs.recordSet, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.ForceAuditRefresh, options.Timings) + storageView, err := service.buildStorageChainView(ctx, authorizationHeader, organization, project, inputs.recordSet, inputs.inventory, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.ForceAuditRefresh, options.Timings) options.Timings.Record("storage_view", time.Since(storageViewStart)) if storageView != nil { options.Timings.RecordMemory( @@ -2175,7 +2175,7 @@ func cleanupSelectionCandidatesForRecords(checksum string, displayPath string, m candidates = append(candidates, match.ObjectID) candidates = append(candidates, match.AccessURLs...) candidates = append(candidates, match.CanonicalAccessURLs...) - candidates = append(candidates, accessURLsForStorage(match)...) + candidates = append(candidates, recordStorageCandidateURLs(match)...) for _, probe := range match.AccessProbes { candidates = append(candidates, probe.ObjectURL) candidates = append(candidates, canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL)) @@ -2385,77 +2385,69 @@ func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[st if recordHasBrokenAccess(record) { return storageFindingBrokenAccessURL } - bucketMatches := matchedBucketObjectURLs(record, bucketObjectsByURL) - if hasExactPathBucketMismatch(record, bucketMatches, bucketObjectsByURL) { + resolution := resolveRecordStorage(record, bucketObjectsByURL) + if hasExactPathBucketMismatch(record, resolution, bucketObjectsByURL) { return storageFindingBrokenBucketMap } if len(repairableBrokenAccessProbes(record)) > 0 { return storageFindingBrokenBucketMap } - if recordHasMatchedCanonicalAccessProbe(record) { + if resolution.hasAcceptedCanonicalProbe { return storageFindingNone } - if inventoryHasValidationMismatch(record, bucketMatches, bucketObjectsByURL) { + if inventoryHasValidationMismatch(record, resolution.matchedBucketObjectURLs, bucketObjectsByURL) { return storageFindingValidationMismatch } - if recordHasValidationMismatchProbe(record) { + if resolution.hasValidationMismatch { return storageFindingValidationMismatch } - if len(bucketMatches) > 0 { + if len(resolution.matchedBucketObjectURLs) > 0 { + if resolution.hasPresentRawAccessProbe && resolution.hasBrokenRawAccessProbe { + return storageFindingBrokenBucketMap + } + if resolution.hasPresentRawAccessProbe && resolution.hasMissingRawAccessProbe { + return storageFindingObjectMissing + } + if resolution.hasPresentRawAccessProbe && resolution.hasRawAccessProbeError { + return storageFindingProbeError + } return storageFindingNone } - if kind := classifyRawAccessURLFindings(record); kind != storageFindingNone { - return kind - } if len(record.AccessProbes) == 0 { - if len(bucketObjectsByURL) > 0 && hasCanonicalBucketURL(record) { + if len(bucketObjectsByURL) > 0 && len(resolution.candidateURLs) > 0 { return storageFindingProbeError } return storageFindingNone } - hasPresent := false - hasMissing := false - hasBrokenBucketMapping := false - hasProbeError := false - hasBucketMatch := len(bucketMatches) > 0 - for _, probe := range record.AccessProbes { - if strings.TrimSpace(probe.Status) == "present" { - hasPresent = true - } - switch strings.TrimSpace(probe.ErrorKind) { - case "credential_missing": - hasBrokenBucketMapping = true + if resolution.hasPresentProbe { + if resolution.hasPresentRawAccessProbe && resolution.hasBrokenRawAccessProbe { + return storageFindingBrokenBucketMap } - switch strings.TrimSpace(probe.Status) { - case "not_found": - hasMissing = true - case "forbidden", "unsupported", "invalid", "error": - hasProbeError = true + if resolution.hasPresentRawAccessProbe && resolution.hasMissingRawAccessProbe { + return storageFindingObjectMissing } - if strings.TrimSpace(probe.ValidationStatus) == "mismatched" { - return storageFindingValidationMismatch + if resolution.hasPresentRawAccessProbe && resolution.hasRawAccessProbeError { + return storageFindingProbeError } - } - if hasBucketMatch || hasPresent { return storageFindingNone } - if hasBrokenBucketMapping { + if resolution.hasBrokenMappingProbe { return storageFindingBrokenBucketMap } - if hasMissing { + if resolution.hasMissingProbe { return storageFindingObjectMissing } - if hasProbeError { + if resolution.hasProbeError { return storageFindingProbeError } return storageFindingNone } -func hasExactPathBucketMismatch(record projectRecordState, bucketMatches []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { - if len(bucketObjectsByURL) == 0 || len(bucketMatches) > 0 { +func hasExactPathBucketMismatch(record projectRecordState, resolution recordStorageResolution, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + if len(bucketObjectsByURL) == 0 || len(resolution.matchedBucketObjectURLs) > 0 { return false } - expectedBucketURLs := accessURLsForStorage(record) + expectedBucketURLs := resolution.candidateURLs if len(expectedBucketURLs) == 0 { return false } @@ -2490,15 +2482,6 @@ func hasExactPathBucketMismatch(record projectRecordState, bucketMatches []strin return false } -func recordHasValidationMismatchProbe(record projectRecordState) bool { - for _, probe := range record.AccessProbes { - if storageProbeValidationMismatchIsSignificant(record, probe) { - return true - } - } - return false -} - func storageProbeValidationMismatchIsSignificant(record projectRecordState, probe gintegrationsyfon.BulkStorageProbeResult) bool { if strings.TrimSpace(probe.ValidationStatus) != "mismatched" { return false @@ -2524,41 +2507,6 @@ func storageProbeValidationMismatchIsSignificant(record projectRecordState, prob return !storageSizesMatchForAudit(record.Size, *probe.SizeBytes) } -func recordHasMatchedCanonicalAccessProbe(record projectRecordState) bool { - rawURLs := make(map[string]struct{}, len(record.AccessURLs)) - for _, accessURL := range record.AccessURLs { - if trimmed := strings.TrimSpace(accessURL); trimmed != "" { - rawURLs[trimmed] = struct{}{} - } - } - canonicalURLs := make(map[string]struct{}, len(record.CanonicalAccessURLs)) - for _, accessURL := range record.CanonicalAccessURLs { - trimmed := strings.TrimSpace(accessURL) - if trimmed == "" { - continue - } - if _, raw := rawURLs[trimmed]; !raw { - canonicalURLs[trimmed] = struct{}{} - } - } - if len(canonicalURLs) == 0 { - return false - } - for _, probe := range record.AccessProbes { - if strings.TrimSpace(probe.Status) != "present" { - continue - } - if _, ok := canonicalURLs[strings.TrimSpace(probe.ObjectURL)]; !ok { - continue - } - switch strings.TrimSpace(probe.ValidationStatus) { - case "", "not_requested", "matched": - return true - } - } - return false -} - func inventoryHasValidationMismatch(record projectRecordState, bucketObjectURLs []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { if len(bucketObjectURLs) == 0 { return false @@ -2895,7 +2843,7 @@ func chainFindingError(kind string, record projectRecordState, probe GitStorageC func recordsReferenceBucketObject(matches []projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { for _, match := range matches { - if len(matchedBucketObjectURLs(match, bucketObjectsByURL)) > 0 { + if len(resolveRecordStorage(match, bucketObjectsByURL).matchedBucketObjectURLs) > 0 { return true } } @@ -2903,16 +2851,7 @@ func recordsReferenceBucketObject(matches []projectRecordState, bucketObjectsByU } func matchedBucketObjectURLs(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []string { - if len(bucketObjectsByURL) == 0 { - return nil - } - matches := make([]string, 0) - for _, objectURL := range recordBucketURLs(record) { - if _, ok := bucketObjectsByURL[objectURL]; ok { - matches = append(matches, objectURL) - } - } - return uniqueStrings(matches) + return resolveRecordStorage(record, bucketObjectsByURL).matchedBucketObjectURLs } func cleanupMatchedBucketObjectURLs(matches []projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []string { @@ -2933,18 +2872,23 @@ func cleanupMatchedBucketObjectURLs(matches []projectRecordState, bucketObjectsB return uniqueStrings(objectURLs) } -func hasCanonicalBucketURL(record projectRecordState) bool { - return len(recordBucketURLs(record)) > 0 +func recordBucketURLs(record projectRecordState) []string { + return recordStorageCandidateURLs(record) } -func recordBucketURLs(record projectRecordState) []string { +func recordStorageCandidateURLs(record projectRecordState) []string { out := make([]string, 0) for _, probe := range record.AccessProbes { if objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL); objectURL != "" { out = append(out, objectURL) } } - for _, accessURL := range accessURLsForStorage(record) { + for _, accessURL := range rawAccessURLsForRecord(record) { + if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { + out = append(out, objectURL) + } + } + for _, accessURL := range record.CanonicalAccessURLs { if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { out = append(out, objectURL) } @@ -2952,6 +2896,95 @@ func recordBucketURLs(record projectRecordState) []string { return uniqueStrings(out) } +type recordStorageResolution struct { + candidateURLs []string + matchedBucketObjectURLs []string + hasPresentProbe bool + hasMissingProbe bool + hasBrokenMappingProbe bool + hasProbeError bool + hasValidationMismatch bool + hasAcceptedCanonicalProbe bool + hasPresentRawAccessProbe bool + hasMissingRawAccessProbe bool + hasBrokenRawAccessProbe bool + hasRawAccessProbeError bool +} + +func resolveRecordStorage(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) recordStorageResolution { + resolution := recordStorageResolution{ + candidateURLs: recordStorageCandidateURLs(record), + } + rawURLs := make(map[string]struct{}, len(record.AccessURLs)) + rawCandidateURLs := make(map[string]struct{}, len(record.AccessURLs)) + for _, accessURL := range record.AccessURLs { + if trimmed := strings.TrimSpace(accessURL); trimmed != "" { + rawURLs[trimmed] = struct{}{} + if objectURL := canonicalStorageURL("", "", trimmed); objectURL != "" { + rawCandidateURLs[objectURL] = struct{}{} + } + } + } + canonicalURLs := make(map[string]struct{}, len(record.CanonicalAccessURLs)) + for _, accessURL := range record.CanonicalAccessURLs { + trimmed := strings.TrimSpace(accessURL) + if trimmed == "" { + continue + } + if _, raw := rawURLs[trimmed]; !raw { + canonicalURLs[trimmed] = struct{}{} + } + } + for _, probe := range record.AccessProbes { + objectURL := syfonProbeObjectURL(probe) + status := strings.TrimSpace(probe.Status) + _, rawAccessProbe := rawCandidateURLs[objectURL] + if status == "present" { + resolution.hasPresentProbe = true + if rawAccessProbe { + resolution.hasPresentRawAccessProbe = true + } + if _, ok := canonicalURLs[objectURL]; ok { + switch strings.TrimSpace(probe.ValidationStatus) { + case "", "not_requested", "matched": + resolution.hasAcceptedCanonicalProbe = true + } + } + } + if storageProbeValidationMismatchIsSignificant(record, probe) { + resolution.hasValidationMismatch = true + } + switch strings.TrimSpace(probe.ErrorKind) { + case "credential_missing": + resolution.hasBrokenMappingProbe = true + if rawAccessProbe { + resolution.hasBrokenRawAccessProbe = true + } + } + switch status { + case "not_found": + resolution.hasMissingProbe = true + if rawAccessProbe { + resolution.hasMissingRawAccessProbe = true + } + case "forbidden", "unsupported", "invalid", "error": + resolution.hasProbeError = true + if rawAccessProbe { + resolution.hasRawAccessProbeError = true + } + } + } + if len(bucketObjectsByURL) > 0 { + for _, objectURL := range resolution.candidateURLs { + if _, ok := bucketObjectsByURL[objectURL]; ok { + resolution.matchedBucketObjectURLs = append(resolution.matchedBucketObjectURLs, objectURL) + } + } + resolution.matchedBucketObjectURLs = uniqueStrings(resolution.matchedBucketObjectURLs) + } + return resolution +} + func accessURLsForStorage(record projectRecordState) []string { if len(record.CanonicalAccessURLs) > 0 { return record.CanonicalAccessURLs @@ -3118,59 +3151,6 @@ func syfonProbeHasMismatch(probe gintegrationsyfon.BulkStorageProbeResult, misma return false } -func classifyRawAccessURLFindings(record projectRecordState) storageFindingKind { - if len(record.AccessProbes) == 0 { - return storageFindingNone - } - if recordHasMatchedCanonicalAccessProbe(record) { - return storageFindingNone - } - probesByURL := make(map[string][]gintegrationsyfon.BulkStorageProbeResult, len(record.AccessProbes)) - for _, probe := range record.AccessProbes { - probesByURL[strings.TrimSpace(probe.ObjectURL)] = append(probesByURL[strings.TrimSpace(probe.ObjectURL)], probe) - } - for _, accessURL := range rawAccessURLsForRecord(record) { - probes := probesByURL[accessURL] - if len(probes) == 0 { - continue - } - hasPresent := false - hasMissing := false - hasBrokenBucketMapping := false - hasProbeError := false - for _, probe := range probes { - if storageProbeValidationMismatchIsSignificant(record, probe) { - return storageFindingValidationMismatch - } - if strings.TrimSpace(probe.Status) == "present" { - hasPresent = true - } - switch strings.TrimSpace(probe.ErrorKind) { - case "credential_missing": - hasBrokenBucketMapping = true - } - switch strings.TrimSpace(probe.Status) { - case "not_found": - hasMissing = true - case "forbidden", "unsupported", "invalid", "error": - hasProbeError = true - } - } - if hasPresent { - continue - } - switch { - case hasBrokenBucketMapping: - return storageFindingBrokenBucketMap - case hasMissing: - return storageFindingObjectMissing - case hasProbeError: - return storageFindingProbeError - } - } - return storageFindingNone -} - func canonicalizeRecordAccessURLs(accessURLs []string, scopes []domain.StorageBucketScope, organization string, project string) []string { out := make([]string, 0, len(accessURLs)) for _, accessURL := range accessURLs { diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index e2b59c8..1ac97dc 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -721,18 +721,19 @@ func (service *StorageAnalyticsService) attachProjectStorageListValidations(ctx }, nil } -func attachProjectStorageInventoryValidations(recordSet *storageAuditRecordSet, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, scopes []domain.StorageBucketScope, organization string, project string) (*storageAuditRecordSet, error) { +func attachProjectStorageInventoryValidations(recordSet *storageAuditRecordSet, inventory []RepoInventoryFile, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, scopes []domain.StorageBucketScope, organization string, project string) (*storageAuditRecordSet, error) { if recordSet == nil { return nil, nil } inventoryBuckets := projectInventoryBuckets(bucketObjectsByURL) + repoPathsByChecksum := repoPathsByChecksumForInventory(inventory) attach := func(input map[string][]projectRecordState) (map[string][]projectRecordState, error) { out := make(map[string][]projectRecordState, len(input)) for checksum, group := range input { states := make([]projectRecordState, 0, len(group)) for _, record := range group { clone := record - probes, err := inventoryValidationProbesForRecord(record, bucketObjectsByURL, inventoryBuckets, scopes, organization, project) + probes, err := inventoryValidationProbesForRecord(record, repoPathsByChecksum[normalizeAnalyticsChecksum(record.Checksum)], bucketObjectsByURL, inventoryBuckets, scopes, organization, project) if err != nil { return nil, err } @@ -757,13 +758,31 @@ func attachProjectStorageInventoryValidations(recordSet *storageAuditRecordSet, }, nil } -func inventoryValidationProbesForRecord(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, inventoryBuckets map[string]struct{}, scopes []domain.StorageBucketScope, organization string, project string) ([]gintegrationsyfon.BulkStorageProbeResult, error) { +func repoPathsByChecksumForInventory(inventory []RepoInventoryFile) map[string][]string { + out := make(map[string][]string) + for _, item := range inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" { + continue + } + out[checksum] = append(out[checksum], item.RepoPath) + } + for checksum, paths := range out { + out[checksum] = uniqueStrings(paths) + } + return out +} + +func inventoryValidationProbesForRecord(record projectRecordState, repoPaths []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, inventoryBuckets map[string]struct{}, scopes []domain.StorageBucketScope, organization string, project string) ([]gintegrationsyfon.BulkStorageProbeResult, error) { probes := make([]gintegrationsyfon.BulkStorageProbeResult, 0) seen := make(map[string]struct{}) accessURLs, err := canonicalizeRecordAccessURLsForProjectInventory(record.AccessURLs, scopes, organization, project) if err != nil { return nil, err } + accessURLs = append(accessURLs, rawProjectBucketAccessURLsForInventory(record.AccessURLs, inventoryBuckets)...) + accessURLs = append(accessURLs, projectScopeRepoPathObjectURLs(repoPaths, scopes, organization, project)...) + accessURLs = uniqueStrings(accessURLs) for _, accessURL := range accessURLs { objectURL := canonicalStorageURL("", "", accessURL) if objectURL == "" { @@ -789,6 +808,54 @@ func inventoryValidationProbesForRecord(record projectRecordState, bucketObjects return probes, nil } +func rawProjectBucketAccessURLsForInventory(accessURLs []string, inventoryBuckets map[string]struct{}) []string { + out := make([]string, 0, len(accessURLs)) + for _, accessURL := range accessURLs { + objectURL := canonicalStorageURL("", "", accessURL) + if objectURL == "" { + continue + } + bucket, _, ok := parseStorageURL(objectURL) + if !ok { + continue + } + if len(inventoryBuckets) > 0 { + if _, ok := inventoryBuckets[bucket]; !ok { + continue + } + } + out = append(out, objectURL) + } + return uniqueStrings(out) +} + +func projectScopeRepoPathObjectURLs(repoPaths []string, scopes []domain.StorageBucketScope, organization string, project string) []string { + out := make([]string, 0, len(repoPaths)) + for _, repoPath := range repoPaths { + normalizedPath := normalizeRepoSubpath(repoPath) + if normalizedPath == "" { + continue + } + for _, scope := range scopes { + if !storageScopeApplies(scope, organization, project) { + continue + } + bucket, prefix, ok := parseStorageScopePath(scope) + if !ok || strings.TrimSpace(bucket) == "" { + continue + } + key := normalizedPath + if strings.TrimSpace(prefix) != "" { + key = path.Join(strings.Trim(strings.TrimSpace(prefix), "/"), normalizedPath) + } + if objectURL := canonicalStorageURL(bucket, key, ""); objectURL != "" { + out = append(out, objectURL) + } + } + } + return uniqueStrings(out) +} + func projectInventoryBuckets(bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) map[string]struct{} { out := make(map[string]struct{}) for objectURL, item := range bucketObjectsByURL { @@ -906,7 +973,7 @@ func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context return view, nil } -func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, forceRefresh bool, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { +func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, inventory []RepoInventoryFile, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, forceRefresh bool, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) view := &storageAuditStorageView{ scopes: append([]domain.StorageBucketScope(nil), scopes...), @@ -941,7 +1008,7 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex } view.bucketObjects, view.bucketObjectsByURL = cloneBucketInventory(validationObjects, validationObjectsByURL) validateStart := time.Now() - probedRecordSet, err := attachProjectStorageInventoryValidations(recordSet, view.bucketObjectsByURL, scopes, organization, project) + probedRecordSet, err := attachProjectStorageInventoryValidations(recordSet, inventory, view.bucketObjectsByURL, scopes, organization, project) if err != nil { return nil, err } @@ -1080,7 +1147,8 @@ func selectTargetedProbeRecordSet(recordSet *storageAuditRecordSet, bucketObject } func recordNeedsTargetedProbe(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { - if len(matchedBucketObjectURLs(record, bucketObjectsByURL)) == 0 { + resolution := resolveRecordStorage(record, bucketObjectsByURL) + if len(resolution.matchedBucketObjectURLs) == 0 { return true } if !sameStringSet(rawAccessURLsForRecord(record), accessURLsForStorage(record)) { @@ -1090,8 +1158,7 @@ func recordNeedsTargetedProbe(record projectRecordState, bucketObjectsByURL map[ if checksum == "" { return false } - matches := matchedBucketObjectURLs(record, bucketObjectsByURL) - for _, objectURL := range matches { + for _, objectURL := range resolution.matchedBucketObjectURLs { item, ok := bucketObjectsByURL[objectURL] if !ok { continue @@ -1228,7 +1295,7 @@ func buildEquivalentRecordKeyIndex(records []projectRecordState) equivalentRecor unknownSizeByName: make(map[string]struct{}), } for _, record := range records { - for _, accessURL := range accessURLsForStorage(record) { + for _, accessURL := range recordStorageCandidateURLs(record) { _, key, ok := parseStorageURL(accessURL) if !ok { continue diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 5cd052e..93c40b5 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -2846,6 +2846,82 @@ func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) } } +func TestBuildStorageChainAuditAcceptsMappedNameAndSourcePathInventoryObjects(t *testing.T) { + sourceChecksum := "610cad76a473c4b0bf43f923b3bb907aac9ac2630d9c3f21aba1c531953c43e4" + mappedChecksum := "b71c3fcf28a6d2cf6e432b21a1734786708517f4808c22f9114abb73a60f4cc7" + sourcePath := "OHSU/koei_chin/visium_hd/1-R1/outs/binned_outputs/square_002um/spatial/tissue_positions.parquet" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + sourcePath: lfsPointer(sourceChecksum, 2048), + "META/Research.ndjson": lfsPointer(mappedChecksum, 4722), + }) + now := time.Date(2026, 7, 8, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-source-path", + Name: "tissue_positions.parquet", + Checksum: sourceChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: 2048, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc/" + sourceChecksum}, + }, + { + ObjectID: "obj-mapped-name", + Name: "ResearchSubject.ndjson", + Checksum: mappedChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: 4722, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc/" + mappedChecksum}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + { + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://bforepc/bforepc-prod/" + sourcePath, + Bucket: "bforepc", + Key: "bforepc-prod/" + sourcePath, + Path: sourcePath, + SizeBytes: 2048, + }, + { + ObjectURL: "s3://bforepc/" + mappedChecksum, + Bucket: "bforepc", + Key: mappedChecksum, + Path: mappedChecksum, + SizeBytes: 4722, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 2 { + t.Fatalf("expected both source-path and mapped-name objects to complete chains, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 0 { + t.Fatalf("expected no missing bucket findings when either candidate exists, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_missing_bucket_object"]; got != 0 { + t.Fatalf("expected no Syfon-only missing bucket findings when either candidate exists, got summary %+v", chain.Summary) + } +} + func TestBuildStorageChainAuditCanonicalizesStaleRecordBucketUsingAuditProjectScopes(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/file.bin": lfsPointer("dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", 100), From 6d15a575fe91ed2f658534c553b91d3a22c0dad9 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 9 Jul 2026 08:47:02 -0700 Subject: [PATCH 30/36] remove false positives --- internal/git/storage_analytics.go | 32 +++------ internal/git/storage_analytics_pipeline.go | 30 +++++--- internal/git/storage_analytics_test.go | 79 ++++++++++++++++++++++ 3 files changed, 111 insertions(+), 30 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index f118165..7a5805c 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -1777,7 +1777,7 @@ func (service *StorageAnalyticsService) attachStorageValidationResults(ctx conte probes = append(probes, result) } } - clone.AccessProbes = probes + clone.AccessProbes = append(append([]gintegrationsyfon.BulkStorageProbeResult(nil), record.AccessProbes...), probes...) states = append(states, clone) } out[checksum] = states @@ -2878,19 +2878,21 @@ func recordBucketURLs(record projectRecordState) []string { func recordStorageCandidateURLs(record projectRecordState) []string { out := make([]string, 0) - for _, probe := range record.AccessProbes { - if objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL); objectURL != "" { + for _, accessURL := range record.CanonicalAccessURLs { + if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { out = append(out, objectURL) } } - for _, accessURL := range rawAccessURLsForRecord(record) { - if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { + for _, probe := range record.AccessProbes { + if objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL); objectURL != "" { out = append(out, objectURL) } } - for _, accessURL := range record.CanonicalAccessURLs { - if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { - out = append(out, objectURL) + if len(record.CanonicalAccessURLs) == 0 { + for _, accessURL := range rawAccessURLsForRecord(record) { + if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { + out = append(out, objectURL) + } } } return uniqueStrings(out) @@ -3183,20 +3185,6 @@ func canonicalizeRecordAccessURLMappings(accessURLs []string, scopes []domain.St return out } -func canonicalizeRecordAccessURLsForProjectInventory(accessURLs []string, scopes []domain.StorageBucketScope, organization string, project string) ([]string, error) { - out := make([]string, 0, len(accessURLs)) - for _, accessURL := range accessURLs { - if objectURL := canonicalizeScopedStorageURL(accessURL, scopes, organization, project); objectURL != "" { - out = append(out, objectURL) - continue - } - if _, _, ok := parseStorageURL(accessURL); ok { - return nil, fmt.Errorf("storage access URL %q could not be mapped into bucket scopes for project %s/%s", strings.TrimSpace(accessURL), strings.TrimSpace(organization), strings.TrimSpace(project)) - } - } - return uniqueStrings(out), nil -} - func canonicalizeScopedStorageURL(accessURL string, scopes []domain.StorageBucketScope, organization string, project string) string { if len(scopes) == 0 { return "" diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 1ac97dc..6fc15bc 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -776,11 +776,10 @@ func repoPathsByChecksumForInventory(inventory []RepoInventoryFile) map[string][ func inventoryValidationProbesForRecord(record projectRecordState, repoPaths []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, inventoryBuckets map[string]struct{}, scopes []domain.StorageBucketScope, organization string, project string) ([]gintegrationsyfon.BulkStorageProbeResult, error) { probes := make([]gintegrationsyfon.BulkStorageProbeResult, 0) seen := make(map[string]struct{}) - accessURLs, err := canonicalizeRecordAccessURLsForProjectInventory(record.AccessURLs, scopes, organization, project) + accessURLs, err := recordAccessURLsForProjectInventory(record.AccessURLs, inventoryBuckets, scopes, organization, project) if err != nil { return nil, err } - accessURLs = append(accessURLs, rawProjectBucketAccessURLsForInventory(record.AccessURLs, inventoryBuckets)...) accessURLs = append(accessURLs, projectScopeRepoPathObjectURLs(repoPaths, scopes, organization, project)...) accessURLs = uniqueStrings(accessURLs) for _, accessURL := range accessURLs { @@ -808,9 +807,14 @@ func inventoryValidationProbesForRecord(record projectRecordState, repoPaths []s return probes, nil } -func rawProjectBucketAccessURLsForInventory(accessURLs []string, inventoryBuckets map[string]struct{}) []string { +func recordAccessURLsForProjectInventory(accessURLs []string, inventoryBuckets map[string]struct{}, scopes []domain.StorageBucketScope, organization string, project string) ([]string, error) { out := make([]string, 0, len(accessURLs)) for _, accessURL := range accessURLs { + mapped := false + if objectURL := canonicalizeScopedStorageURL(accessURL, scopes, organization, project); objectURL != "" { + out = append(out, objectURL) + mapped = true + } objectURL := canonicalStorageURL("", "", accessURL) if objectURL == "" { continue @@ -820,13 +824,16 @@ func rawProjectBucketAccessURLsForInventory(accessURLs []string, inventoryBucket continue } if len(inventoryBuckets) > 0 { - if _, ok := inventoryBuckets[bucket]; !ok { - continue + if _, ok := inventoryBuckets[bucket]; ok { + out = append(out, objectURL) + mapped = true } } - out = append(out, objectURL) + if !mapped { + return nil, fmt.Errorf("storage access URL %q could not be mapped into bucket scopes for project %s/%s", strings.TrimSpace(accessURL), strings.TrimSpace(organization), strings.TrimSpace(project)) + } } - return uniqueStrings(out) + return uniqueStrings(out), nil } func projectScopeRepoPathObjectURLs(repoPaths []string, scopes []domain.StorageBucketScope, organization string, project string) []string { @@ -1339,6 +1346,13 @@ func bucketObjectHasEquivalentSyfonRecord(item gintegrationsyfon.ProjectBucketOb return false } +func recordHasResolvedStorage(record projectRecordState, bucketMatches []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + if len(bucketMatches) > 0 { + return true + } + return resolveRecordStorage(record, bucketObjectsByURL).hasPresentProbe +} + func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccumulator, countCompleteFromSyfon bool) { for _, record := range index.allRecords { gitPaths := uniqueStrings(index.repoPathsByChecksum[normalizeAnalyticsChecksum(record.Checksum)]) @@ -1373,7 +1387,7 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum acc.findings = append(acc.findings, findings...) acc.addCount("probe_error", chainPathCount(gitPaths)) case storageFindingNone: - if countCompleteFromSyfon && len(gitPaths) > 0 && len(bucketMatches) > 0 { + if countCompleteFromSyfon && len(gitPaths) > 0 && recordHasResolvedStorage(record, bucketMatches, index.bucketObjectsByURL) { acc.addCount("bucket_syfon_git_complete", len(gitPaths)) continue } diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 93c40b5..4e96a89 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -2922,6 +2922,85 @@ func TestBuildStorageChainAuditAcceptsMappedNameAndSourcePathInventoryObjects(t } } +func TestBuildStorageChainAuditUsesCanonicalInventoryCandidatesBeforeMissingBucket(t *testing.T) { + sourceChecksum := "db458f223d03fddc541e2ad3c93ff3c152a20ecaa636cdfcb031cff6fe0c6b3a" + mappedChecksum := "b8639eca6eed65224f3af4c70425f578b949ee7202051388c331404cf35073f6" + sourcePath := "OHSU/koei_chin/visium_hd/4-R2/outs/segmented_outputs/analysis/diffexp/gene_expression_kmeans_10_clusters/differential_expression.csv" + mappedRepoPath := "OHSU/koei_chin/visium_hd/4-R2/outs/segmented_outputs/analysis/diffexp/gene_expression_kmeans_2_clusters/differential_expression.csv" + mappedRawURL := "s3://bforepc-prod/97a32a3a-1a30-5e14-bd94-7bf65ac98f27/" + mappedChecksum + sourceRawURL := "s3://bforepc-prod/" + sourcePath + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + sourcePath: lfsPointer(sourceChecksum, 1200), + mappedRepoPath: lfsPointer(mappedChecksum, 1300), + }) + now := time.Date(2026, 7, 9, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-source", + Name: "differential_expression.csv", + Checksum: sourceChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: 1200, + UpdatedAt: &now, + AccessURLs: []string{sourceRawURL}, + }, + { + ObjectID: "obj-mapped", + Name: "differential_expression.csv", + Checksum: mappedChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: 1300, + UpdatedAt: &now, + AccessURLs: []string{mappedRawURL}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + { + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://bforepc/bforepc-prod/" + sourcePath, + Bucket: "bforepc", + Key: "bforepc-prod/" + sourcePath, + Path: sourcePath, + SizeBytes: 1200, + }, + { + ObjectURL: "s3://bforepc/bforepc-prod/" + mappedRepoPath, + Bucket: "bforepc", + Key: "bforepc-prod/" + mappedRepoPath, + Path: mappedRepoPath, + SizeBytes: 1300, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 2 { + t.Fatalf("expected canonical inventory candidates to complete both chains, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 0 { + t.Fatalf("expected no Git+Syfon missing bucket findings after canonical inventory matching, got summary %+v", chain.Summary) + } + if backend.listProbeCalls != 0 { + t.Fatalf("expected no Syfon bulk-list fallback call, got %d calls with items %+v", backend.listProbeCalls, backend.listProbeItems) + } +} + func TestBuildStorageChainAuditCanonicalizesStaleRecordBucketUsingAuditProjectScopes(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/file.bin": lfsPointer("dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", 100), From aec3f3c1730591a6ee892eaa4a9fe096c0601322 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 9 Jul 2026 16:26:06 -0700 Subject: [PATCH 31/36] refactor audit to be better --- internal/git/storage_analytics.go | 330 +++++++++------ internal/git/storage_analytics_pipeline.go | 356 +++++++++++++--- internal/git/storage_analytics_test.go | 390 +++++++++++++++--- internal/git/storage_audit_evidence.go | 56 +++ .../storage_audit_list_confirmation_test.go | 106 +++++ internal/git/storage_audit_registration.go | 233 +++++++++++ .../git/storage_audit_registration_test.go | 88 ++++ internal/git/storage_audit_revalidation.go | 147 +++++++ .../git/storage_audit_revalidation_test.go | 191 +++++++++ internal/git/storage_chain_audit_cache.go | 54 ++- internal/git/types.go | 296 ++----------- internal/integrations/syfon/adapter.go | 69 ++++ internal/integrations/syfon/adapter_test.go | 52 +++ internal/server/http/git/project.go | 15 +- internal/server/http/git/project_test.go | 43 ++ internal/server/http/git/register.go | 1 + internal/server/http/git/repository.go | 6 +- internal/server/http/git/storage_analytics.go | 50 ++- internal/storageaudit/evidence.go | 80 ++++ internal/storageaudit/evidence_test.go | 61 +++ internal/storageaudit/policy.go | 15 + internal/storageaudit/types.go | 301 ++++++++++++++ 22 files changed, 2432 insertions(+), 508 deletions(-) create mode 100644 internal/git/storage_audit_evidence.go create mode 100644 internal/git/storage_audit_list_confirmation_test.go create mode 100644 internal/git/storage_audit_registration.go create mode 100644 internal/git/storage_audit_registration_test.go create mode 100644 internal/git/storage_audit_revalidation.go create mode 100644 internal/git/storage_audit_revalidation_test.go create mode 100644 internal/server/http/git/project_test.go create mode 100644 internal/storageaudit/evidence.go create mode 100644 internal/storageaudit/evidence_test.go create mode 100644 internal/storageaudit/policy.go create mode 100644 internal/storageaudit/types.go diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 7a5805c..492b27d 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -37,6 +37,7 @@ const ( storageActionDeleteBucketObject = "delete_bucket_object" storageActionDeleteBoth = "delete_both" storageActionInspectEvidence = "inspect_evidence" + storageActionCreateSyfonRecord = "create_syfon_record" ) type storageRepairPolicy struct { @@ -50,13 +51,27 @@ func storageRepairPolicyForKind(kind string) storageRepairPolicy { switch strings.TrimSpace(kind) { case "bucket_only_object": return autoRepairPolicy(storageActionDeleteBucketObject, storageActionInspectEvidence) - case "bucket_syfon_no_git", "repo_orphan_live_object": + case "bucket_syfon_no_git": + // Absence from Git is not sufficient to delete storage automatically, but + // a user may explicitly remove a verified Bucket + Syfon orphan. + return storageRepairPolicy{ + actionability: storageActionabilityManualChoice, + actions: []string{ + storageActionDeleteBoth, + storageActionDeleteSyfonRecord, + storageActionDeleteBucketObject, + storageActionInspectEvidence, + }, + defaultAction: storageActionDeleteBoth, + supportsDryRun: true, + } + case "repo_orphan_live_object": return autoRepairPolicy(storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionInspectEvidence) case "repo_orphan_stale_record", "stale_duplicate_record", "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object": return autoRepairPolicy(storageActionDeleteSyfonRecord, storageActionInspectEvidence) case "broken_access_url_error", "broken_bucket_mapping", "syfon_broken_bucket_mapping": return autoRepairPolicy(storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord, storageActionInspectEvidence) - case "storage_validation_mismatch", "git_syfon_metadata_mismatch": + case "storage_validation_mismatch": return storageRepairPolicy{ actionability: storageActionabilityManualChoice, actions: []string{ @@ -68,6 +83,15 @@ func storageRepairPolicyForKind(kind string) storageRepairPolicy { }, supportsDryRun: true, } + case "git_syfon_metadata_mismatch": + // A mismatch cannot establish which of Git, Syfon, or the bucket is + // authoritative. Requiring an explicit out-of-band reconciliation keeps + // a stale checksum from becoming a destructive cleanup candidate. + return storageRepairPolicy{ + actionability: storageActionabilityInspectOnly, + actions: []string{storageActionInspectEvidence}, + defaultAction: storageActionInspectEvidence, + } default: return storageRepairPolicy{ actionability: storageActionabilityInspectOnly, @@ -106,6 +130,7 @@ type storageAnalyticsBackend interface { BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error DeleteProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, objectURLs []string) ([]gintegrationsyfon.ProjectBucketDeleteResult, error) BulkUpdateAccessMethods(ctx context.Context, authorizationHeader string, updates map[string][]gintegrationsyfon.ProjectAccessMethod) error + RegisterProjectObjects(ctx context.Context, authorizationHeader string, candidates []gintegrationsyfon.ProjectObjectRegistration) ([]gintegrationsyfon.ProjectObjectRegistrationResult, error) } type StorageAnalyticsService struct { @@ -117,6 +142,8 @@ type StorageAnalyticsService struct { chainInputCache map[string]cachedChainInputState projectAuditCache map[string]cachedProjectAuditRecordState projectAuditWork map[string]*inflightProjectAuditRecordState + chainAuditRefreshMu sync.Mutex + chainAuditRefreshWork map[string]*inflightStorageChainAuditRefresh chainAuditResponseCache storageChainAuditResponseCache exactProjectJoinCache storageExactProjectJoinCache } @@ -138,12 +165,13 @@ func NewStorageAnalyticsService(storage storageAnalyticsBackend) *StorageAnalyti return nil } return &StorageAnalyticsService{ - storage: storage, - projectJoinCache: map[string]cachedProjectJoinState{}, - projectJoinWork: map[string]*inflightProjectJoinState{}, - chainInputCache: map[string]cachedChainInputState{}, - projectAuditCache: map[string]cachedProjectAuditRecordState{}, - projectAuditWork: map[string]*inflightProjectAuditRecordState{}, + storage: storage, + projectJoinCache: map[string]cachedProjectJoinState{}, + projectJoinWork: map[string]*inflightProjectJoinState{}, + chainInputCache: map[string]cachedChainInputState{}, + projectAuditCache: map[string]cachedProjectAuditRecordState{}, + projectAuditWork: map[string]*inflightProjectAuditRecordState{}, + chainAuditRefreshWork: map[string]*inflightStorageChainAuditRefresh{}, } } @@ -207,6 +235,7 @@ type storageCleanupApplyPlan struct { RepoDeletePaths []string ManualPaths []string SkippedPaths []string + Verifications []storageCleanupVerification } type cleanupAuditModel struct { @@ -456,7 +485,7 @@ func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx co return service.buildStorageChainAuditWithResponseCache(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, normalized) } if service.chainAuditResponseCache != nil && storageChainAuditRootResponseProjectionAllowed(gitSubpath, normalized) { - response, ok, err := service.projectStorageChainAuditFromRootResponseCache(ctx, organization, project, ref, gitSubpath, mirrorPath, repo, hash, normalized) + response, ok, err := service.projectStorageChainAuditFromRootResponseCache(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, normalized) if err != nil { return nil, err } @@ -508,7 +537,11 @@ func normalizeStorageChainAuditOptions(options StorageChainAuditOptions) (Storag } func (service *StorageAnalyticsService) buildStorageChainAuditWithResponseCache(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, error) { - cacheKey := storageChainAuditResponseCacheKey(organization, project, ref, gitSubpath, options.ProbeMode, options.ValidationMode, options.BucketInventoryMode, options.BucketPathPrefix, hash.String()) + syfonRevision, err := service.loadStorageAuditSyfonRevision(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + cacheKey := storageChainAuditResponseCacheKey(organization, project, ref, gitSubpath, options.ProbeMode, options.ValidationMode, options.BucketInventoryMode, options.BucketPathPrefix, hash.String(), syfonRevision) cache := service.chainAuditResponseCache if !options.ForceAuditRefresh { start := time.Now() @@ -519,7 +552,7 @@ func (service *StorageAnalyticsService) buildStorageChainAuditWithResponseCache( } if ok { response := projectStorageChainAuditResponse(cached.Response, options.FindingKind, options.FindingLimit) - applyStorageChainAuditCacheMetadata(response, true, cached.CachedAt, cache.Source(), "") + applyStorageChainAuditCacheMetadata(response, true, cached.CachedAt, cached.RefreshDurationMillis, cache.Source(), "") options.Timings.Record("audit_response_cache_hit", 0) options.Timings.RecordMemory( "audit_response_cache_hit", @@ -534,30 +567,50 @@ func (service *StorageAnalyticsService) buildStorageChainAuditWithResponseCache( options.Timings.Record("audit_response_cache_force_refresh", 0) } - buildOptions := options - buildOptions.FindingKind = "" - buildOptions.FindingLimit = -1 - response, err := service.buildStorageChainAuditFresh(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, buildOptions) + cached, joinedRefresh, err := service.coalesceStorageChainAuditRefresh(ctx, cacheKey, func() (cachedStorageChainAuditResponse, error) { + buildOptions := options + buildOptions.FindingKind = "" + buildOptions.FindingLimit = -1 + refreshStart := time.Now() + response, err := service.buildStorageChainAuditFresh(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, buildOptions) + if err != nil { + return cachedStorageChainAuditResponse{}, err + } + response.Summary.SyfonRevision = syfonRevision + value := cachedStorageChainAuditResponse{ + CachedAt: time.Now(), + RefreshDurationMillis: time.Since(refreshStart).Milliseconds(), + Response: *response, + } + cacheStart := time.Now() + if err := cache.Set(ctx, cacheKey, value, storageChainAuditCacheTTL()); err != nil { + logStorageChainAuditCacheError(options.Timings, cache.Source(), "set", err) + } + options.Timings.Record("audit_response_cache_store", time.Since(cacheStart)) + return value, nil + }) if err != nil { return nil, err } - cachedAt := time.Now() - cacheStart := time.Now() - setErr := cache.Set(ctx, cacheKey, cachedStorageChainAuditResponse{CachedAt: cachedAt, Response: *response}, storageChainAuditCacheTTL()) - options.Timings.Record("audit_response_cache_store", time.Since(cacheStart)) - cacheError := "" - if setErr != nil { - cacheError = setErr.Error() - logStorageChainAuditCacheError(options.Timings, cache.Source(), "set", setErr) - } - projected := projectStorageChainAuditResponse(*response, options.FindingKind, options.FindingLimit) - applyStorageChainAuditCacheMetadata(projected, false, cachedAt, cache.Source(), cacheError) + if joinedRefresh { + options.Timings.Record("audit_response_refresh_join", 0) + } + projected := projectStorageChainAuditResponse(cached.Response, options.FindingKind, options.FindingLimit) + source := cache.Source() + if joinedRefresh { + source += ":refresh_join" + } + applyStorageChainAuditCacheMetadata(projected, false, cached.CachedAt, cached.RefreshDurationMillis, source, "") return projected, nil } -func (service *StorageAnalyticsService) projectStorageChainAuditFromRootResponseCache(ctx context.Context, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, bool, error) { +func (service *StorageAnalyticsService) projectStorageChainAuditFromRootResponseCache(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, bool, error) { cache := service.chainAuditResponseCache - cacheKey := storageChainAuditResponseCacheKey(organization, project, ref, "", options.ProbeMode, options.ValidationMode, options.BucketInventoryMode, "", hash.String()) + syfonRevision, err := service.loadStorageAuditSyfonRevision(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, false, err + } + cacheKey := storageChainAuditResponseCacheKey(organization, project, ref, "", options.ProbeMode, options.ValidationMode, options.BucketInventoryMode, "", hash.String(), syfonRevision) start := time.Now() cached, ok, err := cache.Get(ctx, cacheKey) options.Timings.Record("audit_response_root_cache_lookup", time.Since(start)) @@ -576,7 +629,7 @@ func (service *StorageAnalyticsService) projectStorageChainAuditFromRootResponse return nil, false, err } response := projectStorageChainAuditResponseForSubpath(cached.Response, gitSubpath, inventory, options.FindingKind, options.FindingLimit) - applyStorageChainAuditCacheMetadata(response, true, cached.CachedAt, cache.Source()+":root", "") + applyStorageChainAuditCacheMetadata(response, true, cached.CachedAt, cached.RefreshDurationMillis, cache.Source()+":root", "") options.Timings.Record("audit_response_root_cache_hit", 0) options.Timings.RecordMemory( "audit_response_root_cache_hit", @@ -594,7 +647,7 @@ func (service *StorageAnalyticsService) buildStorageChainAuditFresh(ctx context. bucketPathPrefix := options.BucketPathPrefix start := time.Now() options.Timings.StageStart("chain_setup_total") - inputs, err := service.loadStorageChainInputs(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, bucketMode, bucketPathPrefix, options.Timings) + inputs, err := service.loadStorageChainInputs(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, bucketMode, validationMode, bucketPathPrefix, options.ForceAuditRefresh, options.Timings) options.Timings.Record("chain_setup_total", time.Since(start)) if inputs != nil && inputs.recordSet != nil { options.Timings.RecordMemory( @@ -609,7 +662,7 @@ func (service *StorageAnalyticsService) buildStorageChainAuditFresh(ctx context. } storageViewStart := time.Now() options.Timings.StageStart("storage_view") - storageView, err := service.buildStorageChainView(ctx, authorizationHeader, organization, project, inputs.recordSet, inputs.inventory, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.ForceAuditRefresh, options.Timings) + storageView, err := service.buildStorageChainView(ctx, authorizationHeader, organization, project, inputs.recordSet, inputs.inventory, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.Timings) options.Timings.Record("storage_view", time.Since(storageViewStart)) if storageView != nil { options.Timings.RecordMemory( @@ -624,7 +677,7 @@ func (service *StorageAnalyticsService) buildStorageChainAuditFresh(ctx context. modelStart := time.Now() options.Timings.StageStart("model_build") includeBucketOrigin := bucketMode != StorageChainBucketModeValidate && storageView.bucketInventoryAvailable - model := buildStorageChainAuditModel(gitSubpath, inputs.inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL, includeBucketOrigin) + model := buildStorageChainAuditModel(gitSubpath, inputs.inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL, inputs.scopes, organization, project, includeBucketOrigin) options.Timings.Record("model_build", time.Since(modelStart)) options.Timings.RecordMemory( "model_build", @@ -636,6 +689,8 @@ func (service *StorageAnalyticsService) buildStorageChainAuditFresh(ctx context. model.Summary.BucketInventoryAvailable = storageView.bucketInventoryAvailable model.Summary.BucketInventoryError = storageView.bucketInventoryError model.Summary.ValidationMode = validationMode + model.Summary.GitRevision = hash.String() + model.Summary.ObservedAt = time.Now().UTC().Format(time.RFC3339Nano) if inputs.bucketSummary != nil { exists := inputs.bucketSummary.Exists model.Summary.BucketPathExists = &exists @@ -805,13 +860,14 @@ func logStorageChainAuditCacheError(timings *StorageChainAuditTimings, source st timings.Logf("storage_chain_audit_cache_error %s source=%s operation=%s error=%q", strings.TrimSpace(timings.DebugPrefix), strings.TrimSpace(source), strings.TrimSpace(operation), err.Error()) } -func applyStorageChainAuditCacheMetadata(response *GitStorageChainAuditResponse, hit bool, cachedAt time.Time, source string, cacheError string) { +func applyStorageChainAuditCacheMetadata(response *GitStorageChainAuditResponse, hit bool, cachedAt time.Time, refreshDurationMillis int64, source string, cacheError string) { if response == nil || cachedAt.IsZero() { return } response.Summary.AuditCacheHit = hit response.Summary.AuditCachedAt = cachedAt.UTC().Format(time.RFC3339Nano) response.Summary.AuditCacheAgeSeconds = int64(time.Since(cachedAt).Seconds()) + response.Summary.AuditRefreshDurationMs = refreshDurationMillis response.Summary.AuditCacheSource = strings.TrimSpace(source) response.Summary.AuditCacheError = strings.TrimSpace(cacheError) } @@ -867,12 +923,16 @@ func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, if len(selectedFindings) == 0 { return nil, fmt.Errorf("cleanup apply requires findings from a prior audit; refusing to rebuild audit during apply") } + canonicalFindings, err := service.canonicalizeStorageCleanupFindings(ctx, authorizationHeader, organization, project, selectedFindings) + if err != nil { + return nil, err + } actionSelection := indexCleanupActions(selectedActions) plan := storageCleanupApplyPlan{ UpdateAccessMethods: make(map[string][]gintegrationsyfon.ProjectAccessMethod), } selected := indexCleanupSelection(selectedRepoPaths) - for _, finding := range selectedFindings { + for _, finding := range canonicalFindings { if len(selected) > 0 && !storageApplyFindingSelected(selected, finding) { continue } @@ -890,6 +950,69 @@ func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, return service.executeStorageCleanupApplyPlan(ctx, authorizationHeader, organization, project, plan, dryRun) } +func (service *StorageAnalyticsService) canonicalizeStorageCleanupFindings(ctx context.Context, authorizationHeader string, organization string, project string, findings []GitStorageCleanupApplyFinding) ([]GitStorageCleanupApplyFinding, error) { + if !storageCleanupFindingsContainStorageURL(findings) { + return append([]GitStorageCleanupApplyFinding(nil), findings...), nil + } + scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("load project storage scopes for cleanup apply: %w", err) + } + out := make([]GitStorageCleanupApplyFinding, 0, len(findings)) + for _, finding := range findings { + clone := finding + clone.BucketObjectURL = canonicalizeCleanupStorageURL(finding.BucketObjectURL, scopes, organization, project) + clone.BucketObjectURLs = canonicalizeCleanupStorageURLs(finding.BucketObjectURLs, scopes, organization, project) + clone.AccessURLs = canonicalizeCleanupStorageURLs(finding.AccessURLs, scopes, organization, project) + if finding.Evidence != nil { + evidence := *finding.Evidence + evidence.BucketObjectURLs = canonicalizeCleanupStorageURLs(evidence.BucketObjectURLs, scopes, organization, project) + evidence.AccessURLs = canonicalizeCleanupStorageURLs(evidence.AccessURLs, scopes, organization, project) + clone.Evidence = &evidence + } + out = append(out, clone) + } + return out, nil +} + +func storageCleanupFindingsContainStorageURL(findings []GitStorageCleanupApplyFinding) bool { + for _, finding := range findings { + candidates := []string{finding.NormalizedPath, finding.BucketObjectURL} + candidates = append(candidates, finding.BucketObjectURLs...) + candidates = append(candidates, finding.AccessURLs...) + if finding.Evidence != nil { + candidates = append(candidates, finding.Evidence.BucketObjectURLs...) + candidates = append(candidates, finding.Evidence.AccessURLs...) + } + for _, candidate := range candidates { + if _, _, ok := parseStorageURL(candidate); ok { + return true + } + } + } + return false +} + +func canonicalizeCleanupStorageURL(value string, scopes []domain.StorageBucketScope, organization string, project string) string { + if objectURL := canonicalizeScopedStorageURL(value, scopes, organization, project); objectURL != "" { + return objectURL + } + if objectURL := canonicalStorageURL("", "", value); objectURL != "" { + return objectURL + } + return strings.TrimSpace(value) +} + +func canonicalizeCleanupStorageURLs(values []string, scopes []domain.StorageBucketScope, organization string, project string) []string { + out := make([]string, 0, len(values)) + for _, value := range values { + if canonical := canonicalizeCleanupStorageURL(value, scopes, organization, project); canonical != "" { + out = append(out, canonical) + } + } + return uniqueStrings(out) +} + func resolveStorageCleanupApplyAction(finding GitStorageCleanupApplyFinding, actionSelection map[string]string, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool) (string, error) { kind := strings.TrimSpace(finding.Kind) if !knownStorageRepairKind(kind) { @@ -914,9 +1037,6 @@ func resolveStorageCleanupApplyAction(finding GitStorageCleanupApplyFinding, act if !storageRepairActionAllowed(kind, action) { return "", fmt.Errorf("cleanup action %q is not supported for finding kind %q", action, kind) } - if len(finding.AvailableActions) > 0 && !stringSliceContains(finding.AvailableActions, action) { - return "", fmt.Errorf("cleanup action %q is not advertised for finding kind %q at %q", action, kind, finding.NormalizedPath) - } return action, nil } @@ -980,6 +1100,9 @@ func stringSliceContains(values []string, target string) bool { } func addStorageCleanupApplyFindingToPlan(plan *storageCleanupApplyPlan, finding GitStorageCleanupApplyFinding, action string) error { + if err := appendStorageCleanupVerification(plan, finding, action); err != nil { + return err + } switch action { case storageActionInspectEvidence: plan.ManualPaths = append(plan.ManualPaths, finding.NormalizedPath) @@ -1209,6 +1332,11 @@ func (service *StorageAnalyticsService) executeStorageCleanupApplyPlan(ctx conte DryRun: true, }, nil } + if err := service.verifyStorageCleanupApplyPlan(ctx, authorizationHeader, &plan); err != nil { + return nil, err + } + toDeleteBucketObjects = uniqueStrings(plan.DeleteBucketObjects) + skippedPaths = uniqueStrings(plan.SkippedPaths) if len(plan.UpdateAccessMethods) > 0 { if err := service.storage.BulkUpdateAccessMethods(ctx, authorizationHeader, plan.UpdateAccessMethods); err != nil { return nil, fmt.Errorf("update syfon access methods: %w", err) @@ -1308,15 +1436,15 @@ func storageApplyFindingBucketObjectURLs(finding GitStorageCleanupApplyFinding) candidates := []string{ finding.BucketObjectURL, } - if canonicalStorageURL("", "", finding.NormalizedPath) != "" { - candidates = append(candidates, finding.NormalizedPath) - } candidates = append(candidates, finding.BucketObjectURLs...) candidates = append(candidates, finding.AccessURLs...) if finding.Evidence != nil { candidates = append(candidates, finding.Evidence.BucketObjectURLs...) candidates = append(candidates, finding.Evidence.AccessURLs...) } + if len(uniqueCleanupSelectionCandidates(candidates)) == 0 && canonicalStorageURL("", "", finding.NormalizedPath) != "" { + candidates = append(candidates, finding.NormalizedPath) + } return uniqueCleanupSelectionCandidates(candidates) } @@ -2389,96 +2517,45 @@ func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[st if hasExactPathBucketMismatch(record, resolution, bucketObjectsByURL) { return storageFindingBrokenBucketMap } - if len(repairableBrokenAccessProbes(record)) > 0 { - return storageFindingBrokenBucketMap - } + assessment := assessStorageRecordEvidence(record, len(resolution.matchedBucketObjectURLs) > 0) if resolution.hasAcceptedCanonicalProbe { return storageFindingNone } if inventoryHasValidationMismatch(record, resolution.matchedBucketObjectURLs, bucketObjectsByURL) { return storageFindingValidationMismatch } - if resolution.hasValidationMismatch { + if resolution.hasValidationMismatch || assessment.MetadataMismatch { return storageFindingValidationMismatch } - if len(resolution.matchedBucketObjectURLs) > 0 { - if resolution.hasPresentRawAccessProbe && resolution.hasBrokenRawAccessProbe { - return storageFindingBrokenBucketMap - } - if resolution.hasPresentRawAccessProbe && resolution.hasMissingRawAccessProbe { - return storageFindingObjectMissing - } - if resolution.hasPresentRawAccessProbe && resolution.hasRawAccessProbeError { - return storageFindingProbeError - } + if resolution.hasAcceptedCanonicalProbe || assessment.Present { return storageFindingNone } + if len(repairableBrokenAccessProbes(record)) > 0 || assessment.MappingBroken { + return storageFindingBrokenBucketMap + } if len(record.AccessProbes) == 0 { if len(bucketObjectsByURL) > 0 && len(resolution.candidateURLs) > 0 { return storageFindingProbeError } return storageFindingNone } - if resolution.hasPresentProbe { - if resolution.hasPresentRawAccessProbe && resolution.hasBrokenRawAccessProbe { - return storageFindingBrokenBucketMap - } - if resolution.hasPresentRawAccessProbe && resolution.hasMissingRawAccessProbe { - return storageFindingObjectMissing - } - if resolution.hasPresentRawAccessProbe && resolution.hasRawAccessProbeError { - return storageFindingProbeError - } - return storageFindingNone - } - if resolution.hasBrokenMappingProbe { - return storageFindingBrokenBucketMap - } - if resolution.hasMissingProbe { + if assessment.Missing { return storageFindingObjectMissing } - if resolution.hasProbeError { + if resolution.hasProbeError || assessment.Status == "unknown" { return storageFindingProbeError } return storageFindingNone } func hasExactPathBucketMismatch(record projectRecordState, resolution recordStorageResolution, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { - if len(bucketObjectsByURL) == 0 || len(resolution.matchedBucketObjectURLs) > 0 { - return false - } - expectedBucketURLs := resolution.candidateURLs - if len(expectedBucketURLs) == 0 { - return false - } - expectedBasenames := make(map[string]struct{}, len(expectedBucketURLs)) - for _, accessURL := range expectedBucketURLs { - _, key, ok := parseStorageURL(accessURL) - if !ok { - continue - } - name := strings.TrimSpace(path.Base(key)) - if name == "" { - continue - } - expectedBasenames[name] = struct{}{} - } - if len(expectedBasenames) == 0 { - return false - } - for _, item := range bucketObjectsByURL { - itemName := strings.TrimSpace(path.Base(strings.TrimSpace(item.Key))) - if itemName == "" { - continue - } - if _, ok := expectedBasenames[itemName]; !ok { - continue - } - if record.Size > 0 && item.SizeBytes > 0 && item.SizeBytes != record.Size { - continue - } - return true - } + // A basename or size match elsewhere in a project bucket is not evidence + // that this record's mapping is wrong: BForePC contains many repeated names + // such as features.tsv.gz. Mapping failures must come from an explicit + // resolver/probe error, not an inventory-side similarity heuristic. + _ = record + _ = resolution + _ = bucketObjectsByURL return false } @@ -2648,9 +2725,10 @@ func buildBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStora func buildChainBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStorageChainFinding { objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL) - actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupport("bucket_only_object") + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence("bucket_only_object", "verified") return GitStorageChainFinding{ Kind: "bucket_only_object", + EvidenceStatus: "verified", NormalizedPath: objectURL, ObjectIDs: []string{}, AccessURLs: []string{objectURL}, @@ -2699,7 +2777,8 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, if evidence != nil { evidence.BucketObjectURLs = uniqueStrings(append(evidence.BucketObjectURLs, bucketObjectURLs...)) } - actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupport(kind) + evidenceStatus := storageEvidenceStatus(record, len(bucketObjectURLs) > 0) + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence(kind, evidenceStatus) primaryProbe := selectChainProbe(record, bucketObjectURLs) suggestedFix := suggestedFixForChainFinding(kind, record) suggestedAction := suggestedActionForChainFinding(kind, record) @@ -2711,6 +2790,7 @@ func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, for _, path := range paths { findings = append(findings, GitStorageChainFinding{ Kind: kind, + EvidenceStatus: evidenceStatus, NormalizedPath: path, Checksum: strings.TrimSpace(record.Checksum), SourcePaths: uniqueStrings(gitPaths), @@ -2757,12 +2837,12 @@ func suggestedFixForChainFinding(kind string, record projectRecordState) string if probe.SizeBytes != nil { bucketSize = *probe.SizeBytes } - mismatches = append(mismatches, fmt.Sprintf("Syfon/Git size is %s, bucket inventory reports %s", formatAuditSize(record.Size), formatAuditSize(bucketSize))) + mismatches = append(mismatches, fmt.Sprintf("Syfon record size is %s, bucket inventory reports %s", formatAuditSize(record.Size), formatAuditSize(bucketSize))) } if len(mismatches) == 0 { return "Bucket object exists, but its storage evidence does not match the Syfon/Git record. Review the record and bucket object before applying a destructive fix." } - return strings.Join(mismatches, ". ") + ". Update the stale Syfon metadata or delete and recreate the record after confirming the bucket object is authoritative." + return strings.Join(mismatches, ". ") + ". Because the byte lengths differ, the bucket object cannot have the current Git/Syfon SHA-256. Recompute the bucket object's SHA-256 before manually reconciling Git and Syfon." } func recordHasAccessProbeMismatch(record projectRecordState, mismatch string) bool { @@ -2965,11 +3045,18 @@ func resolveRecordStorage(record projectRecordState, bucketObjectsByURL map[stri } switch status { case "not_found": + if strings.TrimSpace(probe.Operation) == StorageChainValidationModeInventory { + resolution.hasProbeError = true + if rawAccessProbe { + resolution.hasRawAccessProbeError = true + } + continue + } resolution.hasMissingProbe = true if rawAccessProbe { resolution.hasMissingRawAccessProbe = true } - case "forbidden", "unsupported", "invalid", "error": + case "unknown", "forbidden", "unsupported", "invalid", "error": resolution.hasProbeError = true if rawAccessProbe { resolution.hasRawAccessProbeError = true @@ -2995,10 +3082,21 @@ func accessURLsForStorage(record projectRecordState) []string { } func probeAccessURLsForRecord(record projectRecordState) []string { + urls := make([]string, 0, len(record.CanonicalAccessURLs)+len(record.AccessURLs)+len(record.AccessProbes)) if len(record.CanonicalAccessURLs) > 0 { - return record.CanonicalAccessURLs + urls = append(urls, record.CanonicalAccessURLs...) + } else { + urls = append(urls, rawAccessURLsForRecord(record)...) + } + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Operation) != StorageChainValidationModeInventory || strings.TrimSpace(probe.Status) != "unknown" { + continue + } + if objectURL := syfonProbeObjectURL(probe); objectURL != "" { + urls = append(urls, objectURL) + } } - return rawAccessURLsForRecord(record) + return uniqueStrings(urls) } func rawAccessURLsForRecord(record projectRecordState) []string { @@ -3134,11 +3232,7 @@ func syfonProbeObjectURL(probe gintegrationsyfon.BulkStorageProbeResult) string func syfonProbeIsBrokenAccess(probe gintegrationsyfon.BulkStorageProbeResult) bool { switch strings.TrimSpace(probe.ErrorKind) { - case "missing_access_url", "credential_missing": - return true - } - switch strings.TrimSpace(probe.Status) { - case "missing", "forbidden", "unsupported", "invalid", "error": + case "missing_access_url", "scope_not_found", "credential_missing": return true } return false diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 6fc15bc..d212dbf 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -42,6 +42,8 @@ type storageChainIndex struct { allRecords []projectRecordState bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject repoPathsByChecksum map[string][]string + repoPathsByBucketURL map[string][]string + repoChecksumsByPath map[string]string recordsByBucketURL map[string][]projectRecordState equivalentRecordKeys equivalentRecordKeyIndex } @@ -70,7 +72,7 @@ type storageChainInputs struct { bucketInventoryErr error } -func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, bucketMode string, bucketPathPrefix string, timings *StorageChainAuditTimings) (*storageChainInputs, error) { +func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, bucketMode string, validationMode string, bucketPathPrefix string, forceRefresh bool, timings *StorageChainAuditTimings) (*storageChainInputs, error) { type inventoryResult struct { inventory []RepoInventoryFile err error @@ -96,6 +98,14 @@ func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Conte bucketCh := make(chan bucketResult, 1) go func() { + if bucketMode == StorageChainBucketModeValidate && validationMode != StorageChainValidationModeList { + timings.Record("syfon_bucket_inventory_skipped", 0) + bucketCh <- bucketResult{ + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, + } + return + } start := time.Now() timings.StageStart("repo_index") inventory, err := service.loadStorageChainInventory(ctx, ref, gitSubpath, mirrorPath, repo, hash) @@ -125,24 +135,26 @@ func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Conte logStorageChainInputResult("syfon_project_scopes", len(scopes), err) scopeCh <- scopeResult{scopes: scopes, err: err} }() - if bucketMode == StorageChainBucketModeValidate { - bucketCh <- bucketResult{ - bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, - bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, - } - timings.Record("syfon_bucket_inventory_skipped", 0) - logStorageChainInputResult("syfon_bucket_inventory_skipped validate_mode", 0, nil) - } else { - go func() { - start := time.Now() - timings.StageStart("syfon_bucket_inventory") - bucketObjects, bucketObjectsByURL, err := service.loadCachedProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) - timings.Record("syfon_bucket_inventory", time.Since(start)) - timings.RecordMemory("syfon_bucket_inventory", "bucket_objects", len(bucketObjects), "bucket_lookup", len(bucketObjectsByURL)) - logStorageChainInputResult("syfon_bucket_items", len(bucketObjects), err) - bucketCh <- bucketResult{bucketObjects: bucketObjects, bucketObjectsByURL: bucketObjectsByURL, err: err} - }() - } + go func() { + start := time.Now() + timings.StageStart("syfon_bucket_inventory") + var bucketObjects []gintegrationsyfon.ProjectBucketObject + var bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + var err error + if bucketMode == StorageChainBucketModeValidate { + if forceRefresh { + bucketObjects, bucketObjectsByURL, err = service.loadProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + } else { + bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + } + } else { + bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) + } + timings.Record("syfon_bucket_inventory", time.Since(start)) + timings.RecordMemory("syfon_bucket_inventory", "bucket_objects", len(bucketObjects), "bucket_lookup", len(bucketObjectsByURL)) + logStorageChainInputResult("syfon_bucket_items", len(bucketObjects), err) + bucketCh <- bucketResult{bucketObjects: bucketObjects, bucketObjectsByURL: bucketObjectsByURL, err: err} + }() inventory := <-inventoryCh recordSet := <-recordCh @@ -188,8 +200,9 @@ const ( ) type chainAuditAccumulator struct { - findings []GitStorageChainFinding - summary GitStorageChainAuditSummary + findings []GitStorageChainFinding + summary GitStorageChainAuditSummary + missingBucketDebugLogCount int } func newChainSummary(bucketObjectCount, syfonRecordCount, gitTrackedFileCount int) GitStorageChainAuditSummary { @@ -892,7 +905,7 @@ func inventoryPresentProbe(record projectRecordState, objectURL string, expected } return gintegrationsyfon.BulkStorageProbeResult{ ID: storageListValidationRequestKey(objectURL, record.Size, expectedName), - Operation: StorageChainValidationModeList, + Operation: StorageChainValidationModeInventory, ObjectURL: objectURL, Provider: strings.TrimSpace(item.Provider), Bucket: strings.TrimSpace(item.Bucket), @@ -917,16 +930,16 @@ func inventoryMissingProbe(record projectRecordState, objectURL string, expected nameMatch := false return gintegrationsyfon.BulkStorageProbeResult{ ID: storageListValidationRequestKey(objectURL, record.Size, expectedName), - Operation: StorageChainValidationModeList, + Operation: StorageChainValidationModeInventory, ObjectURL: objectURL, Provider: "s3", Bucket: bucket, Key: key, Path: path.Base(key), Exists: false, - Status: "not_found", - Error: fmt.Sprintf("object %q was not found in project bucket inventory", objectURL), - ErrorKind: "object_not_found", + Status: "unknown", + Error: fmt.Sprintf("object %q was absent from project bucket inventory and requires exact verification", objectURL), + ErrorKind: "inventory_miss", ValidationStatus: "unverifiable", SizeMatch: &sizeMatch, NameMatch: &nameMatch, @@ -980,7 +993,7 @@ func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context return view, nil } -func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, inventory []RepoInventoryFile, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, forceRefresh bool, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { +func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, inventory []RepoInventoryFile, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) view := &storageAuditStorageView{ scopes: append([]domain.StorageBucketScope(nil), scopes...), @@ -999,28 +1012,44 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex return view, nil } if validationMode == StorageChainValidationModeList { - inventoryStart := time.Now() - var validationObjects []gintegrationsyfon.ProjectBucketObject - var validationObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject - var err error - if forceRefresh { - validationObjects, validationObjectsByURL, err = service.loadProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") - } else { - validationObjects, validationObjectsByURL, err = service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + if bucketInventoryErr != nil { + return nil, bucketInventoryErr } - timings.Record("syfon_bucket_validation_inventory", time.Since(inventoryStart)) - timings.RecordMemory("syfon_bucket_validation_inventory", "bucket_objects", len(validationObjects), "bucket_lookup", len(validationObjectsByURL)) - if err != nil { - return nil, err - } - view.bucketObjects, view.bucketObjectsByURL = cloneBucketInventory(validationObjects, validationObjectsByURL) + view.bucketObjects = bucketObjects + view.bucketObjectsByURL = bucketObjectsByURL validateStart := time.Now() probedRecordSet, err := attachProjectStorageInventoryValidations(recordSet, inventory, view.bucketObjectsByURL, scopes, organization, project) if err != nil { return nil, err } timings.Record("inventory_list_validation", time.Since(validateStart)) - timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL)) + + candidateStart := time.Now() + candidates := selectInventoryMissRecordSet(probedRecordSet) + timings.Record("exact_list_candidate_selection", time.Since(candidateStart)) + if candidates != nil { + probeStart := time.Now() + exactRecordSet, probeErr := service.attachProjectStorageListValidations(ctx, authorizationHeader, candidates) + timings.Record("targeted_exact_list_validation", time.Since(probeStart)) + if probeErr != nil { + return nil, probeErr + } + probedRecordSet = mergeRecordSetProbes(probedRecordSet, exactRecordSet) + view.bucketObjects, view.bucketObjectsByURL = mergeBucketInventoryWithPresentProbes(view.bucketObjects, view.bucketObjectsByURL, exactRecordSet) + + confirmationCandidates := selectInventoryMissRecordSet(exactRecordSet) + if confirmationCandidates != nil { + confirmationStart := time.Now() + confirmedRecordSet, confirmationErr := service.attachProjectStorageProbes(ctx, authorizationHeader, confirmationCandidates) + timings.Record("targeted_metadata_confirmation", time.Since(confirmationStart)) + if confirmationErr != nil { + return nil, confirmationErr + } + probedRecordSet = mergeRecordSetProbes(probedRecordSet, confirmedRecordSet) + view.bucketObjects, view.bucketObjectsByURL = mergeBucketInventoryWithPresentProbes(view.bucketObjects, view.bucketObjectsByURL, confirmedRecordSet) + } + } + timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL), "exact_probe_records", countRecordSet(candidates)) view.recordsByChecksum = probedRecordSet.recordsByChecksum view.allProjectRecords = probedRecordSet.allProjectRecords return view, nil @@ -1153,6 +1182,79 @@ func selectTargetedProbeRecordSet(recordSet *storageAuditRecordSet, bucketObject } } +func selectInventoryMissRecordSet(recordSet *storageAuditRecordSet) *storageAuditRecordSet { + if recordSet == nil { + return nil + } + selected := make(map[string][]projectRecordState) + for checksum, group := range recordSet.allProjectRecords { + for _, record := range group { + assessment := assessStorageRecordEvidence(record, false) + if assessment.Present || !assessment.HasInventoryMiss { + continue + } + selected[checksum] = append(selected[checksum], record) + } + } + if len(selected) == 0 { + return nil + } + return &storageAuditRecordSet{ + recordsByChecksum: cloneRecordStateMap(selected), + allProjectRecords: cloneRecordStateMap(selected), + } +} + +func countRecordSet(recordSet *storageAuditRecordSet) int { + if recordSet == nil { + return 0 + } + return countRecordStates(recordSet.allProjectRecords) +} + +func mergeBucketInventoryWithPresentProbes(bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, recordSet *storageAuditRecordSet) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { + if recordSet == nil { + return bucketObjects, bucketObjectsByURL + } + if bucketObjectsByURL == nil { + bucketObjectsByURL = make(map[string]gintegrationsyfon.ProjectBucketObject) + } + for _, group := range recordSet.allProjectRecords { + for _, record := range group { + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Status) != "present" { + continue + } + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL) + if objectURL == "" { + continue + } + if _, ok := bucketObjectsByURL[objectURL]; ok { + continue + } + item := bucketObjectFromPresentProbe(objectURL, probe) + bucketObjectsByURL[objectURL] = item + bucketObjects = append(bucketObjects, item) + } + } + } + return bucketObjects, bucketObjectsByURL +} + +func bucketObjectFromPresentProbe(objectURL string, probe gintegrationsyfon.BulkStorageProbeResult) gintegrationsyfon.ProjectBucketObject { + return gintegrationsyfon.ProjectBucketObject{ + ObjectURL: objectURL, + Provider: strings.TrimSpace(probe.Provider), + Bucket: strings.TrimSpace(probe.Bucket), + Key: strings.Trim(strings.TrimSpace(probe.Key), "/"), + Path: strings.TrimSpace(probe.Path), + SizeBytes: derefInt64(probe.SizeBytes), + MetaSHA256: strings.TrimSpace(probe.MetaSHA256), + ETag: strings.TrimSpace(probe.ETag), + LastModified: strings.TrimSpace(probe.LastModified), + } +} + func recordNeedsTargetedProbe(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { resolution := resolveRecordStorage(record, bucketObjectsByURL) if len(resolution.matchedBucketObjectURLs) == 0 { @@ -1236,14 +1338,24 @@ func derefInt64(value *int64) int64 { return *value } -func buildStorageChainIndex(inventory []RepoInventoryFile, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) storageChainIndex { +func buildStorageChainIndex(inventory []RepoInventoryFile, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, scopes []domain.StorageBucketScope, organization string, project string) storageChainIndex { repoPathsByChecksum := make(map[string][]string, len(inventory)) + repoPathsByBucketURL := make(map[string][]string, len(inventory)) + repoChecksumsByPath := make(map[string]string, len(inventory)) for _, item := range inventory { checksum := normalizeAnalyticsChecksum(item.Checksum) if checksum == "" { continue } - repoPathsByChecksum[checksum] = append(repoPathsByChecksum[checksum], item.RepoPath) + repoPath := normalizeRepoSubpath(item.RepoPath) + if repoPath == "" { + continue + } + repoPathsByChecksum[checksum] = append(repoPathsByChecksum[checksum], repoPath) + repoChecksumsByPath[repoPath] = checksum + for _, bucketURL := range projectScopeRepoPathObjectURLs([]string{repoPath}, scopes, organization, project) { + repoPathsByBucketURL[bucketURL] = append(repoPathsByBucketURL[bucketURL], repoPath) + } } allRecords := flattenRecordStates(allProjectRecords) recordsByBucketURL := make(map[string][]projectRecordState) @@ -1257,13 +1369,15 @@ func buildStorageChainIndex(inventory []RepoInventoryFile, allProjectRecords map allRecords: allRecords, bucketObjectsByURL: bucketObjectsByURL, repoPathsByChecksum: repoPathsByChecksum, + repoPathsByBucketURL: repoPathsByBucketURL, + repoChecksumsByPath: repoChecksumsByPath, recordsByBucketURL: recordsByBucketURL, equivalentRecordKeys: buildEquivalentRecordKeyIndex(allRecords), } } -func buildStorageChainAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, includeBucketOrigin bool) *chainAuditModel { - index := buildStorageChainIndex(inventory, allProjectRecords, bucketObjectsByURL) +func buildStorageChainAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, scopes []domain.StorageBucketScope, organization string, project string, includeBucketOrigin bool) *chainAuditModel { + index := buildStorageChainIndex(inventory, allProjectRecords, bucketObjectsByURL, scopes, organization, project) acc := chainAuditAccumulator{ findings: make([]GitStorageChainFinding, 0), summary: newChainSummary(len(bucketObjectsByURL), len(index.allRecords), len(inventory)), @@ -1272,7 +1386,7 @@ func buildStorageChainAuditModel(gitSubpath string, inventory []RepoInventoryFil buildBucketOriginChainFindings(index, &acc) } buildSyfonOriginChainFindings(index, &acc, !includeBucketOrigin) - buildGitOriginChainFindings(index, recordsByChecksum, allProjectRecords, &acc) + buildGitOriginChainFindings(index, recordsByChecksum, allProjectRecords, scopes, organization, project, &acc) return finalizeChainFindings(gitSubpath, acc) } @@ -1288,6 +1402,12 @@ func buildBucketOriginChainFindings(index storageChainIndex, acc *chainAuditAccu acc.addCount("bucket_syfon_git_complete", 1) continue } + if len(index.recordsByBucketURL[bucketURL]) > 0 { + // An exact Syfon-to-bucket URL match is not a bucket orphan, even + // when the Git and Syfon checksums disagree. The Syfon-origin pass + // reports that conflict without offering the bucket for deletion. + continue + } if bucketObjectHasEquivalentSyfonRecord(item, index.equivalentRecordKeys) { continue } @@ -1353,11 +1473,74 @@ func recordHasResolvedStorage(record projectRecordState, bucketMatches []string, return resolveRecordStorage(record, bucketObjectsByURL).hasPresentProbe } +func logMissingBucketCandidateDebug(record projectRecordState, gitPaths []string, bucketMatches []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, acc *chainAuditAccumulator) { + if acc == nil || acc.missingBucketDebugLogCount >= storageChainValidationDebugSampleLimit { + return + } + resolution := resolveRecordStorage(record, bucketObjectsByURL) + log.Printf( + "INFO: storage_chain_missing_bucket_candidates object_id=%s checksum=%s name=%q git_paths=%q raw_access_urls=%q canonical_access_urls=%q candidate_status=%q bucket_matches=%q access_probe_status=%q", + strings.TrimSpace(record.ObjectID), + strings.TrimSpace(record.Checksum), + strings.TrimSpace(record.Name), + strings.Join(uniqueStrings(gitPaths), ","), + strings.Join(rawAccessURLsForRecord(record), ","), + strings.Join(uniqueStrings(record.CanonicalAccessURLs), ","), + strings.Join(storageCandidateStatuses(resolution.candidateURLs, bucketObjectsByURL), ","), + strings.Join(uniqueStrings(bucketMatches), ","), + strings.Join(storageProbeDebugStatuses(record.AccessProbes), ","), + ) + acc.missingBucketDebugLogCount++ +} + +func storageCandidateStatuses(candidateURLs []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []string { + out := make([]string, 0, len(candidateURLs)) + for _, candidate := range uniqueStrings(candidateURLs) { + status := "missing" + if _, ok := bucketObjectsByURL[candidate]; ok { + status = "found" + } + out = append(out, candidate+"="+status) + } + return out +} + +func storageProbeDebugStatuses(probes []gintegrationsyfon.BulkStorageProbeResult) []string { + out := make([]string, 0, len(probes)) + for _, probe := range probes { + objectURL := syfonProbeObjectURL(probe) + if objectURL == "" { + objectURL = strings.TrimSpace(probe.ObjectURL) + } + status := strings.TrimSpace(probe.Status) + if status == "" { + status = "unknown" + } + validation := strings.TrimSpace(probe.ValidationStatus) + if validation != "" { + status += "/" + validation + } + if errKind := strings.TrimSpace(probe.ErrorKind); errKind != "" { + status += "/" + errKind + } + out = append(out, objectURL+"="+status) + } + return out +} + func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccumulator, countCompleteFromSyfon bool) { for _, record := range index.allRecords { gitPaths := uniqueStrings(index.repoPathsByChecksum[normalizeAnalyticsChecksum(record.Checksum)]) bucketMatches := matchedBucketObjectURLs(record, index.bucketObjectsByURL) - switch classifyStorageFinding(record, index.bucketObjectsByURL) { + classification := classifyStorageFinding(record, index.bucketObjectsByURL) + if len(gitPaths) == 0 && len(bucketMatches) > 0 && (classification == storageFindingNone || classification == storageFindingValidationMismatch) { + if samePathGitPaths := repoPathsForBucketURLs(index.repoPathsByBucketURL, bucketMatches); len(samePathGitPaths) > 0 { + findings := buildSamePathChecksumConflictFindings(index, record, samePathGitPaths, bucketMatches) + acc.add("git_syfon_metadata_mismatch", findings...) + continue + } + } + switch classification { case storageFindingBrokenBucketMap: findingRecord := repairableBrokenAccessRecord(record) if len(findingRecord.AccessProbes) == 0 { @@ -1368,6 +1551,7 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum acc.addCount("syfon_broken_bucket_mapping", len(findings)) case storageFindingObjectMissing: if len(gitPaths) > 0 { + logMissingBucketCandidateDebug(record, gitPaths, bucketMatches, index.bucketObjectsByURL, acc) findings := buildChainRecordFindings("syfon_git_no_bucket", record, gitPaths, bucketMatches, "Git and Syfon matched, but the mapped bucket object does not exist.") acc.findings = append(acc.findings, findings...) acc.addCount("syfon_git_no_bucket", len(gitPaths)) @@ -1398,22 +1582,89 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum } } -func buildGitOriginChainFindings(index storageChainIndex, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, acc *chainAuditAccumulator) { +func repoPathsForBucketURLs(repoPathsByBucketURL map[string][]string, bucketURLs []string) []string { + paths := make([]string, 0, len(bucketURLs)) + for _, bucketURL := range uniqueStrings(bucketURLs) { + paths = append(paths, repoPathsByBucketURL[bucketURL]...) + } + return uniqueStrings(paths) +} + +func hasSyfonRecordForBucketURLs(recordsByBucketURL map[string][]projectRecordState, bucketURLs []string) bool { + for _, bucketURL := range uniqueStrings(bucketURLs) { + if len(recordsByBucketURL[bucketURL]) > 0 { + return true + } + } + return false +} + +func buildSamePathChecksumConflictFindings(index storageChainIndex, record projectRecordState, gitPaths []string, bucketMatches []string) []GitStorageChainFinding { + findings := buildChainRecordFindings( + "git_syfon_metadata_mismatch", + record, + gitPaths, + bucketMatches, + "Git and Syfon map to the same bucket object path, but their checksums disagree. Do not delete this object automatically.", + ) + for findingIndex := range findings { + gitChecksum := index.repoChecksumsByPath[normalizeRepoSubpath(findings[findingIndex].NormalizedPath)] + findings[findingIndex].Error = "Git LFS and Syfon map to the same bucket object path but have different SHA-256 values." + findings[findingIndex].SuggestedFix = fmt.Sprintf( + "Git LFS pointer SHA-256 is %s; Syfon record SHA-256 is %s. Recompute the bucket object's SHA-256 and reconcile the stale side manually before changing Git, Syfon, or the bucket object.", + gitChecksum, + strings.TrimSpace(record.Checksum), + ) + } + return findings +} + +func buildGitOriginChainFindings(index storageChainIndex, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, scopes []domain.StorageBucketScope, organization string, project string, acc *chainAuditAccumulator) { for _, item := range index.inventory { checksum := normalizeAnalyticsChecksum(item.Checksum) if len(allProjectRecords[checksum]) > 0 || len(recordsByChecksum[checksum]) > 0 { continue } - actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupport("git_only_no_syfon") + bucketURLs := projectScopeRepoPathObjectURLs([]string{item.RepoPath}, scopes, organization, project) + if hasSyfonRecordForBucketURLs(index.recordsByBucketURL, bucketURLs) { + // The Syfon-origin pass reports an exact-path checksum conflict as a + // metadata mismatch. Do not also offer this Git pointer for Syfon + // creation just because its checksum differs. + continue + } + bucketObjectURL := "" + bucketSizeBytes := int64(0) + bucketEvaluation := "not_evaluated" + if len(bucketURLs) == 1 { + bucketObjectURL = bucketURLs[0] + if bucketObject, ok := index.bucketObjectsByURL[bucketObjectURL]; ok { + bucketSizeBytes = bucketObject.SizeBytes + bucketEvaluation = "present" + } else { + bucketEvaluation = "not_found" + } + } else if len(bucketURLs) > 1 { + bucketEvaluation = "ambiguous_mapping" + } + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence("git_only_no_syfon", "verified") + recommendedAction := "Git checksum has no matching Syfon record. Bucket presence is not claimed by this finding." + if bucketEvaluation == "present" && bucketSizeBytes == item.Size { + recommendedAction = "Git checksum has no matching Syfon record. The mapped bucket object exists and its byte size matches the Git LFS pointer; create a Syfon record after reviewing that RGW did not provide a SHA-256 for verification." + } else if bucketEvaluation == "present" { + recommendedAction = "Git checksum has no matching Syfon record. The mapped bucket object exists but its byte size differs from the Git LFS pointer, so it is not safe to create a Syfon record." + } acc.add("git_only_no_syfon", GitStorageChainFinding{ Kind: "git_only_no_syfon", + EvidenceStatus: "verified", NormalizedPath: item.RepoPath, Checksum: checksum, SourcePaths: []string{item.RepoPath}, ObjectIDs: []string{}, RecordCount: 0, SizeBytes: item.Size, - RecommendedAction: "Git checksum has no matching Syfon record. Bucket presence is not claimed by this finding.", + BucketObjectURL: bucketObjectURL, + BucketSizeBytes: bucketSizeBytes, + RecommendedAction: recommendedAction, Actionability: actionability, AvailableActions: availableActions, DefaultAction: defaultAction, @@ -1422,7 +1673,8 @@ func buildGitOriginChainFindings(index storageChainIndex, recordsByChecksum map[ Checksum: checksum, SourcePaths: []string{item.RepoPath}, ObjectIDs: []string{}, - BucketEvaluation: "not_evaluated", + BucketObjectURLs: bucketURLs, + BucketEvaluation: bucketEvaluation, }, }) } diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 4e96a89..c432908 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -47,6 +47,7 @@ type fakeStorageAnalyticsBackend struct { listProjectBucketObjectsPathPrefix string listProjectBucketInventoryCalls int listProjectBucketInventoryPathPrefix string + listProjectBucketInventoryDelay time.Duration listProjectBucketSummaryCalls int listProjectBucketSummaryMode string bulkGetProjectRecordsCalls int @@ -59,6 +60,7 @@ type fakeStorageAnalyticsBackend struct { deletedStorageIDs []string updatedAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod deletedBucketObjects []string + registeredObjects []gintegrationsyfon.ProjectObjectRegistration } func (fake *fakeStorageAnalyticsBackend) ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) { @@ -188,12 +190,61 @@ func (fake *fakeStorageAnalyticsBackend) ListProjectBucketObjects(ctx context.Co func (fake *fakeStorageAnalyticsBackend) ListProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) { fake.listProjectBucketInventoryCalls++ fake.listProjectBucketInventoryPathPrefix = pathPrefix + if fake.listProjectBucketInventoryDelay > 0 { + time.Sleep(fake.listProjectBucketInventoryDelay) + } if fake.listProjectBucketObjectsErr != nil { return nil, fake.listProjectBucketObjectsErr } return append([]gintegrationsyfon.ProjectBucketObject(nil), fake.bucketObjects...), nil } +func TestBuildStorageChainAuditCoalescesConcurrentForcedRefreshes(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + listProjectBucketInventoryDelay: 100 * time.Millisecond, + projectMetricsSummary: &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 1, + RecordLatestUpdatedTime: "2026-07-01T00:00:00Z", + }, + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/root/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket/root"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/root/data/a.txt", Bucket: "bucket", Key: "root/data/a.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + service.chainAuditResponseCache = newMemoryStorageChainAuditResponseCache() + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate, ForceAuditRefresh: true} + + var wg sync.WaitGroup + errs := make(chan error, 2) + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options) + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("build concurrent forced audit: %v", err) + } + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected concurrent forced refreshes to share one bucket inventory, got %d", backend.listProjectBucketInventoryCalls) + } +} + func (fake *fakeStorageAnalyticsBackend) ListProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) { fake.listProjectBucketSummaryCalls++ fake.listProjectBucketSummaryMode = mode @@ -290,6 +341,24 @@ func (fake *fakeStorageAnalyticsBackend) BulkUpdateAccessMethods(ctx context.Con return nil } +func (fake *fakeStorageAnalyticsBackend) RegisterProjectObjects(ctx context.Context, authorizationHeader string, candidates []gintegrationsyfon.ProjectObjectRegistration) ([]gintegrationsyfon.ProjectObjectRegistrationResult, error) { + fake.registeredObjects = append(fake.registeredObjects, candidates...) + results := make([]gintegrationsyfon.ProjectObjectRegistrationResult, len(candidates)) + for index, candidate := range candidates { + results[index] = gintegrationsyfon.ProjectObjectRegistrationResult{ObjectID: "created-" + candidate.Checksum[:8]} + fake.projectRecords = append(fake.projectRecords, gintegrationsyfon.ProjectRecord{ + ObjectID: results[index].ObjectID, + Name: candidate.Name, + Checksum: candidate.Checksum, + Size: candidate.Size, + Organization: "HTAN_INT", + Project: "BForePC", + AccessURLs: candidate.AccessURLs, + }) + } + return results, nil +} + func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -1146,7 +1215,7 @@ func TestStorageRepairPolicyCoversFindingKinds(t *testing.T) { actionability string }{ {"bucket_only_object", storageActionDeleteBucketObject, []string{storageActionDeleteBucketObject}, storageActionabilityAutoRepair}, - {"bucket_syfon_no_git", storageActionDeleteBoth, []string{storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject}, storageActionabilityAutoRepair}, + {"bucket_syfon_no_git", storageActionDeleteBoth, []string{storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject}, storageActionabilityManualChoice}, {"repo_orphan_live_object", storageActionDeleteBoth, []string{storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject}, storageActionabilityAutoRepair}, {"repo_orphan_stale_record", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, {"stale_duplicate_record", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, @@ -1155,7 +1224,7 @@ func TestStorageRepairPolicyCoversFindingKinds(t *testing.T) { {"broken_bucket_mapping", storageActionRemoveBrokenAccessURLs, []string{storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, {"syfon_broken_bucket_mapping", storageActionRemoveBrokenAccessURLs, []string{storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, {"storage_validation_mismatch", "", []string{storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionDeleteBoth}, storageActionabilityManualChoice}, - {"git_syfon_metadata_mismatch", "", []string{storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionDeleteBoth}, storageActionabilityManualChoice}, + {"git_syfon_metadata_mismatch", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, {"storage_object_missing", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, {"syfon_git_no_bucket", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, {"syfon_missing_bucket_object", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, @@ -1225,10 +1294,10 @@ func TestApplyStorageCleanupRepairMatrix(t *testing.T) { {name: "repo orphan live", finding: applyFinding("repo_orphan_live_object", "s3://bucket/live", []string{"obj-live"}, nil), wantDeletedIDs: []string{"obj-live"}, wantStorageIDs: []string{"obj-live"}}, {name: "repo orphan stale", finding: applyFinding("repo_orphan_stale_record", "s3://bucket/stale", []string{"obj-stale"}, nil), wantDeletedIDs: []string{"obj-stale"}}, {name: "stale duplicate", finding: applyFinding("stale_duplicate_record", "data/a.txt", []string{"obj-old"}, nil), wantDeletedIDs: []string{"obj-old"}}, - {name: "storage missing", finding: applyFinding("storage_object_missing", "data/missing.txt", []string{"obj-missing"}, nil), wantDeletedIDs: []string{"obj-missing"}}, - {name: "syfon git no bucket", finding: applyFinding("syfon_git_no_bucket", "data/missing.txt", []string{"obj-missing"}, nil), wantDeletedIDs: []string{"obj-missing"}}, - {name: "syfon missing bucket object", finding: applyFinding("syfon_missing_bucket_object", "s3://bucket/missing", []string{"obj-missing"}, nil), wantDeletedIDs: []string{"obj-missing"}}, - {name: "git syfon mismatch delete both", finding: applyFinding("git_syfon_metadata_mismatch", "data/mismatch.txt", []string{"obj-mm"}, []string{"s3://bucket/mismatch"}), actions: []GitStorageCleanupApplyAction{{Kind: "git_syfon_metadata_mismatch", NormalizedPath: "data/mismatch.txt", Action: storageActionDeleteBoth}}, wantDeletedIDs: []string{"obj-mm"}, wantBucketURLs: []string{"s3://bucket/mismatch"}}, + {name: "storage missing", finding: applyFinding("storage_object_missing", "data/missing.txt", []string{"obj-missing"}, []string{"s3://bucket/missing"}), wantDeletedIDs: []string{"obj-missing"}}, + {name: "syfon git no bucket", finding: applyFinding("syfon_git_no_bucket", "data/missing.txt", []string{"obj-missing"}, []string{"s3://bucket/missing"}), wantDeletedIDs: []string{"obj-missing"}}, + {name: "syfon missing bucket object", finding: applyFinding("syfon_missing_bucket_object", "s3://bucket/missing", []string{"obj-missing"}, []string{"s3://bucket/missing"}), wantDeletedIDs: []string{"obj-missing"}}, + {name: "git syfon mismatch requires manual review", finding: applyFinding("git_syfon_metadata_mismatch", "data/mismatch.txt", []string{"obj-mm"}, []string{"s3://bucket/mismatch"}), wantManualPath: "data/mismatch.txt"}, {name: "storage mismatch delete both", finding: applyFinding("storage_validation_mismatch", "data/mismatch.txt", []string{"obj-mm"}, []string{"s3://bucket/mismatch"}), actions: []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/mismatch.txt", Action: storageActionDeleteBoth}}, wantDeletedIDs: []string{"obj-mm"}, wantBucketURLs: []string{"s3://bucket/mismatch"}}, {name: "live duplicate conflict", finding: applyFinding("live_duplicate_conflict", "data/dup.txt", []string{"obj-a", "obj-b"}, nil), wantManualPath: "data/dup.txt"}, {name: "git only no syfon", finding: applyFinding("git_only_no_syfon", "data/git-only.txt", nil, nil), wantManualPath: "data/git-only.txt"}, @@ -1236,7 +1305,18 @@ func TestApplyStorageCleanupRepairMatrix(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - backend := &fakeStorageAnalyticsBackend{} + backend := &fakeStorageAnalyticsBackend{probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{}} + action := test.finding.DefaultAction + if len(test.actions) > 0 { + action = test.actions[0].Action + } + if cleanupVerificationExpectation(test.finding.Kind, action) == cleanupExpectationMissing { + backend.probeResults["cleanup-revalidate-0"] = gintegrationsyfon.BulkStorageProbeResult{ + ID: "cleanup-revalidate-0", + Status: "not_found", + ErrorKind: "object_not_found", + } + } service := NewStorageAnalyticsService(backend) result, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", nil, test.actions, []GitStorageCleanupApplyFinding{test.finding}, false, false, false, false, false) if err != nil { @@ -1351,13 +1431,9 @@ func TestApplyStorageCleanupValidation(t *testing.T) { }{ {name: "unknown kind", finding: applyFinding("unknown_kind", "data/a.txt", []string{"obj-a"}, nil), wantErr: "unsupported cleanup finding kind"}, {name: "unsupported action", finding: applyFinding("bucket_only_object", "s3://bucket/a", nil, []string{"s3://bucket/a"}), actions: []GitStorageCleanupApplyAction{{Kind: "bucket_only_object", NormalizedPath: "s3://bucket/a", Action: storageActionDeleteSyfonRecord}}, wantErr: "not supported"}, - {name: "action not advertised", finding: func() GitStorageCleanupApplyFinding { - finding := applyFinding("bucket_only_object", "s3://bucket/a", nil, []string{"s3://bucket/a"}) - finding.AvailableActions = []string{storageActionInspectEvidence} - return finding - }(), wantErr: "not advertised"}, {name: "missing object ids", finding: applyFinding("repo_orphan_stale_record", "s3://bucket/a", nil, nil), wantErr: "missing object_ids"}, - {name: "missing bucket urls", finding: applyFinding("bucket_only_object", "data/a.txt", nil, nil), wantErr: "missing bucket object URLs"}, + {name: "metadata mismatch disallows deletion", finding: applyFinding("git_syfon_metadata_mismatch", "data/a.txt", []string{"obj-a"}, nil), actions: []GitStorageCleanupApplyAction{{Kind: "git_syfon_metadata_mismatch", NormalizedPath: "data/a.txt", Action: storageActionDeleteBucketObject}}, wantErr: "not supported"}, + {name: "missing revalidation urls", finding: applyFinding("syfon_git_no_bucket", "data/a.txt", []string{"obj-a"}, nil), wantErr: "required for exact revalidation"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -1369,7 +1445,40 @@ func TestApplyStorageCleanupValidation(t *testing.T) { } } -func TestBuildStorageCleanupAuditFlagsRecordWhenAnyAccessProbeIsDead(t *testing.T) { +func TestApplyStorageCleanupUsesServerPolicyWhenAuditActionsAreStale(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{}} + service := NewStorageAnalyticsService(backend) + finding := applyFinding("bucket_syfon_no_git", "s3://bucket/no-git", []string{"obj-no-git"}, []string{"s3://bucket/no-git"}) + // An audit may have been rendered before Gecko advertised this manual + // repair. The server policy, rather than this stale client snapshot, must + // decide whether the requested action is allowed. + finding.AvailableActions = []string{storageActionInspectEvidence} + response, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{finding.NormalizedPath}, + []GitStorageCleanupApplyAction{{ + Kind: finding.Kind, + NormalizedPath: finding.NormalizedPath, + Action: storageActionDeleteBoth, + }}, + []GitStorageCleanupApplyFinding{finding}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply stale bucket + Syfon audit finding: %v", err) + } + assertStringSet(t, "deleted IDs", response.DeletedRecordIDs, []string{"obj-no-git"}) + assertStringSet(t, "deleted bucket URLs", response.DeletedBucketObjectURLs, []string{"s3://bucket/no-git"}) +} + +func TestBuildStorageCleanupAuditAcceptsRecordWhenAnyAccessProbeIsLive(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), }) @@ -1412,9 +1521,10 @@ func TestBuildStorageCleanupAuditFlagsRecordWhenAnyAccessProbeIsDead(t *testing. if err != nil { t.Fatalf("build cleanup audit: %v", err) } - finding := assertHasCleanupFinding(t, cleanup.Findings, "storage_object_missing", "data/a.txt") - if finding.Evidence == nil || !contains(finding.Evidence.AccessURLs, "s3://bucket/legacy-a") { - t.Fatalf("expected dead raw access URL evidence, got %+v", finding) + for _, finding := range cleanup.Findings { + if finding.NormalizedPath == "data/a.txt" && finding.Kind == "storage_object_missing" { + t.Fatalf("a live canonical locator must suppress a missing-record finding: %+v", finding) + } } } @@ -1750,8 +1860,11 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { if backend.probeCalls != 0 { t.Fatalf("expected validate mode to skip HEAD probes, got %d calls", backend.probeCalls) } - if backend.listProbeCalls != 0 || len(backend.listProbeItems) != 0 { - t.Fatalf("expected project inventory validation to skip bulk LIST calls, got calls=%d items=%+v", backend.listProbeCalls, backend.listProbeItems) + if backend.listProbeCalls != 1 || len(backend.listProbeItems) != 1 { + t.Fatalf("expected one exact LIST for the inventory-missing candidate, got calls=%d items=%+v", backend.listProbeCalls, backend.listProbeItems) + } + if backend.listProbeItems[0].ObjectURL != "s3://bucket/syfon-only.txt" { + t.Fatalf("expected only the inventory-missing locator to be verified, got %+v", backend.listProbeItems) } if chain.Summary.BucketPathExists != nil || chain.Summary.BucketSummaryMode != "" { t.Fatalf("expected validate mode not to claim bucket prefix summary, got %+v", chain.Summary) @@ -1759,21 +1872,25 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { if chain.Summary.ValidationMode != StorageChainValidationModeList { t.Fatalf("expected LIST validation mode, got %+v", chain.Summary) } - if chain.Summary.BucketObjectCount != 2 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { - t.Fatalf("expected validate mode to count project inventory without bucket-only findings, got summary %+v", chain.Summary) + if chain.Summary.BucketObjectCount != 3 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { + t.Fatalf("expected validate mode to merge the exact-present object without bucket-only findings, got summary %+v", chain.Summary) } if chain.Summary.CountsByKind["bucket_syfon_git_complete"] != 2 { t.Fatalf("expected validate mode to count clean Syfon/Git/bucket join, got summary %+v", chain.Summary) } - if chain.Summary.CountsByKind["syfon_missing_bucket_object"] != 1 { - t.Fatalf("expected validate mode to report missing Syfon bucket object, got summary %+v", chain.Summary) + if chain.Summary.CountsByKind["syfon_missing_bucket_object"] != 0 || chain.Summary.CountsByKind["bucket_syfon_no_git"] != 1 { + t.Fatalf("expected exact LIST presence to suppress the inventory false positive, got summary %+v", chain.Summary) } if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { t.Fatalf("expected basename mismatch not to be a broken bucket mapping, got summary %+v", chain.Summary) } assertNoChainFinding(t, chain.Findings, "bucket_only_object") assertNoChainFinding(t, chain.Findings, "syfon_broken_bucket_mapping") - assertHasChainFinding(t, chain.Findings, "syfon_missing_bucket_object", "s3://bucket/syfon-only.txt") + assertNoChainFinding(t, chain.Findings, "syfon_missing_bucket_object") + finding := assertHasChainFinding(t, chain.Findings, "bucket_syfon_no_git", "s3://bucket/syfon-only.txt") + if finding.EvidenceStatus != "verified" || finding.Actionability != storageActionabilityInspectOnly || finding.DefaultAction != storageActionInspectEvidence { + t.Fatalf("expected verified but non-destructive Git-absence finding, got %+v", finding) + } } func TestBuildStorageChainAuditMetadataModeUsesMetadataValidation(t *testing.T) { @@ -1895,7 +2012,8 @@ func TestBuildStorageChainAuditSuggestsFixForLargeSizeDrift(t *testing.T) { t.Fatalf("build chain audit: %v", err) } finding := assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/a.txt") - if !strings.Contains(finding.SuggestedFix, "Syfon/Git size is 44 B, bucket inventory reports 40 B") { + if !strings.Contains(finding.SuggestedFix, "Syfon record size is 44 B, bucket inventory reports 40 B") || + !strings.Contains(finding.SuggestedFix, "bucket object cannot have the current Git/Syfon SHA-256") { t.Fatalf("expected size-drift suggested fix, got %+v", finding) } if finding.SuggestedAction != "" { @@ -2082,6 +2200,19 @@ func TestBuildStorageChainAuditProjectsGitSubpathFromRootResponseCache(t *testin if backend.listProjectAuditRecordsCalls != 1 || backend.listProjectBucketInventoryCalls != 1 { t.Fatalf("expected root audit to load Syfon records and bucket inventory once, got records=%d bucket=%d", backend.listProjectAuditRecordsCalls, backend.listProjectBucketInventoryCalls) } + cache.mu.Lock() + for key, entry := range cache.entries { + entry.value.RefreshDurationMillis = 123 + cache.entries[key] = entry + } + cache.mu.Unlock() + cachedRoot, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options) + if err != nil { + t.Fatalf("build cached root chain audit: %v", err) + } + if !cachedRoot.Summary.AuditCacheHit || cachedRoot.Summary.AuditRefreshDurationMs != 123 { + t.Fatalf("expected cached root audit to include refresh duration, got %+v", cachedRoot.Summary) + } subpath, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options) if err != nil { t.Fatalf("build subpath chain audit: %v", err) @@ -2089,6 +2220,9 @@ func TestBuildStorageChainAuditProjectsGitSubpathFromRootResponseCache(t *testin if !subpath.Summary.AuditCacheHit || subpath.Summary.AuditCacheSource != "memory:root" { t.Fatalf("expected subpath audit to project from root response cache, got %+v", subpath.Summary) } + if subpath.Summary.AuditRefreshDurationMs != 123 { + t.Fatalf("expected subpath audit to include root refresh duration, got %+v", subpath.Summary) + } if subpath.PathPrefix != "data" || subpath.Summary.GitTrackedFileCount != 1 { t.Fatalf("expected data-scoped audit projection, got path=%q summary=%+v", subpath.PathPrefix, subpath.Summary) } @@ -2846,6 +2980,65 @@ func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) } } +func TestBuildStorageChainAuditClassifiesSameBucketPathChecksumConflictAsMetadataMismatch(t *testing.T) { + const repoPath = "JHU/ashley_kiemen/hematoxylin_eosin_stain/Level_2/HTA201_3/HTA201_3_ometif/HTA201_3_1_0022.offsets.json" + const gitChecksum = "99ec07eaecf97dc1cd7b7c4fb717c0661062eb57013d51b81780cbd3a302e323" + const syfonChecksum = "6b33300205aa4730ace8a04147c9223d547b1bf4f54001602d4ec147b7841a4f" + const size = int64(43) + const canonicalURL = "s3://bforepc/bforepc-prod/" + repoPath + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + repoPath: lfsPointer(gitChecksum, size), + }) + now := time.Date(2026, 7, 6, 18, 27, 23, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{{ + ObjectID: "93fd2da3-7b66-5232-bb77-002b78cb8ade", + Name: "HTA201_3_1_0022.offsets.json", + Checksum: syfonChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: size, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc-prod/" + repoPath}, + }}, + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: canonicalURL, + Bucket: "bforepc", + Key: "bforepc-prod/" + repoPath, + Path: repoPath, + SizeBytes: size, + }}, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) + mismatch := assertHasChainFinding(t, findings, "git_syfon_metadata_mismatch", repoPath) + if mismatch.Actionability != storageActionabilityInspectOnly || mismatch.DefaultAction != storageActionInspectEvidence || !contains(mismatch.AvailableActions, storageActionInspectEvidence) { + t.Fatalf("expected checksum conflict to be inspect-only, got %+v", mismatch) + } + if contains(mismatch.AvailableActions, storageActionDeleteBoth) || contains(mismatch.AvailableActions, storageActionDeleteBucketObject) { + t.Fatalf("checksum conflict must not advertise bucket deletion, got %+v", mismatch.AvailableActions) + } + if !strings.Contains(mismatch.SuggestedFix, gitChecksum) || !strings.Contains(mismatch.SuggestedFix, syfonChecksum) { + t.Fatalf("expected both conflicting checksums in the suggested fix, got %q", mismatch.SuggestedFix) + } + for _, kind := range []string{"bucket_only_object", "bucket_syfon_no_git", "git_only_no_syfon"} { + if got := chain.Summary.CountsByKind[kind]; got != 0 { + t.Fatalf("expected same-path checksum conflict to suppress %s, got summary %+v", kind, chain.Summary) + } + } +} + func TestBuildStorageChainAuditAcceptsMappedNameAndSourcePathInventoryObjects(t *testing.T) { sourceChecksum := "610cad76a473c4b0bf43f923b3bb907aac9ac2630d9c3f21aba1c531953c43e4" mappedChecksum := "b71c3fcf28a6d2cf6e432b21a1734786708517f4808c22f9114abb73a60f4cc7" @@ -3001,6 +3194,71 @@ func TestBuildStorageChainAuditUsesCanonicalInventoryCandidatesBeforeMissingBuck } } +func TestBuildStorageChainAuditMetadataConfirmationSuppressesBForePCListFalsePositive(t *testing.T) { + const repoPath = "OHSU/koei_chin/visium_hd/4-R2/outs/binned_outputs/square_002um/filtered_feature_bc_matrix/barcodes.tsv.gz" + const checksum = "a9809ffd53130272f775ad1bbea3166dbb4853bc6fa73bfa7a51535c4bc4f599" + const size = int64(46063992) + const canonicalURL = "s3://bforepc/bforepc-prod/" + repoPath + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + repoPath: lfsPointer(checksum, size), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{{ + ObjectID: "b5162717-bf49-528a-a2a8-47fbd3b0f090", + Name: "barcodes.tsv.gz", + Checksum: checksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: size, + AccessURLs: []string{"s3://bforepc-prod/" + repoPath}, + }}, + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + listProbeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageListValidationRequestKey(canonicalURL, size, "barcodes.tsv.gz"): { + ID: storageListValidationRequestKey(canonicalURL, size, "barcodes.tsv.gz"), + ObjectURL: canonicalURL, + Status: "not_found", + ErrorKind: "object_not_found", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "HTAN_INT", + "BForePC", + refName, + "", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build BForePC chain audit: %v", err) + } + if backend.listProbeCalls != 1 || len(backend.listProbeItems) != 1 || backend.listProbeItems[0].ObjectURL != canonicalURL { + t.Fatalf("expected one exact LIST for %q, got calls=%d items=%+v", canonicalURL, backend.listProbeCalls, backend.listProbeItems) + } + if backend.probeCalls != 1 || len(backend.probeItems) != 1 || backend.probeItems[0].ObjectURL != canonicalURL { + t.Fatalf("expected one metadata confirmation for the false LIST miss at %q, got calls=%d items=%+v", canonicalURL, backend.probeCalls, backend.probeItems) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 0 { + t.Fatalf("metadata presence must suppress the false LIST missing-bucket finding, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected the exact-present object to complete the chain, got summary %+v", chain.Summary) + } +} + func TestBuildStorageChainAuditCanonicalizesStaleRecordBucketUsingAuditProjectScopes(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/file.bin": lfsPointer("dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", 100), @@ -3263,7 +3521,7 @@ func TestCanonicalizeScopedStorageURLMatchesSyfonDownloadScopeResolution(t *test } } -func TestBuildStorageChainAuditFlagsExactPathMismatchWhenHashExistsElsewhereInBucket(t *testing.T) { +func TestBuildStorageChainAuditDoesNotInferMappingMismatchFromRelocatedBasename(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "CONFIG/cbds-BForePC.json": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 23739), }) @@ -3330,17 +3588,53 @@ func TestBuildStorageChainAuditFlagsExactPathMismatchWhenHashExistsElsewhereInBu if err != nil { t.Fatalf("build chain audit: %v", err) } - if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { - t.Fatalf("expected exact mapped-key miss to surface as broken bucket mapping, got %+v", chain.Summary) + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected relocated basename not to imply a broken bucket mapping, got %+v", chain.Summary) } if got := chain.Summary.CountsByKind["bucket_only_object"]; got != 0 { t.Fatalf("expected relocated hash not to be double-counted as bucket-only, got %+v", chain.Summary) } - if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 0 { - t.Fatalf("expected exact mapped-key miss to block clean-chain count, got %+v", chain.Summary) + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 1 { + t.Fatalf("expected exact mapped-key miss to remain a missing-object finding, got %+v", chain.Summary) } findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) - assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "CONFIG/cbds-BForePC.json") + assertHasChainFinding(t, findings, "syfon_git_no_bucket", "CONFIG/cbds-BForePC.json") +} + +func TestClassifyStorageFindingDoesNotTreatRepeatedBasenameAsBrokenMapping(t *testing.T) { + record := projectRecordState{ + ProjectRecord: gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Size: 100, + AccessURLs: []string{"s3://bucket-alias/path/features.tsv.gz"}, + }, + CanonicalAccessURLs: []string{"s3://bucket/root/path/features.tsv.gz"}, + AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ + { + ObjectURL: "s3://bucket/root/path/features.tsv.gz", + Operation: StorageChainValidationModeList, + Bucket: "bucket", + Key: "root/path/features.tsv.gz", + Status: "not_found", + Exists: false, + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }, + }, + } + bucketObjects := map[string]gintegrationsyfon.ProjectBucketObject{ + "s3://bucket/root/unrelated/features.tsv.gz": { + ObjectURL: "s3://bucket/root/unrelated/features.tsv.gz", + Bucket: "bucket", + Key: "root/unrelated/features.tsv.gz", + SizeBytes: 100, + }, + } + + if got := classifyStorageFinding(record, bucketObjects); got != storageFindingObjectMissing { + t.Fatalf("expected a repeated basename without a verified raw object to remain missing, got %s", got) + } } func TestClassifyStorageFindingSuppressesRawURLFailuresWhenScopedProbeMatches(t *testing.T) { @@ -3454,8 +3748,8 @@ func TestClassifyStorageFindingSurfacesRepairableBrokenAccessMethod(t *testing.T }, } - if got := classifyStorageFinding(record, nil); got != storageFindingBrokenBucketMap { - t.Fatalf("expected repairable broken access method finding, got %s", got) + if got := classifyStorageFinding(record, nil); got != storageFindingNone { + t.Fatalf("expected the live canonical locator to keep the record connected, got %s", got) } broken := repairableBrokenAccessRecord(record) if len(broken.AccessProbes) != 1 || broken.AccessProbes[0].ObjectURL != "s3://retired-bucket/JHU/slide.ome.tiff" { @@ -3706,33 +4000,29 @@ func TestApplyStorageCleanupDeletesSelectedBucketSyfonNoGitChainFinding(t *testi if err != nil { t.Fatalf("apply cleanup: %v", err) } - if !contains(response.DeletedRecordIDs, "obj-no-git") { - t.Fatalf("expected selected Syfon record to be deleted, got %+v", response) - } - if !contains(response.DeletedBucketObjectURLs, "s3://bucket/no-git") { - t.Fatalf("expected selected bucket object to be deleted, got %+v", response) + if !contains(response.DeletedRecordIDs, "obj-no-git") || !contains(response.DeletedBucketObjectURLs, "s3://bucket/no-git") { + t.Fatalf("expected selected record and bucket object deletion, got %+v", response) } - if !contains(backend.deletedIDs, "obj-no-git") { - t.Fatalf("expected backend Syfon delete, got %+v", backend.deletedIDs) - } - if !contains(backend.deletedBucketObjects, "s3://bucket/no-git") { - t.Fatalf("expected backend bucket delete, got %+v", backend.deletedBucketObjects) + if !contains(backend.deletedIDs, "obj-no-git") || !contains(backend.deletedBucketObjects, "s3://bucket/no-git") { + t.Fatalf("expected backend deletion, IDs=%v bucket=%v", backend.deletedIDs, backend.deletedBucketObjects) } } -func TestBuildStorageChainAuditMarksBucketSyfonNoGitActionable(t *testing.T) { - actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupport("bucket_syfon_no_git") - if actionability != storageActionabilityAutoRepair { - t.Fatalf("expected auto-repair actionability, got %q", actionability) +func TestBuildStorageChainAuditMarksBucketSyfonNoGitManualRepair(t *testing.T) { + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence("bucket_syfon_no_git", "verified") + if actionability != storageActionabilityManualChoice { + t.Fatalf("expected manual-choice actionability, got %q", actionability) } if defaultAction != storageActionDeleteBoth { t.Fatalf("expected delete-both default action, got %q", defaultAction) } if !supportsDryRun { - t.Fatal("expected bucket+Syfon/no-Git to support dry-run") + t.Fatal("manual repair must advertise a dry-run") } - if !contains(availableActions, storageActionDeleteBoth) || !contains(availableActions, storageActionDeleteSyfonRecord) || !contains(availableActions, storageActionDeleteBucketObject) { - t.Fatalf("expected destructive chain actions, got %+v", availableActions) + for _, action := range []string{storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject} { + if !contains(availableActions, action) { + t.Fatalf("expected action %q, got %+v", action, availableActions) + } } } diff --git a/internal/git/storage_audit_evidence.go b/internal/git/storage_audit_evidence.go new file mode 100644 index 0000000..30d991a --- /dev/null +++ b/internal/git/storage_audit_evidence.go @@ -0,0 +1,56 @@ +package git + +import ( + "context" + "fmt" + "strings" + + "github.com/calypr/gecko/internal/storageaudit" +) + +func (service *StorageAnalyticsService) loadStorageAuditSyfonRevision(ctx context.Context, authorizationHeader string, organization string, project string) (string, error) { + summary, err := service.storage.GetProjectMetricsSummary(ctx, authorizationHeader, organization, project) + if err != nil { + return "", fmt.Errorf("get Syfon revision for storage audit: %w", err) + } + if summary == nil { + return "", fmt.Errorf("get Syfon revision for storage audit: response summary is missing") + } + return fmt.Sprintf( + "%d:%s:%s", + summary.RecordCount, + strings.TrimSpace(summary.RecordLatestUpdatedTime), + strings.TrimSpace(summary.RecordRevision), + ), nil +} + +func assessStorageRecordEvidence(record projectRecordState, bucketObserved bool) storageaudit.Assessment { + probes := make([]storageaudit.Probe, 0, len(record.AccessProbes)) + for _, probe := range record.AccessProbes { + probes = append(probes, storageaudit.Probe{ + Operation: strings.TrimSpace(probe.Operation), + Status: strings.TrimSpace(probe.Status), + ErrorKind: strings.TrimSpace(probe.ErrorKind), + ValidationStatus: strings.TrimSpace(probe.ValidationStatus), + }) + } + return storageaudit.Assess(probes, bucketObserved) +} + +func storageEvidenceStatus(record projectRecordState, bucketObserved bool) string { + return string(assessStorageRecordEvidence(record, bucketObserved).Status) +} + +func storageChainActionSupportForEvidence(kind string, evidenceStatus string) (string, []string, string, bool) { + if strings.TrimSpace(kind) == "git_only_no_syfon" && strings.TrimSpace(evidenceStatus) == string(storageaudit.EvidenceVerified) { + return storageActionabilityManualChoice, + []string{storageActionCreateSyfonRecord, storageActionInspectEvidence}, + storageActionCreateSyfonRecord, + false + } + policy := storageRepairPolicyForKind(kind) + if policy.actionability == storageActionabilityAutoRepair && !storageaudit.AllowsAutomaticRepair(kind, storageaudit.EvidenceStatus(evidenceStatus)) { + policy = storageRepairPolicyForKind("probe_error") + } + return policy.actionability, append([]string(nil), policy.actions...), policy.defaultAction, policy.supportsDryRun +} diff --git a/internal/git/storage_audit_list_confirmation_test.go b/internal/git/storage_audit_list_confirmation_test.go new file mode 100644 index 0000000..0403954 --- /dev/null +++ b/internal/git/storage_audit_list_confirmation_test.go @@ -0,0 +1,106 @@ +package git + +import ( + "context" + "testing" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" +) + +func TestBuildStorageChainAuditRequiresMetadataToConfirmListMiss(t *testing.T) { + const ( + repoPath = "data/file.bin" + checksum = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + canonicalURL = "s3://bucket/root/data/file.bin" + ) + tests := []struct { + name string + metadataResult gintegrationsyfon.BulkStorageProbeResult + wantMissing int + wantProbeError int + }{ + { + name: "metadata confirms missing", + metadataResult: gintegrationsyfon.BulkStorageProbeResult{ + Status: "not_found", + ErrorKind: "object_not_found", + }, + wantMissing: 1, + }, + { + name: "metadata failure leaves absence unknown", + metadataResult: gintegrationsyfon.BulkStorageProbeResult{ + Status: "error", + ErrorKind: "storage_error", + Error: "metadata lookup failed", + }, + wantProbeError: 1, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + repoPath: lfsPointer(checksum, 100), + }) + listKey := storageListValidationRequestKey(canonicalURL, 100, "file.bin") + metadataKey := storageProbeRequestKey(canonicalURL, 100, checksum) + metadataResult := test.metadataResult + metadataResult.ID = metadataKey + metadataResult.ObjectURL = canonicalURL + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{{ + ObjectID: "object-id", + Name: "file.bin", + Checksum: checksum, + Organization: "org", + Project: "project", + Size: 100, + AccessURLs: []string{canonicalURL}, + }}, + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bucket", + Organization: "org", + ProjectID: "project", + Path: "s3://bucket/root", + }}, + listProbeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + listKey: { + ID: listKey, + ObjectURL: canonicalURL, + Status: "not_found", + ErrorKind: "object_not_found", + }, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + metadataKey: metadataResult, + }, + } + + chain, err := NewStorageAnalyticsService(backend).BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "project", + refName, + "", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build storage chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != test.wantMissing { + t.Fatalf("unexpected missing count %d in summary %+v", got, chain.Summary) + } + if got := chain.Summary.CountsByKind["probe_error"]; got != test.wantProbeError { + t.Fatalf("unexpected probe-error count %d in summary %+v", got, chain.Summary) + } + if backend.listProbeCalls != 1 || backend.probeCalls != 1 { + t.Fatalf("expected one LIST candidate check and one metadata confirmation, got list=%d metadata=%d", backend.listProbeCalls, backend.probeCalls) + } + }) + } +} diff --git a/internal/git/storage_audit_registration.go b/internal/git/storage_audit_registration.go new file mode 100644 index 0000000..4cff8fc --- /dev/null +++ b/internal/git/storage_audit_registration.go @@ -0,0 +1,233 @@ +package git + +import ( + "context" + "fmt" + "path" + "sort" + "strings" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +type gitOnlyRegistrationCandidate struct { + checksum string + name string + size int64 + accessURLs []string + resultIndexes []int +} + +// RegisterGitOnlySyfonRecords restores only missing Syfon records. It always +// re-reads the Git LFS pointer, confirms the current scoped bucket object, and +// rejects size drift before sending a DRS registration request to Syfon. +func (service *StorageAnalyticsService) RegisterGitOnlySyfonRecords(ctx context.Context, authorizationHeader string, organization string, project string, ref string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, request GitOnlySyfonRegistrationRequest) (*GitOnlySyfonRegistrationResponse, error) { + expectedRevision := strings.TrimSpace(request.ExpectedGitRevision) + if expectedRevision == "" { + return nil, fmt.Errorf("git-only Syfon registration requires expected_git_revision") + } + if expectedRevision != hash.String() { + return nil, fmt.Errorf("git-only Syfon registration refused because Git revision changed from %s to %s; refresh the audit and try again", expectedRevision, hash.String()) + } + paths := normalizedGitOnlyRegistrationPaths(request.RepoPaths) + if len(paths) == 0 { + return nil, fmt.Errorf("git-only Syfon registration requires at least one repo path") + } + inventory, err := service.loadStorageChainInventory(ctx, ref, "", mirrorPath, repo, hash) + if err != nil { + return nil, fmt.Errorf("load Git inventory for Syfon registration: %w", err) + } + byPath := make(map[string]RepoInventoryFile, len(inventory)) + for _, item := range inventory { + byPath[normalizeRepoSubpath(item.RepoPath)] = item + } + results := make([]GitOnlySyfonRegistrationResult, len(paths)) + checksums := make([]string, 0, len(paths)) + for index, repoPath := range paths { + results[index].NormalizedPath = repoPath + item, ok := byPath[repoPath] + if !ok { + results[index].Status = "skipped" + results[index].Reason = "path is no longer a Git LFS pointer at the audited revision" + continue + } + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" || item.Size < 0 { + results[index].Status = "skipped" + results[index].Reason = "Git LFS pointer does not contain a valid SHA-256 and size" + continue + } + results[index].Checksum = checksum + results[index].GitSizeBytes = item.Size + checksums = append(checksums, checksum) + } + existing, err := service.storage.BulkGetProjectRecordsByChecksum(ctx, authorizationHeader, organization, project, uniqueStrings(checksums)) + if err != nil { + return nil, fmt.Errorf("recheck Syfon records before registration: %w", err) + } + scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("load project storage scopes before registration: %w", err) + } + probes := make([]gintegrationsyfon.BulkStorageProbeItem, 0, len(results)) + probeResultIndexes := make(map[string]int, len(results)) + for index := range results { + result := &results[index] + if result.Status != "" { + continue + } + if len(existing[result.Checksum]) > 0 { + result.Status = "skipped" + result.Reason = "a Syfon record for this SHA-256 already exists" + continue + } + urls := projectScopeRepoPathObjectURLs([]string{result.NormalizedPath}, scopes, organization, project) + if len(urls) != 1 { + result.Status = "skipped" + result.Reason = gitOnlyRegistrationScopeReason(urls, scopes, organization, project) + continue + } + probeID := fmt.Sprintf("git-only-register-%d", index) + size := result.GitSizeBytes + probes = append(probes, gintegrationsyfon.BulkStorageProbeItem{ + ID: probeID, + ObjectURL: urls[0], + ExpectedSizeBytes: &size, + }) + probeResultIndexes[probeID] = index + } + if len(probes) > 0 { + observed, err := service.storage.BulkProbeStorageObjects(ctx, authorizationHeader, probes) + if err != nil { + return nil, fmt.Errorf("recheck bucket objects before registration: %w", err) + } + byProbeID := make(map[string]gintegrationsyfon.BulkStorageProbeResult, len(observed)) + for _, probe := range observed { + byProbeID[strings.TrimSpace(probe.ID)] = probe + } + for _, item := range probes { + index := probeResultIndexes[item.ID] + result := &results[index] + probe, ok := byProbeID[item.ID] + if !ok { + result.Status = "skipped" + result.Reason = "bucket verification did not return a result" + continue + } + result.BucketObjectURL = canonicalStorageURL(probe.Bucket, probe.Key, item.ObjectURL) + if result.BucketObjectURL == "" { + result.BucketObjectURL = item.ObjectURL + } + result.BucketSizeBytes = derefInt64(probe.SizeBytes) + if !probe.Exists || strings.TrimSpace(probe.Status) != "present" { + result.Status = "skipped" + result.Reason = "mapped bucket object is not currently present" + continue + } + if probe.SizeBytes == nil || *probe.SizeBytes != result.GitSizeBytes { + result.Status = "skipped" + result.Reason = fmt.Sprintf("Git LFS size is %d B but bucket size is %d B", result.GitSizeBytes, result.BucketSizeBytes) + continue + } + result.Status = "eligible" + } + } + candidates := buildGitOnlyRegistrationCandidates(results) + if len(candidates) > 0 { + registrations := make([]gintegrationsyfon.ProjectObjectRegistration, 0, len(candidates)) + controlledAccess := []string{"/organization/" + strings.TrimSpace(organization) + "/project/" + strings.TrimSpace(project)} + for _, candidate := range candidates { + registrations = append(registrations, gintegrationsyfon.ProjectObjectRegistration{ + Name: candidate.name, + Checksum: candidate.checksum, + Size: candidate.size, + ControlledAccess: controlledAccess, + AccessURLs: candidate.accessURLs, + }) + } + registrationResults, err := service.storage.RegisterProjectObjects(ctx, authorizationHeader, registrations) + if err != nil { + return nil, fmt.Errorf("register missing Syfon records: %w", err) + } + for candidateIndex, candidate := range candidates { + objectID := registrationResults[candidateIndex].ObjectID + for _, resultIndex := range candidate.resultIndexes { + results[resultIndex].Status = "created" + results[resultIndex].ObjectID = objectID + results[resultIndex].Reason = "Syfon record created after Git pointer and bucket size verification; bucket SHA-256 was not available for verification" + } + } + service.evictProjectJoinCache(organization, project) + service.evictProjectAuditRecordCache(organization, project) + } + return &GitOnlySyfonRegistrationResponse{GitRevision: hash.String(), Results: results}, nil +} + +func normalizedGitOnlyRegistrationPaths(paths []string) []string { + out := make([]string, 0, len(paths)) + seen := make(map[string]struct{}, len(paths)) + for _, repoPath := range paths { + normalized := normalizeRepoSubpath(repoPath) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + sort.Strings(out) + return out +} + +func gitOnlyRegistrationScopeReason(urls []string, scopes []domain.StorageBucketScope, organization string, project string) string { + if len(urls) == 0 { + return fmt.Sprintf("no configured bucket mapping applies to project %s/%s", organization, project) + } + return fmt.Sprintf("multiple configured bucket mappings apply (%d); choose a single storage mapping before creating a Syfon record", len(urls)) +} + +func buildGitOnlyRegistrationCandidates(results []GitOnlySyfonRegistrationResult) []gitOnlyRegistrationCandidate { + byChecksum := make(map[string]*gitOnlyRegistrationCandidate) + for index := range results { + result := &results[index] + if result.Status != "eligible" { + continue + } + candidate := byChecksum[result.Checksum] + if candidate == nil { + candidate = &gitOnlyRegistrationCandidate{ + checksum: result.Checksum, + name: path.Base(result.NormalizedPath), + size: result.GitSizeBytes, + } + byChecksum[result.Checksum] = candidate + } + if candidate.size != result.GitSizeBytes { + result.Status = "skipped" + result.Reason = "matching Git SHA-256 has inconsistent pointer sizes" + continue + } + candidate.accessURLs = append(candidate.accessURLs, result.BucketObjectURL) + candidate.resultIndexes = append(candidate.resultIndexes, index) + } + checksums := make([]string, 0, len(byChecksum)) + for checksum := range byChecksum { + checksums = append(checksums, checksum) + } + sort.Strings(checksums) + out := make([]gitOnlyRegistrationCandidate, 0, len(checksums)) + for _, checksum := range checksums { + candidate := byChecksum[checksum] + candidate.accessURLs = uniqueStrings(candidate.accessURLs) + if len(candidate.resultIndexes) == 0 { + continue + } + out = append(out, *candidate) + } + return out +} diff --git a/internal/git/storage_audit_registration_test.go b/internal/git/storage_audit_registration_test.go new file mode 100644 index 0000000..477e20c --- /dev/null +++ b/internal/git/storage_audit_registration_test.go @@ -0,0 +1,88 @@ +package git + +import ( + "context" + "testing" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" +) + +func TestRegisterGitOnlySyfonRecordsUsesScopedBucketEvidence(t *testing.T) { + checksum := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "OHSU/slide.ome.tiff": lfsPointer(checksum, 100), + }) + objectURL := "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff" + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: objectURL, + Bucket: "bforepc", + Key: "bforepc-prod/OHSU/slide.ome.tiff", + SizeBytes: 100, + }}, + } + service := NewStorageAnalyticsService(backend) + + audit, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build audit: %v", err) + } + finding := assertHasChainFinding(t, audit.Findings, "git_only_no_syfon", "OHSU/slide.ome.tiff") + if finding.BucketObjectURL != objectURL || finding.BucketSizeBytes != 100 || finding.Evidence == nil || finding.Evidence.BucketEvaluation != "present" { + t.Fatalf("expected audit to expose mapped bucket evidence, got %+v", finding) + } + + response, err := service.RegisterGitOnlySyfonRecords(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, mirrorPath, repo, hash, GitOnlySyfonRegistrationRequest{ + ExpectedGitRevision: hash.String(), + RepoPaths: []string{"OHSU/slide.ome.tiff"}, + }) + if err != nil { + t.Fatalf("register missing Syfon record: %v", err) + } + if len(response.Results) != 1 || response.Results[0].Status != "created" || response.Results[0].BucketObjectURL != objectURL { + t.Fatalf("unexpected registration response: %+v", response) + } + if len(backend.registeredObjects) != 1 { + t.Fatalf("expected one Syfon registration, got %+v", backend.registeredObjects) + } + registered := backend.registeredObjects[0] + if registered.Checksum != checksum || registered.Size != 100 || registered.Name != "slide.ome.tiff" { + t.Fatalf("unexpected Syfon candidate: %+v", registered) + } + if len(registered.AccessURLs) != 1 || registered.AccessURLs[0] != objectURL { + t.Fatalf("expected canonical scoped access URL, got %+v", registered.AccessURLs) + } +} + +func TestRegisterGitOnlySyfonRecordsRefusesBucketSizeMismatch(t *testing.T) { + checksum := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.bin": lfsPointer(checksum, 100), + }) + objectURL := "s3://bucket/root/data/a.bin" + actualSize := int64(99) + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{{Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket/root"}}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + "git-only-register-0": {ID: "git-only-register-0", ObjectURL: objectURL, Exists: true, Status: "present", SizeBytes: &actualSize}, + }, + } + service := NewStorageAnalyticsService(backend) + response, err := service.RegisterGitOnlySyfonRecords(context.Background(), "Bearer token", "org", "proj", refName, mirrorPath, repo, hash, GitOnlySyfonRegistrationRequest{ + ExpectedGitRevision: hash.String(), + RepoPaths: []string{"data/a.bin"}, + }) + if err != nil { + t.Fatalf("register missing Syfon record: %v", err) + } + if len(backend.registeredObjects) != 0 || len(response.Results) != 1 || response.Results[0].Status != "skipped" { + t.Fatalf("expected size mismatch to block registration, got response=%+v registrations=%+v", response, backend.registeredObjects) + } +} diff --git a/internal/git/storage_audit_revalidation.go b/internal/git/storage_audit_revalidation.go new file mode 100644 index 0000000..caf39aa --- /dev/null +++ b/internal/git/storage_audit_revalidation.go @@ -0,0 +1,147 @@ +package git + +import ( + "context" + "fmt" + "strings" + + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" +) + +const ( + cleanupExpectationPresent = "present" + cleanupExpectationMissing = "missing" +) + +type storageCleanupVerification struct { + FindingKind string + NormalizedPath string + ObjectURL string + Expectation string +} + +func appendStorageCleanupVerification(plan *storageCleanupApplyPlan, finding GitStorageCleanupApplyFinding, action string) error { + if plan == nil || action == storageActionInspectEvidence { + return nil + } + expectation := cleanupVerificationExpectation(finding.Kind, action) + if expectation == "" { + return nil + } + objectURLs := storageApplyFindingBucketObjectURLs(finding) + if len(objectURLs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing bucket object URLs required for exact revalidation", finding.Kind, finding.NormalizedPath) + } + for _, objectURL := range objectURLs { + plan.Verifications = append(plan.Verifications, storageCleanupVerification{ + FindingKind: strings.TrimSpace(finding.Kind), + NormalizedPath: strings.TrimSpace(finding.NormalizedPath), + ObjectURL: strings.TrimSpace(objectURL), + Expectation: expectation, + }) + } + return nil +} + +func cleanupVerificationExpectation(kind string, action string) string { + switch action { + case storageActionDeleteBucketObject, storageActionDeleteBoth: + return cleanupExpectationPresent + case storageActionDeleteSyfonRecord: + switch strings.TrimSpace(kind) { + case "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object": + return cleanupExpectationMissing + } + } + return "" +} + +func (service *StorageAnalyticsService) verifyStorageCleanupApplyPlan(ctx context.Context, authorizationHeader string, plan *storageCleanupApplyPlan) error { + if plan == nil { + return nil + } + verifications := uniqueStorageCleanupVerifications(plan.Verifications) + if len(verifications) == 0 { + return nil + } + items := make([]gintegrationsyfon.BulkStorageProbeItem, 0, len(verifications)) + for index, verification := range verifications { + items = append(items, gintegrationsyfon.BulkStorageProbeItem{ + ID: fmt.Sprintf("cleanup-revalidate-%d", index), + ObjectURL: verification.ObjectURL, + }) + } + results, err := service.storage.BulkProbeStorageObjects(ctx, authorizationHeader, items) + if err != nil { + return fmt.Errorf("revalidate cleanup storage evidence: %w", err) + } + if len(results) != len(items) { + return fmt.Errorf("revalidate cleanup storage evidence: expected %d results, received %d", len(items), len(results)) + } + for index, result := range results { + if cleanupObjectVanishedAfterAudit(verifications[index], result) { + plan.DeleteBucketObjects = differenceStrings(plan.DeleteBucketObjects, []string{verifications[index].ObjectURL}) + plan.SkippedPaths = append(plan.SkippedPaths, verifications[index].NormalizedPath) + continue + } + if err := validateStorageCleanupResult(verifications[index], result); err != nil { + return err + } + } + return nil +} + +func cleanupObjectVanishedAfterAudit(verification storageCleanupVerification, result gintegrationsyfon.BulkStorageProbeResult) bool { + if verification.Expectation != cleanupExpectationPresent { + return false + } + if strings.TrimSpace(result.Status) != "not_found" || strings.TrimSpace(result.ErrorKind) != "object_not_found" { + return false + } + switch strings.TrimSpace(verification.FindingKind) { + case "bucket_only_object", "bucket_syfon_no_git": + return true + default: + return false + } +} + +func uniqueStorageCleanupVerifications(input []storageCleanupVerification) []storageCleanupVerification { + seen := make(map[string]struct{}, len(input)) + out := make([]storageCleanupVerification, 0, len(input)) + for _, verification := range input { + key := verification.Expectation + "\x00" + verification.ObjectURL + if verification.ObjectURL == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, verification) + } + return out +} + +func validateStorageCleanupResult(verification storageCleanupVerification, result gintegrationsyfon.BulkStorageProbeResult) error { + status := strings.TrimSpace(result.Status) + switch verification.Expectation { + case cleanupExpectationPresent: + if status == "present" && result.Exists { + return nil + } + case cleanupExpectationMissing: + if status == "not_found" && strings.TrimSpace(result.ErrorKind) == "object_not_found" { + return nil + } + } + return fmt.Errorf( + "revalidate cleanup finding %q for %q: expected %s, got status=%q error_kind=%q error=%q", + verification.FindingKind, + verification.ObjectURL, + verification.Expectation, + status, + strings.TrimSpace(result.ErrorKind), + strings.TrimSpace(result.Error), + ) +} diff --git a/internal/git/storage_audit_revalidation_test.go b/internal/git/storage_audit_revalidation_test.go new file mode 100644 index 0000000..c49cc9e --- /dev/null +++ b/internal/git/storage_audit_revalidation_test.go @@ -0,0 +1,191 @@ +package git + +import ( + "context" + "strings" + "testing" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" +) + +func TestApplyStorageCleanupRejectsStaleMissingObjectEvidence(t *testing.T) { + finding := applyFinding( + "syfon_git_no_bucket", + "data/file.bin", + []string{"object-id"}, + []string{"s3://bucket/data/file.bin"}, + ) + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + + _, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "project", + nil, + nil, + []GitStorageCleanupApplyFinding{finding}, + false, + false, + false, + false, + false, + ) + if err == nil || !strings.Contains(err.Error(), "expected missing, got status=\"present\"") { + t.Fatalf("expected stale missing evidence to abort cleanup, got %v", err) + } + if len(backend.deletedIDs) != 0 || len(backend.deletedBucketObjects) != 0 { + t.Fatalf("revalidation failure must not mutate storage, deleted IDs=%v bucket objects=%v", backend.deletedIDs, backend.deletedBucketObjects) + } +} + +func TestApplyStorageCleanupAcceptsFreshExactMissingEvidence(t *testing.T) { + finding := applyFinding( + "syfon_git_no_bucket", + "data/file.bin", + []string{"object-id"}, + []string{"s3://bucket/data/file.bin"}, + ) + backend := &fakeStorageAnalyticsBackend{ + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + "cleanup-revalidate-0": { + ID: "cleanup-revalidate-0", + Status: "not_found", + ErrorKind: "object_not_found", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + result, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "project", + nil, + nil, + []GitStorageCleanupApplyFinding{finding}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply cleanup with fresh exact evidence: %v", err) + } + if len(result.DeletedRecordIDs) != 1 || result.DeletedRecordIDs[0] != "object-id" { + t.Fatalf("expected exact-missing record deletion, got %+v", result) + } +} + +func TestApplyStorageCleanupCanonicalizesScopedBucketURLsBeforeRevalidation(t *testing.T) { + rawURL := "s3://bforepc-prod/02bd1708-9071-5950-aed4-38a7f3090900/file.bin" + canonicalURL := "s3://bforepc/bforepc-prod/02bd1708-9071-5950-aed4-38a7f3090900/file.bin" + finding := applyFinding( + "bucket_syfon_no_git", + rawURL, + []string{"object-id"}, + []string{rawURL}, + ) + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + "cleanup-revalidate-0": { + ID: "cleanup-revalidate-0", + ObjectURL: canonicalURL, + Bucket: "bforepc", + Key: "bforepc-prod/02bd1708-9071-5950-aed4-38a7f3090900/file.bin", + Exists: true, + Status: "present", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + result, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "HTAN_INT", + "BForePC", + []string{rawURL}, + []GitStorageCleanupApplyAction{{ + Kind: finding.Kind, + NormalizedPath: finding.NormalizedPath, + Action: storageActionDeleteBoth, + }}, + []GitStorageCleanupApplyFinding{finding}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply canonicalized cleanup: %v", err) + } + if len(backend.probeItems) != 1 || backend.probeItems[0].ObjectURL != canonicalURL { + t.Fatalf("expected canonical revalidation target %q, got %+v", canonicalURL, backend.probeItems) + } + if len(backend.deletedBucketObjects) != 1 || backend.deletedBucketObjects[0] != canonicalURL { + t.Fatalf("expected canonical bucket delete target %q, got %+v", canonicalURL, backend.deletedBucketObjects) + } + if len(result.DeletedRecordIDs) != 1 || result.DeletedRecordIDs[0] != "object-id" { + t.Fatalf("expected Syfon record deletion, got %+v", result) + } +} + +func TestApplyStorageCleanupContinuesWhenBucketOrphanVanishesAfterAudit(t *testing.T) { + missingURL := "s3://bucket/orphan-gone" + presentURL := "s3://bucket/orphan-present" + missingFinding := applyFinding("bucket_syfon_no_git", missingURL, []string{"object-gone"}, []string{missingURL}) + presentFinding := applyFinding("bucket_syfon_no_git", presentURL, []string{"object-present"}, []string{presentURL}) + backend := &fakeStorageAnalyticsBackend{ + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + "cleanup-revalidate-0": { + ID: "cleanup-revalidate-0", + ObjectURL: missingURL, + Status: "not_found", + ErrorKind: "object_not_found", + }, + "cleanup-revalidate-1": { + ID: "cleanup-revalidate-1", + ObjectURL: presentURL, + Exists: true, + Status: "present", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + result, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "project", + []string{missingURL, presentURL}, + []GitStorageCleanupApplyAction{ + {Kind: missingFinding.Kind, NormalizedPath: missingFinding.NormalizedPath, Action: storageActionDeleteBoth}, + {Kind: presentFinding.Kind, NormalizedPath: presentFinding.NormalizedPath, Action: storageActionDeleteBoth}, + }, + []GitStorageCleanupApplyFinding{missingFinding, presentFinding}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply cleanup after one bucket object vanished: %v", err) + } + assertStringSet(t, "deleted Syfon records", result.DeletedRecordIDs, []string{"object-gone", "object-present"}) + assertStringSet(t, "deleted bucket objects", result.DeletedBucketObjectURLs, []string{presentURL}) + assertStringSet(t, "skipped paths", result.SkippedPaths, []string{missingURL}) +} diff --git a/internal/git/storage_chain_audit_cache.go b/internal/git/storage_chain_audit_cache.go index 7aa2c4f..6575505 100644 --- a/internal/git/storage_chain_audit_cache.go +++ b/internal/git/storage_chain_audit_cache.go @@ -17,8 +17,8 @@ import ( ) const ( - defaultStorageChainAuditCacheTTL = 24 * time.Hour - storageChainAuditCacheKeyPrefix = "gecko:storage_chain_audit:v1:" + defaultStorageChainAuditCacheTTL = 5 * time.Minute + storageChainAuditCacheKeyPrefix = "gecko:storage_chain_audit:v2:" storageExactJoinCacheKeyPrefix = "gecko:storage_exact_join:v1:" ) @@ -29,8 +29,44 @@ type storageChainAuditResponseCache interface { } type cachedStorageChainAuditResponse struct { - CachedAt time.Time `json:"cached_at"` - Response GitStorageChainAuditResponse `json:"response"` + CachedAt time.Time `json:"cached_at"` + RefreshDurationMillis int64 `json:"refresh_duration_ms,omitempty"` + Response GitStorageChainAuditResponse `json:"response"` +} + +type inflightStorageChainAuditRefresh struct { + done chan struct{} + value cachedStorageChainAuditResponse + err error +} + +// coalesceStorageChainAuditRefresh ensures a root audit has one authoritative +// refresh result per cache key. The UI can issue overlapping forced refreshes; +// without this guard, a partial bucket LIST from the later request can replace +// the completed response from the earlier request. +func (service *StorageAnalyticsService) coalesceStorageChainAuditRefresh(ctx context.Context, key string, refresh func() (cachedStorageChainAuditResponse, error)) (cachedStorageChainAuditResponse, bool, error) { + service.chainAuditRefreshMu.Lock() + if work := service.chainAuditRefreshWork[key]; work != nil { + service.chainAuditRefreshMu.Unlock() + select { + case <-ctx.Done(): + return cachedStorageChainAuditResponse{}, true, ctx.Err() + case <-work.done: + return cloneCachedStorageChainAuditResponse(work.value), true, work.err + } + } + work := &inflightStorageChainAuditRefresh{done: make(chan struct{})} + service.chainAuditRefreshWork[key] = work + service.chainAuditRefreshMu.Unlock() + + value, err := refresh() + service.chainAuditRefreshMu.Lock() + work.value = cloneCachedStorageChainAuditResponse(value) + work.err = err + delete(service.chainAuditRefreshWork, key) + close(work.done) + service.chainAuditRefreshMu.Unlock() + return value, false, err } type storageExactProjectJoinCache interface { @@ -291,13 +327,14 @@ func storageChainAuditCacheTTL() time.Duration { return time.Duration(seconds) * time.Second } -func storageChainAuditResponseCacheKey(organization string, project string, ref string, gitSubpath string, probeMode string, validationMode string, bucketMode string, bucketPathPrefix string, hash string) string { +func storageChainAuditResponseCacheKey(organization string, project string, ref string, gitSubpath string, probeMode string, validationMode string, bucketMode string, bucketPathPrefix string, hash string, syfonRevision string) string { body := fmt.Sprintf( - "org=%s\nproject=%s\nref=%s\nhash=%s\ngit_subpath=%s\nprobe_mode=%s\nvalidation_mode=%s\nbucket_mode=%s\nbucket_path_prefix=%s", + "org=%s\nproject=%s\nref=%s\nhash=%s\nsyfon_revision=%s\ngit_subpath=%s\nprobe_mode=%s\nvalidation_mode=%s\nbucket_mode=%s\nbucket_path_prefix=%s", strings.TrimSpace(organization), strings.TrimSpace(project), strings.TrimSpace(ref), strings.TrimSpace(hash), + strings.TrimSpace(syfonRevision), normalizeRepoSubpath(gitSubpath), strings.TrimSpace(probeMode), strings.TrimSpace(validationMode), @@ -321,8 +358,9 @@ func storageExactProjectJoinCacheKey(organization string, project string, hash s func cloneCachedStorageChainAuditResponse(value cachedStorageChainAuditResponse) cachedStorageChainAuditResponse { return cachedStorageChainAuditResponse{ - CachedAt: value.CachedAt, - Response: cloneStorageChainAuditResponse(value.Response), + CachedAt: value.CachedAt, + RefreshDurationMillis: value.RefreshDurationMillis, + Response: cloneStorageChainAuditResponse(value.Response), } } diff --git a/internal/git/types.go b/internal/git/types.go index edb7fb9..93685dc 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -12,6 +12,7 @@ import ( "github.com/calypr/gecko/internal/httpclient" "github.com/calypr/gecko/internal/integrations/fence" gitapi "github.com/calypr/gecko/internal/integrations/github" + "github.com/calypr/gecko/internal/storageaudit" "github.com/jmoiron/sqlx" ) @@ -342,275 +343,32 @@ type GitStorageFolderResponse struct { Children GitStorageChildrenResponse `json:"children"` } -type GitProjectDiffAuditRequest struct { - GitSubpath string `json:"git_subpath,omitempty"` - Ref string `json:"ref,omitempty"` -} - -type GitAuditEvidence struct { - Checksum string `json:"checksum,omitempty"` - SourcePaths []string `json:"source_paths,omitempty"` - ObjectIDs []string `json:"object_ids,omitempty"` - AccessURLs []string `json:"access_urls,omitempty"` - BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` - Buckets []string `json:"buckets,omitempty"` - Keys []string `json:"keys,omitempty"` - StorageOperations []string `json:"storage_operations,omitempty"` - ProbeStatuses []string `json:"probe_statuses,omitempty"` - ValidationStates []string `json:"validation_states,omitempty"` - ErrorKinds []string `json:"error_kinds,omitempty"` - Errors []string `json:"errors,omitempty"` - BucketEvaluation string `json:"bucket_evaluation,omitempty"` -} - -type GitProjectDiffFinding struct { - Kind string `json:"kind"` - NormalizedPath string `json:"normalized_path"` - Checksum string `json:"checksum,omitempty"` - SourcePaths []string `json:"source_paths,omitempty"` - ObjectIDs []string `json:"object_ids"` - RecordCount int `json:"record_count"` - SizeBytes int64 `json:"size_bytes,omitempty"` - DownloadCount int64 `json:"download_count,omitempty"` - LastDownload string `json:"last_download_time,omitempty"` - RecommendedAction string `json:"recommended_action"` - Evidence *GitAuditEvidence `json:"evidence,omitempty"` -} - -type GitProjectDiffSummary struct { - CountsByKind map[string]int `json:"counts_by_kind"` - TotalFindings int `json:"total_findings"` - IndexedPathCount int `json:"indexed_path_count"` - ExpectedPathCount int `json:"expected_path_count"` - MatchedPathCount int `json:"matched_path_count"` - IncludesRepoManifest bool `json:"includes_repo_manifest"` - ScannedRecordCount int `json:"scanned_record_count"` -} - -type GitProjectDiffAuditResponse struct { - Findings []GitProjectDiffFinding `json:"findings"` - Summary GitProjectDiffSummary `json:"summary"` - PathPrefix string `json:"path_prefix"` -} - -type GitStorageCleanupAuditRequest struct { - GitSubpath string `json:"git_subpath,omitempty"` - Ref string `json:"ref,omitempty"` - CheckStorage bool `json:"check_storage,omitempty"` - SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` -} - -type GitStorageChainAuditRequest struct { - GitSubpath string `json:"git_subpath,omitempty"` - Ref string `json:"ref,omitempty"` - ProbeMode string `json:"probe_mode,omitempty"` - ValidationMode string `json:"validation_mode,omitempty"` - BucketInventoryMode string `json:"bucket_inventory_mode,omitempty"` - BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` - FindingKind string `json:"finding_kind,omitempty"` - FindingLimit int `json:"finding_limit,omitempty"` - ForceAuditRefresh bool `json:"force_audit_refresh,omitempty"` - // ForceBucketInventoryRefresh is accepted as a compatibility alias for older frontend branches. - ForceBucketInventoryRefresh bool `json:"force_bucket_inventory_refresh,omitempty"` -} - -type GitStorageChainFinding struct { - Kind string `json:"kind"` - NormalizedPath string `json:"normalized_path"` - Checksum string `json:"checksum,omitempty"` - SourcePaths []string `json:"source_paths,omitempty"` - ObjectIDs []string `json:"object_ids"` - Records []GitStorageCleanupRecordAudit `json:"records,omitempty"` - AccessURLs []string `json:"access_urls,omitempty"` - BucketObjectURL string `json:"bucket_object_url,omitempty"` - ResolvedBucket string `json:"resolved_bucket,omitempty"` - ResolvedKey string `json:"resolved_key,omitempty"` - ProbeStatus string `json:"probe_status,omitempty"` - ErrorKind string `json:"error_kind,omitempty"` - Error string `json:"error,omitempty"` - RecordCount int `json:"record_count"` - SizeBytes int64 `json:"size_bytes,omitempty"` - RecommendedAction string `json:"recommended_action"` - SuggestedFix string `json:"suggested_fix,omitempty"` - SuggestedAction string `json:"suggested_action,omitempty"` - Actionability string `json:"actionability,omitempty"` - AvailableActions []string `json:"available_actions,omitempty"` - DefaultAction string `json:"default_action,omitempty"` - SupportsDryRun bool `json:"supports_dry_run,omitempty"` - Evidence *GitAuditEvidence `json:"evidence,omitempty"` -} - -type GitStorageChainAuditSummary struct { - CountsByKind map[string]int `json:"counts_by_kind"` - TotalFindings int `json:"total_findings"` - ReturnedFindings int `json:"returned_findings"` - FindingsTruncated bool `json:"findings_truncated"` - FindingLimit int `json:"finding_limit,omitempty"` - ValidationMode string `json:"validation_mode,omitempty"` - BucketObjectCount int `json:"bucket_object_count"` - SyfonRecordCount int `json:"syfon_record_count"` - GitTrackedFileCount int `json:"git_tracked_file_count"` - BucketPathExists *bool `json:"bucket_path_exists,omitempty"` - BucketPathObjectURL string `json:"bucket_path_object_url,omitempty"` - BucketSummaryMode string `json:"bucket_summary_mode,omitempty"` - BucketInventoryAvailable bool `json:"bucket_inventory_available"` - BucketInventoryError string `json:"bucket_inventory_error,omitempty"` - AuditCacheHit bool `json:"audit_cache_hit,omitempty"` - AuditCachedAt string `json:"audit_cached_at,omitempty"` - AuditCacheAgeSeconds int64 `json:"audit_cache_age_seconds,omitempty"` - AuditCacheSource string `json:"audit_cache_source,omitempty"` - AuditCacheError string `json:"audit_cache_error,omitempty"` -} - -type GitStorageChainIssueGroup struct { - Kind string `json:"kind"` - FindingCount int `json:"finding_count"` - PathCount int `json:"path_count"` - RecordCount int `json:"record_count"` - ObjectCount int `json:"object_count"` - TotalBytes int64 `json:"total_bytes,omitempty"` -} - -type GitStorageChainAuditResponse struct { - Findings []GitStorageChainFinding `json:"findings"` - Groups []GitStorageChainIssueGroup `json:"groups,omitempty"` - Summary GitStorageChainAuditSummary `json:"summary"` - PathPrefix string `json:"path_prefix"` - BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` -} - -type GitStorageCleanupAccessProbe struct { - URL string `json:"url"` - Operation string `json:"operation,omitempty"` - Provider string `json:"provider,omitempty"` - Bucket string `json:"bucket,omitempty"` - Key string `json:"key,omitempty"` - Path string `json:"path,omitempty"` - Exists *bool `json:"exists,omitempty"` - Status string `json:"status,omitempty"` - Error string `json:"error,omitempty"` - ErrorKind string `json:"error_kind,omitempty"` - SizeBytes *int64 `json:"size_bytes,omitempty"` - MetaSHA256 string `json:"meta_sha256,omitempty"` - ETag string `json:"etag,omitempty"` - LastModified string `json:"last_modified,omitempty"` - ValidationStatus string `json:"validation_status,omitempty"` - SizeMatch *bool `json:"size_match,omitempty"` - NameMatch *bool `json:"name_match,omitempty"` - SHA256Match *bool `json:"sha256_match,omitempty"` - ValidationMismatches []string `json:"validation_mismatches,omitempty"` -} - -type GitStorageCleanupAccessMethod struct { - AccessID string `json:"access_id,omitempty"` - Type string `json:"type,omitempty"` - URL string `json:"url,omitempty"` - Headers []string `json:"headers,omitempty"` -} - -type GitStorageCleanupRecordAudit struct { - ObjectID string `json:"object_id"` - Checksum string `json:"checksum,omitempty"` - NormalizedPath string `json:"normalized_path,omitempty"` - CleanupScope string `json:"cleanup_scope"` - AccessURLs []string `json:"access_urls,omitempty"` - AccessMethods []GitStorageCleanupAccessMethod `json:"access_methods,omitempty"` - AccessProbes []GitStorageCleanupAccessProbe `json:"access_probes"` - Status string `json:"status,omitempty"` - Error string `json:"error,omitempty"` - SizeBytes int64 `json:"size,omitempty"` - LastUpdated string `json:"updated_time,omitempty"` - DownloadCount int64 `json:"download_count,omitempty"` - LastDownload string `json:"last_download_time,omitempty"` -} - -type GitStorageCleanupFinding struct { - Kind string `json:"kind"` - NormalizedPath string `json:"normalized_path"` - Checksum string `json:"checksum,omitempty"` - ObjectIDs []string `json:"object_ids"` - Records []GitStorageCleanupRecordAudit `json:"records"` - RecommendedAction string `json:"recommended_action"` - RepoDeleteCandidate bool `json:"repo_delete_candidate"` - CleanupScope string `json:"cleanup_scope"` - SizeBytes int64 `json:"total_bytes,omitempty"` - LastUpdated string `json:"last_updated,omitempty"` - DownloadCount int64 `json:"download_count,omitempty"` - LastDownload string `json:"last_download_time,omitempty"` - Actionability string `json:"actionability,omitempty"` - AvailableActions []string `json:"available_actions,omitempty"` - DefaultAction string `json:"default_action,omitempty"` - SupportsDryRun bool `json:"supports_dry_run,omitempty"` - Evidence *GitAuditEvidence `json:"evidence,omitempty"` -} - -type GitStorageCleanupApplyAction struct { - Action string `json:"action"` - Kind string `json:"kind,omitempty"` - NormalizedPath string `json:"normalized_path,omitempty"` -} - -type GitStorageCleanupApplyFinding struct { - Kind string `json:"kind"` - NormalizedPath string `json:"normalized_path"` - ObjectIDs []string `json:"object_ids,omitempty"` - Records []GitStorageCleanupRecordAudit `json:"records,omitempty"` - BucketObjectURL string `json:"bucket_object_url,omitempty"` - BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` - AccessURLs []string `json:"access_urls,omitempty"` - AvailableActions []string `json:"available_actions,omitempty"` - DefaultAction string `json:"default_action,omitempty"` - SuggestedAction string `json:"suggested_action,omitempty"` - Evidence *GitAuditEvidence `json:"evidence,omitempty"` -} - -type GitStorageCleanupAuditSummary struct { - CountsByKind map[string]int `json:"counts_by_kind"` - TotalFindings int `json:"total_findings"` - ManualFindingCount int `json:"manual_finding_count"` - RepoDeleteCandidateCount int `json:"repo_delete_candidate_count"` - StaleDuplicateCount int `json:"stale_duplicate_count"` - RepoOrphanCount int `json:"repo_orphan_count"` -} - -type GitStorageCleanupAuditResponse struct { - Findings []GitStorageCleanupFinding `json:"findings"` - Summary GitStorageCleanupAuditSummary `json:"summary"` - ExpectedPathCount int `json:"expected_path_count"` - IncludesRepoManifest bool `json:"includes_repo_manifest"` - PathPrefix string `json:"path_prefix"` -} - -type GitStorageCleanupApplyRequest struct { - GitSubpath string `json:"git_subpath,omitempty"` - Ref string `json:"ref,omitempty"` - DeleteRepoOrphans bool `json:"delete_repo_orphans,omitempty"` - DeleteStaleDuplicates bool `json:"delete_stale_duplicates,omitempty"` - DeleteBucketOnlyObjects bool `json:"delete_bucket_only_objects,omitempty"` - RepairBrokenBucketMappings bool `json:"repair_broken_bucket_mappings,omitempty"` - DryRun bool `json:"dry_run,omitempty"` - SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` - Actions []GitStorageCleanupApplyAction `json:"actions,omitempty"` - Findings []GitStorageCleanupApplyFinding `json:"findings,omitempty"` -} - -type GitStorageCleanupPurgeResult struct { - ObjectID string `json:"object_id"` - Success *bool `json:"success"` - Status string `json:"status,omitempty"` - Error string `json:"error,omitempty"` -} - -type GitStorageCleanupApplyResponse struct { - DeletedRecordIDs []string `json:"deleted_record_ids"` - DeletedBucketObjectURLs []string `json:"deleted_bucket_object_urls"` - UpdatedRecordIDs []string `json:"updated_record_ids"` - PurgeResults []GitStorageCleanupPurgeResult `json:"purge_results"` - RepoDeletePaths []string `json:"repo_delete_paths"` - ManualPaths []string `json:"manual_paths"` - SkippedPaths []string `json:"skipped_paths"` - DryRun bool `json:"dry_run"` -} +type GitProjectDiffAuditRequest = storageaudit.ProjectDiffAuditRequest +type GitAuditEvidence = storageaudit.Evidence +type GitProjectDiffFinding = storageaudit.ProjectDiffFinding +type GitProjectDiffSummary = storageaudit.ProjectDiffSummary +type GitProjectDiffAuditResponse = storageaudit.ProjectDiffAuditResponse + +type GitStorageCleanupAuditRequest = storageaudit.CleanupAuditRequest +type GitStorageChainAuditRequest = storageaudit.ChainAuditRequest +type GitStorageChainFinding = storageaudit.ChainFinding +type GitStorageChainAuditSummary = storageaudit.ChainSummary +type GitStorageChainIssueGroup = storageaudit.ChainIssueGroup +type GitStorageChainAuditResponse = storageaudit.ChainAuditResponse +type GitOnlySyfonRegistrationRequest = storageaudit.GitOnlySyfonRegistrationRequest +type GitOnlySyfonRegistrationResult = storageaudit.GitOnlySyfonRegistrationResult +type GitOnlySyfonRegistrationResponse = storageaudit.GitOnlySyfonRegistrationResponse +type GitStorageCleanupAccessProbe = storageaudit.CleanupAccessProbe +type GitStorageCleanupAccessMethod = storageaudit.CleanupAccessMethod +type GitStorageCleanupRecordAudit = storageaudit.CleanupRecordAudit +type GitStorageCleanupFinding = storageaudit.CleanupFinding +type GitStorageCleanupApplyAction = storageaudit.CleanupApplyAction +type GitStorageCleanupApplyFinding = storageaudit.CleanupApplyFinding +type GitStorageCleanupAuditSummary = storageaudit.CleanupAuditSummary +type GitStorageCleanupAuditResponse = storageaudit.CleanupAuditResponse +type GitStorageCleanupApplyRequest = storageaudit.CleanupApplyRequest +type GitStorageCleanupPurgeResult = storageaudit.CleanupPurgeResult +type GitStorageCleanupApplyResponse = storageaudit.CleanupApplyResponse type GitUploadSessionFileManifest struct { Name string `json:"name"` diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index c048ff5..98b9d42 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -156,6 +156,20 @@ type ProjectBucketDeleteResult struct { Error string } +// ProjectObjectRegistration is the narrow subset of DRS registration Gecko +// needs when it restores a Syfon record for an existing Git LFS pointer. +type ProjectObjectRegistration struct { + Name string + Checksum string + Size int64 + ControlledAccess []string + AccessURLs []string +} + +type ProjectObjectRegistrationResult struct { + ObjectID string +} + func NewManager(baseURL string, client *http.Client) *Manager { httpClient := client if httpClient == nil { @@ -639,6 +653,61 @@ func (manager *Manager) BulkDeleteObjects(ctx context.Context, authorizationHead return nil } +func (manager *Manager) RegisterProjectObjects(ctx context.Context, authorizationHeader string, candidates []ProjectObjectRegistration) ([]ProjectObjectRegistrationResult, error) { + if len(candidates) == 0 { + return []ProjectObjectRegistrationResult{}, nil + } + client, err := manager.drsClient(authorizationHeader) + if err != nil { + return nil, err + } + request := drsapi.RegisterObjectsJSONRequestBody{Candidates: make([]drsapi.DrsObjectCandidate, 0, len(candidates))} + for _, candidate := range candidates { + name := strings.TrimSpace(candidate.Name) + checksum := normalizeSHA256(candidate.Checksum) + accessURLs := uniqueNonEmptyStrings(candidate.AccessURLs) + if name == "" || checksum == "" || candidate.Size < 0 || len(accessURLs) == 0 { + return nil, fmt.Errorf("register Syfon object: candidate is missing a name, SHA-256, size, or access URL") + } + accessMethods := make([]drsapi.AccessMethod, 0, len(accessURLs)) + for index, accessURL := range accessURLs { + accessID := fmt.Sprintf("s3-%d", index+1) + accessURLCopy := accessURL + accessMethods = append(accessMethods, drsapi.AccessMethod{ + AccessId: &accessID, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: accessURLCopy}, + Type: drsapi.AccessMethodType("s3"), + }) + } + controlledAccess := uniqueNonEmptyStrings(candidate.ControlledAccess) + request.Candidates = append(request.Candidates, drsapi.DrsObjectCandidate{ + Name: &name, + Size: candidate.Size, + Checksums: []drsapi.Checksum{{Type: "sha256", Checksum: checksum}}, + ControlledAccess: &controlledAccess, + AccessMethods: &accessMethods, + }) + } + response, err := client.RegisterObjectsWithResponse(ctx, request) + if err != nil { + return nil, fmt.Errorf("register Syfon objects: %w", err) + } + if response.StatusCode() != http.StatusCreated || response.JSON201 == nil { + return nil, fmt.Errorf("register Syfon objects failed with status %d: %s", response.StatusCode(), strings.TrimSpace(string(response.Body))) + } + if len(response.JSON201.Objects) != len(candidates) { + return nil, fmt.Errorf("register Syfon objects: expected %d objects, received %d", len(candidates), len(response.JSON201.Objects)) + } + results := make([]ProjectObjectRegistrationResult, len(response.JSON201.Objects)) + for index, object := range response.JSON201.Objects { + results[index] = ProjectObjectRegistrationResult{ObjectID: strings.TrimSpace(object.Id)} + } + return results, nil +} + func (manager *Manager) DeleteProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, objectURLs []string) ([]ProjectBucketDeleteResult, error) { requestBody := struct { Organization string `json:"organization,omitempty"` diff --git a/internal/integrations/syfon/adapter_test.go b/internal/integrations/syfon/adapter_test.go index 13c2401..bfe8ce3 100644 --- a/internal/integrations/syfon/adapter_test.go +++ b/internal/integrations/syfon/adapter_test.go @@ -329,6 +329,58 @@ func TestBulkDeleteObjectsUsesBulkDRSDeleteEndpoint(t *testing.T) { } } +func TestRegisterProjectObjectsUsesDRSRegistrationEndpoint(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/ga4gh/drs/v1/objects/register" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer token" { + t.Fatalf("expected authorization header") + } + var request struct { + Candidates []struct { + Name string `json:"name"` + Size int64 `json:"size"` + ControlledAccess []string `json:"controlled_access"` + Checksums []struct { + Type string `json:"type"` + Checksum string `json:"checksum"` + } `json:"checksums"` + AccessMethods []struct { + AccessURL struct { + URL string `json:"url"` + } `json:"access_url"` + } `json:"access_methods"` + } `json:"candidates"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode request: %v", err) + } + if len(request.Candidates) != 1 || request.Candidates[0].Name != "a.bin" || request.Candidates[0].Size != 100 || len(request.Candidates[0].Checksums) != 1 || request.Candidates[0].Checksums[0].Type != "sha256" || request.Candidates[0].Checksums[0].Checksum != strings.Repeat("a", 64) || len(request.Candidates[0].AccessMethods) != 1 || request.Candidates[0].AccessMethods[0].AccessURL.URL != "s3://bucket/root/a.bin" { + t.Fatalf("unexpected registration request: %+v", request) + } + return &http.Response{ + StatusCode: http.StatusCreated, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"objects":[{"id":"obj-created"}]}`)), + }, nil + })} + manager := NewManager("http://syfon.example", client) + results, err := manager.RegisterProjectObjects(context.Background(), "Bearer token", []ProjectObjectRegistration{{ + Name: "a.bin", + Checksum: strings.Repeat("a", 64), + Size: 100, + ControlledAccess: []string{"/organization/org/project/proj"}, + AccessURLs: []string{"s3://bucket/root/a.bin"}, + }}) + if err != nil { + t.Fatalf("RegisterProjectObjects returned error: %v", err) + } + if !reflect.DeepEqual(results, []ProjectObjectRegistrationResult{{ObjectID: "obj-created"}}) { + t.Fatalf("unexpected registration result: %+v", results) + } +} + func TestBulkProbeStorageObjectsBatchesRequests(t *testing.T) { t.Helper() diff --git a/internal/server/http/git/project.go b/internal/server/http/git/project.go index ee93c3e..1511e08 100644 --- a/internal/server/http/git/project.go +++ b/internal/server/http/git/project.go @@ -105,14 +105,14 @@ func (handler *Handler) loadGitProjectState(projectID string, identity git.GitRe return state, nil } -func (handler *Handler) ensureMirrorReadyForRead(ctx context.Context, authorizationHeader string, projectID string, identity git.GitRepositoryIdentity, state *geckodb.GitProjectState) (*geckodb.GitProjectState, error) { +func (handler *Handler) ensureMirrorReadyForRead(ctx context.Context, authorizationHeader string, projectID string, identity git.GitRepositoryIdentity, state *geckodb.GitProjectState, requireCurrentMirror bool) (*geckodb.GitProjectState, error) { if state == nil || !state.InstallationID.Valid { return state, nil } if strings.TrimSpace(state.MirrorPath) == "" { return state, nil } - if _, err := os.Stat(state.MirrorPath); err == nil { + if !shouldRefreshMirrorForRead(state.MirrorPath, requireCurrentMirror) { return state, nil } org, project, _ := strings.Cut(projectID, "/") @@ -136,6 +136,15 @@ func (handler *Handler) ensureMirrorReadyForRead(ctx context.Context, authorizat return updatedState, nil } +func mirrorExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func shouldRefreshMirrorForRead(mirrorPath string, requireCurrentMirror bool) bool { + return requireCurrentMirror || !mirrorExists(mirrorPath) +} + func (handler *Handler) handleGitProjectsGET(ctx fiber.Ctx) error { states, err := geckodb.ListGitProjectStates(handler.db) if err != nil { @@ -219,7 +228,7 @@ func (handler *Handler) handleGitProjectGET(ctx fiber.Ctx) error { if authorizationHeader != "" { refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, false) if err != nil { handler.logger.Warning("failed to warm git mirror for %s: %v", projectID, err) } diff --git a/internal/server/http/git/project_test.go b/internal/server/http/git/project_test.go new file mode 100644 index 0000000..9f9488a --- /dev/null +++ b/internal/server/http/git/project_test.go @@ -0,0 +1,43 @@ +package git + +import ( + "path/filepath" + "testing" +) + +func TestShouldRefreshMirrorForRead(t *testing.T) { + existingMirrorPath := t.TempDir() + missingMirrorPath := filepath.Join(t.TempDir(), "missing.git") + + tests := []struct { + name string + mirrorPath string + requireCurrentMirror bool + want bool + }{ + { + name: "existing mirror can serve regular reads", + mirrorPath: existingMirrorPath, + want: false, + }, + { + name: "storage audit refreshes existing mirror", + mirrorPath: existingMirrorPath, + requireCurrentMirror: true, + want: true, + }, + { + name: "missing mirror refreshes regular reads", + mirrorPath: missingMirrorPath, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := shouldRefreshMirrorForRead(test.mirrorPath, test.requireCurrentMirror); got != test.want { + t.Fatalf("expected refresh=%t, got %t", test.want, got) + } + }) + } +} diff --git a/internal/server/http/git/register.go b/internal/server/http/git/register.go index 2562dc7..d9a923c 100644 --- a/internal/server/http/git/register.go +++ b/internal/server/http/git/register.go @@ -52,6 +52,7 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler projectGitWrite.Post("/update", projectWriteAuth, handler.handleGitProjectUpdatePOST) projectGitWrite.Post("/repair/project-diff/audit", projectReadAuth, handler.handleGitProjectDiffAuditPOST) projectGitWrite.Post("/repair/storage-chain/audit", projectReadAuth, handler.handleGitProjectStorageChainAuditPOST) + projectGitWrite.Post("/repair/storage-chain/register-git-only", projectWriteAuth, handler.handleGitProjectStorageChainRegisterGitOnlyPOST) projectGitWrite.Post("/repair/storage-cleanup/audit", projectReadAuth, handler.handleGitProjectStorageCleanupAuditPOST) projectGitWrite.Post("/repair/storage-cleanup/apply", projectReadAuth, handler.handleGitProjectStorageCleanupApplyPOST) projectGitWrite.Post("/uploads/session", projectWriteAuth, handler.handleGitProjectUploadSessionPOST) diff --git a/internal/server/http/git/repository.go b/internal/server/http/git/repository.go index 370f82d..d7d7b44 100644 --- a/internal/server/http/git/repository.go +++ b/internal/server/http/git/repository.go @@ -36,7 +36,7 @@ func (handler *Handler) handleGitProjectRefsGET(ctx fiber.Ctx) error { if authorizationHeader != "" { refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, false) if err != nil { handler.logger.Warning("failed to warm git mirror for %s refs: %v", projectID, err) } @@ -76,7 +76,7 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { if authorizationHeader != "" { refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, false) if err != nil { handler.logger.Warning("failed to warm git mirror for %s tree: %v", projectID, err) } @@ -153,7 +153,7 @@ func (handler *Handler) handleGitProjectManifestGET(ctx fiber.Ctx) error { if authorizationHeader != "" { refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, false) if err != nil { handler.logger.Warning("failed to warm git mirror for %s manifest: %v", projectID, err) } diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go index 815d667..c98aea6 100644 --- a/internal/server/http/git/storage_analytics.go +++ b/internal/server/http/git/storage_analytics.go @@ -283,7 +283,7 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err findingKind := strings.TrimSpace(requestBody.FindingKind) timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_prefix=%q finding_kind=%q finding_limit=%d", projectCtx.projectID, projectCtx.refName, gitSubpath, validationMode, probeMode, bucketMode, bucketPathPrefix, findingKind, findingLimit) handler.logger.Info("storage_chain_audit_request_start %s", timings.DebugPrefix) - forceAuditRefresh := requestBody.ForceAuditRefresh || requestBody.ForceBucketInventoryRefresh + forceAuditRefresh := requestBody.Refresh || requestBody.ForceAuditRefresh || requestBody.ForceBucketInventoryRefresh response, err := handler.storageAnalytics.BuildStorageChainAuditWithOptions( ctx.Context(), projectCtx.authorizationHeader, @@ -325,6 +325,36 @@ func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) err return writeErr } +func (handler *Handler) handleGitProjectStorageChainRegisterGitOnlyPOST(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + requestBody := gitcore.GitOnlySyfonRegistrationRequest{} + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + projectCtx.applyRequestRef(requestBody.Ref) + requestBody.Ref = projectCtx.refName + requestBody.RepoPaths = normalizeAnalyticsPathList(requestBody.RepoPaths) + response, err := handler.storageAnalytics.RegisterGitOnlySyfonRecords( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + requestBody, + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, "", err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + func formatStorageChainTimingSnapshot(timings *gitcore.StorageChainAuditTimings) string { parts := make([]string, 0) for _, stage := range timings.Snapshot() { @@ -452,10 +482,14 @@ func (handler *Handler) parseCleanupAnalyticsRequest(ctx fiber.Ctx) (*gitAnalyti } func (handler *Handler) resolveGitAnalyticsContext(ctx fiber.Ctx) (*gitAnalyticsContext, *httputil.ErrorResponse) { - return handler.resolveGitAnalyticsContextWithTimings(ctx, nil) + return handler.resolveGitAnalyticsContextWithMirrorPolicy(ctx, nil, false) } func (handler *Handler) resolveGitAnalyticsContextWithTimings(ctx fiber.Ctx, timings *gitcore.StorageChainAuditTimings) (*gitAnalyticsContext, *httputil.ErrorResponse) { + return handler.resolveGitAnalyticsContextWithMirrorPolicy(ctx, timings, true) +} + +func (handler *Handler) resolveGitAnalyticsContextWithMirrorPolicy(ctx fiber.Ctx, timings *gitcore.StorageChainAuditTimings, requireCurrentMirror bool) (*gitAnalyticsContext, *httputil.ErrorResponse) { if handler.storageAnalytics == nil { response := httputil.NewError("internal_error", "storage analytics service is not configured", http.StatusInternalServerError, nil, nil) response.WriteLog(handler.logger) @@ -482,9 +516,14 @@ func (handler *Handler) resolveGitAnalyticsContextWithTimings(ctx fiber.Ctx, tim timings.StageStart("mirror_warmup") refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, requireCurrentMirror) timings.Record("mirror_warmup", time.Since(start)) if err != nil { + if requireCurrentMirror { + response := httputil.NewError("integration_error", fmt.Sprintf("storage audit requires a current Git mirror: %s", err), http.StatusBadGateway, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } handler.logger.Warning("failed to warm git mirror for %s analytics: %v", projectID, err) } } @@ -572,7 +611,7 @@ func (handler *Handler) writeGitAnalyticsError(ctx fiber.Ctx, projectID string, if strings.Contains(errorMessage, "storage children cursor") { statusCode = http.StatusBadRequest errorType = "invalid_request" - } else if strings.Contains(errorMessage, "cleanup apply") || strings.Contains(errorMessage, "cleanup finding") || strings.Contains(errorMessage, "cleanup action") || strings.Contains(errorMessage, "unsupported cleanup") || strings.Contains(errorMessage, "selected cleanup paths") { + } else if strings.Contains(errorMessage, "cleanup apply") || strings.Contains(errorMessage, "cleanup finding") || strings.Contains(errorMessage, "cleanup action") || strings.Contains(errorMessage, "unsupported cleanup") || strings.Contains(errorMessage, "selected cleanup paths") || strings.Contains(errorMessage, "git-only syfon registration") { statusCode = http.StatusBadRequest errorType = "invalid_request" } else if strings.Contains(errorMessage, "git tree path") { @@ -604,7 +643,7 @@ func (handler *Handler) logStorageChainAuditTimings(projectCtx *gitAnalyticsCont bucketPathExists = strconv.FormatBool(*response.Summary.BucketPathExists) } handler.logger.Info( - "storage_chain_audit project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_exists=%s bucket_summary_mode=%s bucket_inventory_available=%t audit_cache_hit=%t audit_cache_source=%s audit_cache_age_seconds=%d findings=%d returned_findings=%d findings_truncated=%t finding_limit=%d bucket_objects=%d syfon_records=%d git_files=%d %s", + "storage_chain_audit project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_exists=%s bucket_summary_mode=%s bucket_inventory_available=%t audit_cache_hit=%t audit_cache_source=%s audit_cache_age_seconds=%d audit_refresh_duration_ms=%d findings=%d returned_findings=%d findings_truncated=%t finding_limit=%d bucket_objects=%d syfon_records=%d git_files=%d %s", projectCtx.projectID, projectCtx.refName, gitSubpath, @@ -617,6 +656,7 @@ func (handler *Handler) logStorageChainAuditTimings(projectCtx *gitAnalyticsCont response.Summary.AuditCacheHit, response.Summary.AuditCacheSource, response.Summary.AuditCacheAgeSeconds, + response.Summary.AuditRefreshDurationMs, response.Summary.TotalFindings, response.Summary.ReturnedFindings, response.Summary.FindingsTruncated, diff --git a/internal/storageaudit/evidence.go b/internal/storageaudit/evidence.go new file mode 100644 index 0000000..e1b40d9 --- /dev/null +++ b/internal/storageaudit/evidence.go @@ -0,0 +1,80 @@ +package storageaudit + +import "strings" + +type EvidenceStatus string + +const ( + EvidenceVerified EvidenceStatus = "verified" + EvidenceUnknown EvidenceStatus = "unknown" +) + +const ( + OperationInventory = "inventory" + OperationList = "list" + OperationMetadata = "metadata" +) + +type Probe struct { + Operation string + Status string + ErrorKind string + ValidationStatus string +} + +type Assessment struct { + Status EvidenceStatus + Present bool + Missing bool + MappingBroken bool + MetadataMismatch bool + HasInventoryMiss bool + HasExactEvidence bool +} + +func Assess(probes []Probe, bucketObserved bool) Assessment { + assessment := Assessment{Status: EvidenceUnknown} + if bucketObserved { + assessment.Present = true + assessment.Status = EvidenceVerified + } + for _, probe := range probes { + operation := strings.TrimSpace(probe.Operation) + status := strings.TrimSpace(probe.Status) + errorKind := strings.TrimSpace(probe.ErrorKind) + validation := strings.TrimSpace(probe.ValidationStatus) + + if operation == OperationInventory && (status == "unknown" || errorKind == "inventory_miss") { + assessment.HasInventoryMiss = true + continue + } + isExact := operation == OperationList || operation == OperationMetadata + if isExact { + assessment.HasExactEvidence = true + } + if status == "present" { + assessment.Present = true + assessment.Status = EvidenceVerified + } + // Exact LIST is useful for discovering a candidate, but some S3-compatible + // backends can return an empty page under load without returning an error. + // Only a metadata lookup (HEAD) is authoritative enough to prove absence. + if operation == OperationMetadata && status == "not_found" && (errorKind == "" || errorKind == "object_not_found") { + assessment.Missing = true + assessment.Status = EvidenceVerified + } + switch errorKind { + case "missing_access_url", "scope_not_found", "credential_missing": + assessment.MappingBroken = true + assessment.Status = EvidenceVerified + } + if validation == "mismatched" { + assessment.MetadataMismatch = true + assessment.Status = EvidenceVerified + } + } + if assessment.Present { + assessment.Missing = false + } + return assessment +} diff --git a/internal/storageaudit/evidence_test.go b/internal/storageaudit/evidence_test.go new file mode 100644 index 0000000..8c139f9 --- /dev/null +++ b/internal/storageaudit/evidence_test.go @@ -0,0 +1,61 @@ +package storageaudit + +import "testing" + +func TestAssessTreatsInventoryMissAsUnknown(t *testing.T) { + assessment := Assess([]Probe{{ + Operation: OperationInventory, + Status: "unknown", + ErrorKind: "inventory_miss", + ValidationStatus: "unverifiable", + }}, false) + if assessment.Status != EvidenceUnknown || assessment.Missing || !assessment.HasInventoryMiss { + t.Fatalf("expected an unverified inventory candidate, got %+v", assessment) + } +} + +func TestAssessTreatsExactListMissAsUnconfirmed(t *testing.T) { + assessment := Assess([]Probe{{ + Operation: OperationList, + Status: "not_found", + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }}, false) + if assessment.Status != EvidenceUnknown || assessment.Missing || !assessment.HasExactEvidence { + t.Fatalf("expected exact LIST absence to require metadata confirmation, got %+v", assessment) + } +} + +func TestAssessTreatsMetadataMissAsVerified(t *testing.T) { + assessment := Assess([]Probe{{ + Operation: OperationMetadata, + Status: "not_found", + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }}, false) + if assessment.Status != EvidenceVerified || !assessment.Missing || !assessment.HasExactEvidence { + t.Fatalf("expected metadata absence to be verified, got %+v", assessment) + } +} + +func TestAssessLetsPresentLocatorWinOverMissingAlias(t *testing.T) { + assessment := Assess([]Probe{ + {Operation: OperationList, Status: "not_found", ErrorKind: "object_not_found"}, + {Operation: OperationList, Status: "present", ValidationStatus: "matched"}, + }, false) + if assessment.Status != EvidenceVerified || !assessment.Present || assessment.Missing { + t.Fatalf("expected any present canonical locator to connect the record, got %+v", assessment) + } +} + +func TestAutomaticRepairRequiresVerifiedMissingEvidence(t *testing.T) { + if AllowsAutomaticRepair("syfon_git_no_bucket", EvidenceUnknown) { + t.Fatal("unknown evidence must not allow automatic repair") + } + if !AllowsAutomaticRepair("syfon_git_no_bucket", EvidenceVerified) { + t.Fatal("verified missing evidence should allow the missing-record repair") + } + if AllowsAutomaticRepair("bucket_syfon_no_git", EvidenceVerified) { + t.Fatal("absence from Git is informational and must not auto-delete storage") + } +} diff --git a/internal/storageaudit/policy.go b/internal/storageaudit/policy.go new file mode 100644 index 0000000..a2da529 --- /dev/null +++ b/internal/storageaudit/policy.go @@ -0,0 +1,15 @@ +package storageaudit + +import "strings" + +func AllowsAutomaticRepair(kind string, evidenceStatus EvidenceStatus) bool { + if evidenceStatus != EvidenceVerified { + return false + } + switch strings.TrimSpace(kind) { + case "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object": + return true + default: + return false + } +} diff --git a/internal/storageaudit/types.go b/internal/storageaudit/types.go new file mode 100644 index 0000000..f86c81f --- /dev/null +++ b/internal/storageaudit/types.go @@ -0,0 +1,301 @@ +package storageaudit + +type Evidence struct { + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` + Buckets []string `json:"buckets,omitempty"` + Keys []string `json:"keys,omitempty"` + StorageOperations []string `json:"storage_operations,omitempty"` + ProbeStatuses []string `json:"probe_statuses,omitempty"` + ValidationStates []string `json:"validation_states,omitempty"` + ErrorKinds []string `json:"error_kinds,omitempty"` + Errors []string `json:"errors,omitempty"` + BucketEvaluation string `json:"bucket_evaluation,omitempty"` +} + +type ProjectDiffAuditRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` +} + +type ProjectDiffFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids"` + RecordCount int `json:"record_count"` + SizeBytes int64 `json:"size_bytes,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` + RecommendedAction string `json:"recommended_action"` + Evidence *Evidence `json:"evidence,omitempty"` +} + +type ProjectDiffSummary struct { + CountsByKind map[string]int `json:"counts_by_kind"` + TotalFindings int `json:"total_findings"` + IndexedPathCount int `json:"indexed_path_count"` + ExpectedPathCount int `json:"expected_path_count"` + MatchedPathCount int `json:"matched_path_count"` + IncludesRepoManifest bool `json:"includes_repo_manifest"` + ScannedRecordCount int `json:"scanned_record_count"` +} + +type ProjectDiffAuditResponse struct { + Findings []ProjectDiffFinding `json:"findings"` + Summary ProjectDiffSummary `json:"summary"` + PathPrefix string `json:"path_prefix"` +} + +type CleanupAuditRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + CheckStorage bool `json:"check_storage,omitempty"` + SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` +} + +type ChainAuditRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + Refresh bool `json:"refresh,omitempty"` + ProbeMode string `json:"probe_mode,omitempty"` + ValidationMode string `json:"validation_mode,omitempty"` + BucketInventoryMode string `json:"bucket_inventory_mode,omitempty"` + BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` + FindingKind string `json:"finding_kind,omitempty"` + FindingLimit int `json:"finding_limit,omitempty"` + ForceAuditRefresh bool `json:"force_audit_refresh,omitempty"` + ForceBucketInventoryRefresh bool `json:"force_bucket_inventory_refresh,omitempty"` +} + +type ChainFinding struct { + Kind string `json:"kind"` + EvidenceStatus string `json:"evidence_status"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids"` + Records []CleanupRecordAudit `json:"records,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + BucketObjectURL string `json:"bucket_object_url,omitempty"` + ResolvedBucket string `json:"resolved_bucket,omitempty"` + ResolvedKey string `json:"resolved_key,omitempty"` + ProbeStatus string `json:"probe_status,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + Error string `json:"error,omitempty"` + RecordCount int `json:"record_count"` + SizeBytes int64 `json:"size_bytes,omitempty"` + BucketSizeBytes int64 `json:"bucket_size_bytes,omitempty"` + RecommendedAction string `json:"recommended_action"` + SuggestedFix string `json:"suggested_fix,omitempty"` + SuggestedAction string `json:"suggested_action,omitempty"` + Actionability string `json:"actionability,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + SupportsDryRun bool `json:"supports_dry_run,omitempty"` + Evidence *Evidence `json:"evidence,omitempty"` +} + +// GitOnlySyfonRegistrationRequest names Git LFS paths that the caller wants +// registered in Syfon after Gecko has revalidated their scoped bucket objects. +type GitOnlySyfonRegistrationRequest struct { + Ref string `json:"ref,omitempty"` + ExpectedGitRevision string `json:"expected_git_revision"` + RepoPaths []string `json:"repo_paths"` +} + +type GitOnlySyfonRegistrationResult struct { + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + GitSizeBytes int64 `json:"git_size_bytes,omitempty"` + BucketObjectURL string `json:"bucket_object_url,omitempty"` + BucketSizeBytes int64 `json:"bucket_size_bytes,omitempty"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + ObjectID string `json:"object_id,omitempty"` +} + +type GitOnlySyfonRegistrationResponse struct { + GitRevision string `json:"git_revision"` + Results []GitOnlySyfonRegistrationResult `json:"results"` +} + +type ChainSummary struct { + CountsByKind map[string]int `json:"counts_by_kind"` + TotalFindings int `json:"total_findings"` + ReturnedFindings int `json:"returned_findings"` + FindingsTruncated bool `json:"findings_truncated"` + FindingLimit int `json:"finding_limit,omitempty"` + ValidationMode string `json:"validation_mode,omitempty"` + GitRevision string `json:"git_revision,omitempty"` + SyfonRevision string `json:"syfon_revision,omitempty"` + ObservedAt string `json:"observed_at,omitempty"` + BucketObjectCount int `json:"bucket_object_count"` + SyfonRecordCount int `json:"syfon_record_count"` + GitTrackedFileCount int `json:"git_tracked_file_count"` + BucketPathExists *bool `json:"bucket_path_exists,omitempty"` + BucketPathObjectURL string `json:"bucket_path_object_url,omitempty"` + BucketSummaryMode string `json:"bucket_summary_mode,omitempty"` + BucketInventoryAvailable bool `json:"bucket_inventory_available"` + BucketInventoryError string `json:"bucket_inventory_error,omitempty"` + AuditCacheHit bool `json:"audit_cache_hit,omitempty"` + AuditCachedAt string `json:"audit_cached_at,omitempty"` + AuditCacheAgeSeconds int64 `json:"audit_cache_age_seconds,omitempty"` + AuditRefreshDurationMs int64 `json:"audit_refresh_duration_ms,omitempty"` + AuditCacheSource string `json:"audit_cache_source,omitempty"` + AuditCacheError string `json:"audit_cache_error,omitempty"` +} + +type ChainIssueGroup struct { + Kind string `json:"kind"` + FindingCount int `json:"finding_count"` + PathCount int `json:"path_count"` + RecordCount int `json:"record_count"` + ObjectCount int `json:"object_count"` + TotalBytes int64 `json:"total_bytes,omitempty"` +} + +type ChainAuditResponse struct { + Findings []ChainFinding `json:"findings"` + Groups []ChainIssueGroup `json:"groups,omitempty"` + Summary ChainSummary `json:"summary"` + PathPrefix string `json:"path_prefix"` + BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` +} + +type CleanupAccessProbe struct { + URL string `json:"url"` + Operation string `json:"operation,omitempty"` + Provider string `json:"provider,omitempty"` + Bucket string `json:"bucket,omitempty"` + Key string `json:"key,omitempty"` + Path string `json:"path,omitempty"` + Exists *bool `json:"exists,omitempty"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + MetaSHA256 string `json:"meta_sha256,omitempty"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` + ValidationStatus string `json:"validation_status,omitempty"` + SizeMatch *bool `json:"size_match,omitempty"` + NameMatch *bool `json:"name_match,omitempty"` + SHA256Match *bool `json:"sha256_match,omitempty"` + ValidationMismatches []string `json:"validation_mismatches,omitempty"` +} + +type CleanupAccessMethod struct { + AccessID string `json:"access_id,omitempty"` + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Headers []string `json:"headers,omitempty"` +} + +type CleanupRecordAudit struct { + ObjectID string `json:"object_id"` + Checksum string `json:"checksum,omitempty"` + NormalizedPath string `json:"normalized_path,omitempty"` + CleanupScope string `json:"cleanup_scope"` + AccessURLs []string `json:"access_urls,omitempty"` + AccessMethods []CleanupAccessMethod `json:"access_methods,omitempty"` + AccessProbes []CleanupAccessProbe `json:"access_probes"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` + SizeBytes int64 `json:"size,omitempty"` + LastUpdated string `json:"updated_time,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` +} + +type CleanupFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + ObjectIDs []string `json:"object_ids"` + Records []CleanupRecordAudit `json:"records"` + RecommendedAction string `json:"recommended_action"` + RepoDeleteCandidate bool `json:"repo_delete_candidate"` + CleanupScope string `json:"cleanup_scope"` + SizeBytes int64 `json:"total_bytes,omitempty"` + LastUpdated string `json:"last_updated,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` + Actionability string `json:"actionability,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + SupportsDryRun bool `json:"supports_dry_run,omitempty"` + Evidence *Evidence `json:"evidence,omitempty"` +} + +type CleanupApplyAction struct { + Action string `json:"action"` + Kind string `json:"kind,omitempty"` + NormalizedPath string `json:"normalized_path,omitempty"` +} + +type CleanupApplyFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + ObjectIDs []string `json:"object_ids,omitempty"` + Records []CleanupRecordAudit `json:"records,omitempty"` + BucketObjectURL string `json:"bucket_object_url,omitempty"` + BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + SuggestedAction string `json:"suggested_action,omitempty"` + Evidence *Evidence `json:"evidence,omitempty"` +} + +type CleanupAuditSummary struct { + CountsByKind map[string]int `json:"counts_by_kind"` + TotalFindings int `json:"total_findings"` + ManualFindingCount int `json:"manual_finding_count"` + RepoDeleteCandidateCount int `json:"repo_delete_candidate_count"` + StaleDuplicateCount int `json:"stale_duplicate_count"` + RepoOrphanCount int `json:"repo_orphan_count"` +} + +type CleanupAuditResponse struct { + Findings []CleanupFinding `json:"findings"` + Summary CleanupAuditSummary `json:"summary"` + ExpectedPathCount int `json:"expected_path_count"` + IncludesRepoManifest bool `json:"includes_repo_manifest"` + PathPrefix string `json:"path_prefix"` +} + +type CleanupApplyRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + DeleteRepoOrphans bool `json:"delete_repo_orphans,omitempty"` + DeleteStaleDuplicates bool `json:"delete_stale_duplicates,omitempty"` + DeleteBucketOnlyObjects bool `json:"delete_bucket_only_objects,omitempty"` + RepairBrokenBucketMappings bool `json:"repair_broken_bucket_mappings,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` + Actions []CleanupApplyAction `json:"actions,omitempty"` + Findings []CleanupApplyFinding `json:"findings,omitempty"` +} + +type CleanupPurgeResult struct { + ObjectID string `json:"object_id"` + Success *bool `json:"success"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` +} + +type CleanupApplyResponse struct { + DeletedRecordIDs []string `json:"deleted_record_ids"` + DeletedBucketObjectURLs []string `json:"deleted_bucket_object_urls"` + UpdatedRecordIDs []string `json:"updated_record_ids"` + PurgeResults []CleanupPurgeResult `json:"purge_results"` + RepoDeletePaths []string `json:"repo_delete_paths"` + ManualPaths []string `json:"manual_paths"` + SkippedPaths []string `json:"skipped_paths"` + DryRun bool `json:"dry_run"` +} From 3351cafa64e96ec6fecc7e30afdad96af429ef67 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 9 Jul 2026 17:51:01 -0700 Subject: [PATCH 32/36] remove deadcode, fix false positive issue --- docs/docs.go | 4 - internal/db/config.go | 4 - internal/db/git_pending.go | 246 ------------------ internal/db/git_project.go | 4 - internal/git/response.go | 42 --- internal/git/service.go | 7 - internal/git/service_test.go | 22 +- internal/git/storage_analytics.go | 36 --- internal/git/storage_analytics_pipeline.go | 132 +--------- internal/git/storage_analytics_test.go | 26 +- .../git/storage_audit_registration_test.go | 2 +- internal/git/storage_index.go | 10 - internal/git/tree_response_test.go | 10 - internal/integrations/fence/client.go | 20 -- internal/integrations/fence/client_test.go | 43 --- internal/server/http/config/config.go | 13 - internal/server/http/git/helpers.go | 8 - internal/server/http/shared/helpers.go | 66 +---- internal/server/middleware/access.go | 16 -- internal/server/middleware/auth.go | 57 ---- internal/server/middleware/fence_user_test.go | 16 -- internal/server/middleware/git_test.go | 13 - internal/server/middleware/middleware.go | 22 -- internal/storageaudit/evidence.go | 9 +- internal/storageaudit/evidence_test.go | 18 +- tests/integration/middleware_test.go | 31 --- 26 files changed, 39 insertions(+), 838 deletions(-) delete mode 100644 internal/db/git_pending.go diff --git a/docs/docs.go b/docs/docs.go index 6a93eab..7dbcb41 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1893,7 +1893,3 @@ var SwaggerInfo = &swag.Spec{ LeftDelim: "{{", RightDelim: "}}", } - -func init() { - swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) -} diff --git a/internal/db/config.go b/internal/db/config.go index 36194cd..b9d0144 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -95,10 +95,6 @@ func ConfigPUTGeneric(db *sqlx.DB, configId string, configType string, data any) return ConfigPUTGenericContext(context.Background(), db, configId, configType, data) } -func ConfigPUTGenericTx(tx *sqlx.Tx, configId string, configType string, data any) error { - return ConfigPUTGenericTxContext(context.Background(), tx, configId, configType, data) -} - func ConfigPUTGenericContext(ctx context.Context, db *sqlx.DB, configId string, configType string, data any) error { if db == nil { return nil diff --git a/internal/db/git_pending.go b/internal/db/git_pending.go deleted file mode 100644 index e6e69a7..0000000 --- a/internal/db/git_pending.go +++ /dev/null @@ -1,246 +0,0 @@ -package db - -import ( - "database/sql" - "encoding/json" - "errors" - "fmt" - - "github.com/jmoiron/sqlx" -) - -func UpsertGitPendingRepository(db *sqlx.DB, pending GitPendingRepository) error { - if db == nil { - return nil - } - _, err := db.NamedExec(` - INSERT INTO config_schema.git_pending_repository ( - id, installation_id, setup_session_id, created_by_user_id, source, organization, repo_id, repo_name, repo_full_name, repo_html_url, repo_clone_url, repo_host, repo_owner, repo_path, added_at, updated_at, resolved_at, removed_at - ) VALUES ( - :id, :installation_id, :setup_session_id, :created_by_user_id, :source, :organization, :repo_id, :repo_name, :repo_full_name, :repo_html_url, :repo_clone_url, :repo_host, :repo_owner, :repo_path, :added_at, :updated_at, :resolved_at, :removed_at - ) - ON CONFLICT (id) DO UPDATE SET - id = EXCLUDED.id, - setup_session_id = EXCLUDED.setup_session_id, - created_by_user_id = EXCLUDED.created_by_user_id, - source = EXCLUDED.source, - organization = EXCLUDED.organization, - repo_name = EXCLUDED.repo_name, - repo_full_name = EXCLUDED.repo_full_name, - repo_html_url = EXCLUDED.repo_html_url, - repo_clone_url = EXCLUDED.repo_clone_url, - repo_host = EXCLUDED.repo_host, - repo_owner = EXCLUDED.repo_owner, - repo_path = EXCLUDED.repo_path, - added_at = EXCLUDED.added_at, - updated_at = EXCLUDED.updated_at, - resolved_at = EXCLUDED.resolved_at, - removed_at = EXCLUDED.removed_at - `, pending) - if err != nil { - return fmt.Errorf("upsert git pending repository: %w", err) - } - return nil -} - -func gitPendingRepositorySelectSQL() string { - return `SELECT id, installation_id, setup_session_id, created_by_user_id, source, organization, repo_id, repo_name, repo_full_name, repo_html_url, repo_clone_url, repo_host, repo_owner, repo_path, added_at, updated_at, resolved_at, removed_at FROM config_schema.git_pending_repository` -} - -func ListGitPendingRepositoriesByInstallation(db *sqlx.DB, installationID int64) ([]GitPendingRepository, error) { - if db == nil { - return []GitPendingRepository{}, nil - } - records := []GitPendingRepository{} - if err := db.Select(&records, gitPendingRepositorySelectSQL()+` - WHERE installation_id = $1 AND resolved_at IS NULL AND removed_at IS NULL - ORDER BY added_at, repo_full_name - `, installationID); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return []GitPendingRepository{}, nil - } - return nil, err - } - return records, nil -} - -func ListGitPendingRepositoriesByUser(db *sqlx.DB, userID string, installationID int64, setupSessionID string) ([]GitPendingRepository, error) { - if db == nil { - return []GitPendingRepository{}, nil - } - records := []GitPendingRepository{} - query := gitPendingRepositorySelectSQL() + ` - WHERE created_by_user_id = $1 - AND resolved_at IS NULL - AND removed_at IS NULL` - args := []any{userID} - if installationID > 0 { - args = append(args, installationID) - query += fmt.Sprintf(" AND installation_id = $%d", len(args)) - } - if setupSessionID != "" { - args = append(args, setupSessionID) - query += fmt.Sprintf(" AND setup_session_id = $%d", len(args)) - } - query += " ORDER BY added_at, repo_full_name" - if err := db.Select(&records, query, args...); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return []GitPendingRepository{}, nil - } - return nil, err - } - return records, nil -} - -func ListGitPendingRepositories(db *sqlx.DB) ([]GitPendingRepository, error) { - if db == nil { - return []GitPendingRepository{}, nil - } - records := []GitPendingRepository{} - if err := db.Select(&records, gitPendingRepositorySelectSQL()+` - WHERE resolved_at IS NULL AND removed_at IS NULL - ORDER BY added_at, repo_full_name - `); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return []GitPendingRepository{}, nil - } - return nil, err - } - return records, nil -} - -func GitPendingRepositoryByID(db *sqlx.DB, id string) (*GitPendingRepository, error) { - if db == nil || id == "" { - return nil, nil - } - var pending GitPendingRepository - if err := db.Get(&pending, gitPendingRepositorySelectSQL()+` WHERE id = $1`, id); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - return nil, fmt.Errorf("get git pending repository by id: %w", err) - } - return &pending, nil -} - -func ResolveGitPendingRepositoryByID(db *sqlx.DB, id string) error { - if db == nil || id == "" { - return nil - } - _, err := db.Exec(`UPDATE config_schema.git_pending_repository SET resolved_at = NOW(), updated_at = NOW() WHERE id = $1`, id) - if err != nil { - return fmt.Errorf("resolve git pending repository by id: %w", err) - } - return nil -} - -func ResolveGitPendingRepositoriesByRepo(db *sqlx.DB, installationID int64, repoHost string, repoOwner string, repoPath string) error { - if db == nil { - return nil - } - _, err := db.Exec(` - UPDATE config_schema.git_pending_repository - SET resolved_at = NOW(), updated_at = NOW() - WHERE installation_id = $1 - AND repo_host = $2 - AND repo_owner = $3 - AND repo_path = $4 - AND resolved_at IS NULL - AND removed_at IS NULL - `, installationID, repoHost, repoOwner, repoPath) - if err != nil { - return fmt.Errorf("resolve git pending repositories by repo: %w", err) - } - return nil -} - -func ResolveGitPendingRepositoriesByRepositoryIdentity(db *sqlx.DB, repoHost string, repoOwner string, repoPath string) error { - if db == nil { - return nil - } - _, err := db.Exec(` - UPDATE config_schema.git_pending_repository - SET resolved_at = NOW(), updated_at = NOW() - WHERE repo_host = $1 - AND repo_owner = $2 - AND repo_path = $3 - AND resolved_at IS NULL - AND removed_at IS NULL - `, repoHost, repoOwner, repoPath) - if err != nil { - return fmt.Errorf("resolve git pending repositories by repository identity: %w", err) - } - return nil -} - -func RemoveGitPendingRepository(db *sqlx.DB, installationID int64, repoID int64) error { - if db == nil { - return nil - } - _, err := db.Exec(` - UPDATE config_schema.git_pending_repository - SET removed_at = NOW(), updated_at = NOW() - WHERE installation_id = $1 AND repo_id = $2 AND removed_at IS NULL - `, installationID, repoID) - if err != nil { - return fmt.Errorf("remove git pending repository: %w", err) - } - return nil -} - -func UpsertGitSetupSession(db *sqlx.DB, session GitSetupSession) error { - if db == nil { - return nil - } - _, err := db.NamedExec(` - INSERT INTO config_schema.git_setup_session ( - id, created_by_user_id, organization, installation_id, before_repo_ids, created_at, updated_at, completed_at - ) VALUES ( - :id, :created_by_user_id, :organization, :installation_id, :before_repo_ids, :created_at, :updated_at, :completed_at - ) - ON CONFLICT (id) DO UPDATE SET - created_by_user_id = EXCLUDED.created_by_user_id, - organization = EXCLUDED.organization, - installation_id = EXCLUDED.installation_id, - before_repo_ids = EXCLUDED.before_repo_ids, - updated_at = EXCLUDED.updated_at, - completed_at = EXCLUDED.completed_at - `, session) - if err != nil { - return fmt.Errorf("upsert git setup session: %w", err) - } - return nil -} - -func GitSetupSessionByID(db *sqlx.DB, id string) (*GitSetupSession, error) { - if db == nil || id == "" { - return nil, nil - } - var session GitSetupSession - err := db.Get(&session, `SELECT id, created_by_user_id, organization, installation_id, before_repo_ids, created_at, updated_at, completed_at FROM config_schema.git_setup_session WHERE id = $1`, id) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - return nil, err - } - return &session, nil -} - -func EncodeRepoIDs(repoIDs []int64) string { - body, err := json.Marshal(repoIDs) - if err != nil { - return "[]" - } - return string(body) -} - -func DecodeRepoIDs(raw string) map[int64]struct{} { - repoIDs := []int64{} - _ = json.Unmarshal([]byte(raw), &repoIDs) - indexed := make(map[int64]struct{}, len(repoIDs)) - for _, repoID := range repoIDs { - indexed[repoID] = struct{}{} - } - return indexed -} diff --git a/internal/db/git_project.go b/internal/db/git_project.go index 39121ef..d8a8080 100644 --- a/internal/db/git_project.go +++ b/internal/db/git_project.go @@ -143,10 +143,6 @@ func UpsertGitProjectState(db *sqlx.DB, state GitProjectState) error { return UpsertGitProjectStateContext(context.Background(), db, state) } -func UpsertGitProjectStateTx(tx *sqlx.Tx, state GitProjectState) error { - return UpsertGitProjectStateTxContext(context.Background(), tx, state) -} - func UpsertGitProjectStateContext(ctx context.Context, db *sqlx.DB, state GitProjectState) error { if db == nil { return nil diff --git a/internal/git/response.go b/internal/git/response.go index 2df659e..25cc24b 100644 --- a/internal/git/response.go +++ b/internal/git/response.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" "log" "net/http" "path/filepath" @@ -270,47 +269,6 @@ func BuildGitRefsResponse(projectID string, defaultBranch string, repo *gogit.Re return &GitProjectRefsResponse{ProjectID: projectID, DefaultBranch: defaultBranch, Refs: refs}, nil } -func BuildGitFileResponse(projectID string, ref string, path string, repo *gogit.Repository, hash plumbing.Hash) (*GitProjectFileResponse, error) { - commit, err := repo.CommitObject(hash) - if err != nil { - return nil, fmt.Errorf("load commit for ref %s: %w", ref, err) - } - tree, err := commit.Tree() - if err != nil { - return nil, fmt.Errorf("load git tree for ref %s: %w", ref, err) - } - normalizedPath := strings.Trim(strings.TrimSpace(path), "/") - if normalizedPath == "" { - return nil, fmt.Errorf("file path is required") - } - file, err := tree.File(normalizedPath) - if err != nil { - return nil, fmt.Errorf("load git file %s: %w", normalizedPath, err) - } - reader, err := file.Reader() - if err != nil { - return nil, fmt.Errorf("open git file %s: %w", normalizedPath, err) - } - defer reader.Close() - const inlineLimit = 256 * 1024 - contentBytes, err := io.ReadAll(io.LimitReader(reader, inlineLimit+1)) - if err != nil { - return nil, fmt.Errorf("read git file content for %s: %w", normalizedPath, err) - } - if len(contentBytes) > inlineLimit { - contentBytes = contentBytes[:inlineLimit] - } - return &GitProjectFileResponse{ - ProjectID: projectID, - Ref: ref, - Path: normalizedPath, - Name: filepath.Base(normalizedPath), - Hash: file.Hash.String(), - Size: file.Size, - LFSPointer: ParseGitLFSPointer(contentBytes), - }, nil -} - func BuildGitHubFileResponse(projectID string, ref string, path string, metadata *github.RepositoryContent, contentBytes []byte) *GitProjectFileResponse { name := filepath.Base(strings.Trim(strings.TrimSpace(path), "/")) hash := "" diff --git a/internal/git/service.go b/internal/git/service.go index ed79b20..5e03fcd 100644 --- a/internal/git/service.go +++ b/internal/git/service.go @@ -224,13 +224,6 @@ func (service *GitService) ListInstallationRepositories(ctx context.Context, aut return service.fenceAPI.ListInstallationRepositories(ctx, authorizationHeader, organization, owner, installationID) } -func (service *GitService) RequestInstallationStatus(ctx context.Context, authorizationHeader string, organization string, identity GitRepositoryIdentity) (GitRepositoryInstallationStatus, error) { - if service.fenceAPI == nil { - return GitRepositoryInstallationStatus{}, fmt.Errorf("fence client is not initialized") - } - return service.fenceAPI.RequestInstallationStatus(ctx, authorizationHeader, organization, identity) -} - func (service *GitService) RequestInstallationToken(ctx context.Context, authorizationHeader string, organization string, project string, identity GitRepositoryIdentity, access string) (string, error) { if service.fenceAPI == nil { return "", fmt.Errorf("fence client is not initialized") diff --git a/internal/git/service_test.go b/internal/git/service_test.go index d59a361..d332c4b 100644 --- a/internal/git/service_test.go +++ b/internal/git/service_test.go @@ -74,16 +74,6 @@ func TestSyncRepositoryMirrorPullsUpdatesAndReadsTree(t *testing.T) { if err != nil { t.Fatalf("resolve updated HEAD: %v", err) } - fileResponse, err := BuildGitFileResponse("org-a/proj-a", refName, "README.md", mirrorRepo, hash) - if err != nil { - t.Fatalf("build updated file response: %v", err) - } - if fileResponse.Name != "README.md" { - t.Fatalf("expected README.md file name, got %q", fileResponse.Name) - } - if fileResponse.Hash == "" { - t.Fatal("expected file hash to be populated") - } } func TestBuildGitRefsResponseIncludesRemoteBranches(t *testing.T) { @@ -164,23 +154,13 @@ func TestBuildGitRefsResponseIncludesRemoteBranches(t *testing.T) { t.Fatalf("expected benchmarking branch in refs response, got %+v", refsResponse.Refs) } - refName, hash, err := ResolveGitReference(mirrorRepo, "benchmarking", defaultBranch) + refName, _, err := ResolveGitReference(mirrorRepo, "benchmarking", defaultBranch) if err != nil { t.Fatalf("resolve benchmarking branch: %v", err) } if refName != "benchmarking" { t.Fatalf("expected resolved ref name benchmarking, got %q", refName) } - fileResponse, err := BuildGitFileResponse("org-a/proj-a", refName, "benchmark.txt", mirrorRepo, hash) - if err != nil { - t.Fatalf("build branch file response: %v", err) - } - if fileResponse.Name != "benchmark.txt" { - t.Fatalf("expected benchmark.txt file name, got %q", fileResponse.Name) - } - if fileResponse.Hash == "" { - t.Fatal("expected branch file hash to be populated") - } } func TestBuildGitManifestResponseRecursesAndPaginates(t *testing.T) { diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 492b27d..b9ea2a2 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -294,14 +294,6 @@ type projectAuditRecordValidator struct { RecordRevision string } -func BuildGitRepoInventory(ref string, gitSubpath string, repo *gogit.Repository, hash plumbing.Hash) ([]RepoInventoryFile, error) { - index, err := buildRepoAnalyticsIndex(ref, repo, hash) - if err != nil { - return nil, err - } - return filterRepoInventoryFiles(index, gitSubpath) -} - func (service *StorageAnalyticsService) BuildStorageSummary(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitStorageSummaryResponse, error) { index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, false) if err != nil { @@ -472,10 +464,6 @@ func (service *StorageAnalyticsService) BuildStorageCleanupAudit(ctx context.Con }, model, nil } -func (service *StorageAnalyticsService) BuildStorageChainAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitStorageChainAuditResponse, error) { - return service.BuildStorageChainAuditWithOptions(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, StorageChainAuditOptions{}) -} - func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, error) { normalized, err := normalizeStorageChainAuditOptions(options) if err != nil { @@ -1483,20 +1471,6 @@ func cleanupActionKey(kind string, path string) string { return strings.TrimSpace(kind) + "::" + normalizeRepoSubpath(path) } -func cleanupActionForFinding(index map[string]string, finding GitStorageCleanupFinding) string { - if len(index) == 0 { - return "" - } - path := normalizeRepoSubpath(finding.NormalizedPath) - if path == "" { - return "" - } - if action := strings.TrimSpace(index[cleanupActionKey(finding.Kind, path)]); action != "" { - return action - } - return strings.TrimSpace(index[cleanupActionKey("", path)]) -} - func (service *StorageAnalyticsService) loadJoinState(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, forceRefresh bool) (*repoAnalyticsIndex, []RepoInventoryFile, map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) if err != nil { @@ -2453,11 +2427,6 @@ func storageCleanupActionSupport(kind string) (string, []string, string, bool) { return policy.actionability, append([]string(nil), policy.actions...), policy.defaultAction, policy.supportsDryRun } -func storageChainActionSupport(kind string) (string, []string, string, bool) { - policy := storageRepairPolicyForKind(kind) - return policy.actionability, append([]string(nil), policy.actions...), policy.defaultAction, policy.supportsDryRun -} - func compareRecordState(left projectRecordState, right projectRecordState) int { if left.Usage.DownloadCount != right.Usage.DownloadCount { if left.Usage.DownloadCount > right.Usage.DownloadCount { @@ -2845,11 +2814,6 @@ func suggestedFixForChainFinding(kind string, record projectRecordState) string return strings.Join(mismatches, ". ") + ". Because the byte lengths differ, the bucket object cannot have the current Git/Syfon SHA-256. Recompute the bucket object's SHA-256 before manually reconciling Git and Syfon." } -func recordHasAccessProbeMismatch(record projectRecordState, mismatch string) bool { - _, ok := firstAccessProbeMismatch(record, mismatch) - return ok -} - func firstAccessProbeMismatch(record projectRecordState, mismatch string) (GitStorageCleanupAccessProbe, bool) { for _, probe := range accessProbesForRecord(record) { if accessProbeHasMismatch(probe, mismatch) { diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index d212dbf..8bdd6f7 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -351,14 +351,6 @@ func applyScopeCanonicalization(recordSet *storageAuditRecordSet, scopes []domai } } -func (service *StorageAnalyticsService) loadProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) (*storageAuditRecordSet, error) { - projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project, pathPrefix) - if err != nil { - return nil, fmt.Errorf("list syfon project audit records: %w", err) - } - return buildProjectAuditRecordSet(projectRecords), nil -} - func (service *StorageAnalyticsService) loadCachedProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string) (*storageAuditRecordSet, error) { projectRecords, err := service.loadCachedProjectAuditRecords(ctx, authorizationHeader, organization, project) if err != nil { @@ -469,54 +461,6 @@ func projectAuditRecordValidatorFromSummary(summary gintegrationsyfon.ProjectMet } } -func filterProjectAuditRecordSet(recordSet *storageAuditRecordSet, pathPrefix string, scopes []domain.StorageBucketScope, organization string, project string) *storageAuditRecordSet { - if recordSet == nil || normalizeRepoSubpath(pathPrefix) == "" { - return recordSet - } - filtered := make(map[string][]projectRecordState, len(recordSet.allProjectRecords)) - for checksum, group := range recordSet.allProjectRecords { - for _, record := range group { - if projectAuditRecordMatchesPathPrefix(record.ProjectRecord, pathPrefix, scopes, organization, project) { - filtered[checksum] = append(filtered[checksum], record) - } - } - } - return &storageAuditRecordSet{ - recordsByChecksum: cloneRecordStateMap(filtered), - allProjectRecords: cloneRecordStateMap(filtered), - } -} - -func projectAuditRecordMatchesPathPrefix(record gintegrationsyfon.ProjectRecord, pathPrefix string, scopes []domain.StorageBucketScope, organization string, project string) bool { - normalizedPrefix := normalizeRepoSubpath(pathPrefix) - if normalizedPrefix == "" { - return true - } - for _, accessURL := range projectAuditRecordPathURLs(record, scopes, organization, project) { - _, key, ok := parseStorageURL(accessURL) - if !ok { - continue - } - key = normalizeRepoSubpath(key) - if key == normalizedPrefix || strings.HasPrefix(key, normalizedPrefix+"/") { - return true - } - } - return false -} - -func projectAuditRecordPathURLs(record gintegrationsyfon.ProjectRecord, scopes []domain.StorageBucketScope, organization string, project string) []string { - out := make([]string, 0, len(record.AccessURLs)+len(record.AccessMethods)*2) - out = append(out, record.AccessURLs...) - for _, method := range record.AccessMethods { - if trimmed := strings.TrimSpace(method.URL); trimmed != "" { - out = append(out, trimmed) - } - } - out = append(out, canonicalizeRecordAccessURLs(out, scopes, organization, project)...) - return uniqueStrings(out) -} - func (service *StorageAnalyticsService) loadProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { bucketObjects, err := service.storage.ListProjectBucketObjects(ctx, authorizationHeader, organization, project, bucketPathPrefix) if err != nil { @@ -593,34 +537,6 @@ func (service *StorageAnalyticsService) loadCachedProjectBucketValidationInvento return objects, lookup, nil } -func (service *StorageAnalyticsService) loadCachedProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) { - cacheKey := service.projectChainInputCacheKey(organization, project) + "::bucket-summary::" + strings.TrimSpace(mode) - service.chainInputMu.RLock() - cached, ok := service.chainInputCache[cacheKey] - service.chainInputMu.RUnlock() - if ok && time.Now().Before(cached.expiresAt) && cached.bucketSummary != nil { - summary := *cached.bucketSummary - return &summary, nil - } - summary, err := service.storage.ListProjectBucketSummary(ctx, authorizationHeader, organization, project, mode) - if err != nil { - return nil, err - } - service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { - if summary == nil { - state.bucketSummary = nil - return - } - copy := *summary - state.bucketSummary = © - }) - if summary == nil { - return nil, nil - } - copy := *summary - return ©, nil -} - func (service *StorageAnalyticsService) projectChainInputCacheKey(organization string, project string) string { return strings.TrimSpace(organization) + "/" + strings.TrimSpace(project) } @@ -947,52 +863,6 @@ func inventoryMissingProbe(record projectRecordState, objectURL string, expected } } -func (service *StorageAnalyticsService) loadStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet) (*storageAuditStorageView, error) { - scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) - if err != nil { - return nil, err - } - recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) - view := &storageAuditStorageView{ - scopes: scopes, - recordsByChecksum: recordSet.recordsByChecksum, - allProjectRecords: recordSet.allProjectRecords, - bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, - bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, - bucketInventoryAvailable: true, - } - bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project, "") - if err != nil { - if !shouldDegradeBucketInventory(err) { - return nil, err - } - view.bucketInventoryAvailable = false - view.bucketInventoryError = strings.TrimSpace(err.Error()) - probedRecordSet, probeErr := service.attachProjectStorageProbes(ctx, authorizationHeader, recordSet) - if probeErr != nil { - return nil, probeErr - } - view.recordsByChecksum = probedRecordSet.recordsByChecksum - view.allProjectRecords = probedRecordSet.allProjectRecords - view.bucketObjects, view.bucketObjectsByURL = synthesizeBucketInventoryFromProbes(probedRecordSet.allProjectRecords) - return view, nil - } - view.bucketObjects = bucketObjects - view.bucketObjectsByURL = bucketObjectsByURL - - probeCandidates := selectTargetedProbeRecordSet(recordSet, bucketObjectsByURL) - if probeCandidates != nil { - probedSubset, probeErr := service.attachProjectStorageProbes(ctx, authorizationHeader, probeCandidates) - if probeErr != nil { - return nil, probeErr - } - merged := mergeRecordSetProbes(recordSet, probedSubset) - view.recordsByChecksum = merged.recordsByChecksum - view.allProjectRecords = merged.allProjectRecords - } - return view, nil -} - func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, inventory []RepoInventoryFile, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) view := &storageAuditStorageView{ @@ -1567,7 +1437,7 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum } acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) case storageFindingBrokenAccessURL, storageFindingProbeError: - findings := buildChainRecordFindings("probe_error", record, gitPaths, bucketMatches, "Bucket verification failed before Gecko could classify this record cleanly.") + findings := buildChainRecordFindings("probe_error", record, gitPaths, bucketMatches, "Storage probes could not confirm this mapped object; Gecko does not treat it as missing or offer cleanup.") acc.findings = append(acc.findings, findings...) acc.addCount("probe_error", chainPathCount(gitPaths)) case storageFindingNone: diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index c432908..e1b087a 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -367,9 +367,13 @@ func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { "data/nested/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), "plain.txt": "not lfs\n", }) - inventory, err := BuildGitRepoInventory(refName, "data", repo, hash) + index, err := buildRepoAnalyticsIndex(refName, repo, hash) if err != nil { - t.Fatalf("build repo inventory: %v", err) + t.Fatalf("build repo analytics index: %v", err) + } + inventory, err := filterRepoInventoryFiles(index, "data") + if err != nil { + t.Fatalf("filter repo inventory: %v", err) } if len(inventory) != 4 { t.Fatalf("expected 4 lfs files under data, got %+v", inventory) @@ -1700,7 +1704,7 @@ func TestBuildStorageChainAuditUsesBucketFirstFindingKinds(t *testing.T) { } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build chain audit: %v", err) } @@ -1746,7 +1750,7 @@ func TestBuildStorageChainAuditUsesProjectAuditSourcesAndTargetsProbes(t *testin } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build chain audit: %v", err) } @@ -2691,7 +2695,7 @@ func TestBuildStorageChainAuditSurfacesMetadataMismatchAndEvidence(t *testing.T) } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build chain audit: %v", err) } @@ -2884,7 +2888,7 @@ func TestBuildStorageChainAuditUsesScopedProjectRecordsForGitJoin(t *testing.T) } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build chain audit: %v", err) } @@ -2953,7 +2957,7 @@ func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "data", mirrorPath, repo, hash) + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build chain audit: %v", err) } @@ -3017,7 +3021,7 @@ func TestBuildStorageChainAuditClassifiesSameBucketPathChecksumConflictAsMetadat } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash) + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build chain audit: %v", err) } @@ -3584,7 +3588,7 @@ func TestBuildStorageChainAuditDoesNotInferMappingMismatchFromRelocatedBasename( } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "CONFIG", mirrorPath, repo, hash) + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "CONFIG", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build chain audit: %v", err) } @@ -3821,7 +3825,7 @@ func TestBuildStorageChainAuditNormalizesChecksumJoinKeys(t *testing.T) { } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build chain audit: %v", err) } @@ -3931,7 +3935,7 @@ func TestBuildStorageChainAuditReturnsFullFindings(t *testing.T) { } service := NewStorageAnalyticsService(backend) - chain, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build chain audit: %v", err) } diff --git a/internal/git/storage_audit_registration_test.go b/internal/git/storage_audit_registration_test.go index 477e20c..408c03b 100644 --- a/internal/git/storage_audit_registration_test.go +++ b/internal/git/storage_audit_registration_test.go @@ -30,7 +30,7 @@ func TestRegisterGitOnlySyfonRecordsUsesScopedBucketEvidence(t *testing.T) { } service := NewStorageAnalyticsService(backend) - audit, err := service.BuildStorageChainAudit(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash) + audit, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{}) if err != nil { t.Fatalf("build audit: %v", err) } diff --git a/internal/git/storage_index.go b/internal/git/storage_index.go index f485bf8..76ccd3e 100644 --- a/internal/git/storage_index.go +++ b/internal/git/storage_index.go @@ -55,16 +55,6 @@ func repoAnalyticsCacheKey(mirrorPath string, hash plumbing.Hash) string { return strings.TrimSpace(mirrorPath) + "::" + hash.String() } -func (cache *repoAnalyticsIndexMemoryCache) get(mirrorPath string, hash plumbing.Hash) *repoAnalyticsIndex { - cache.mu.RLock() - defer cache.mu.RUnlock() - entry := cache.entries[repoAnalyticsCacheKey(mirrorPath, hash)] - if entry == nil { - return nil - } - return entry -} - func (cache *repoAnalyticsIndexMemoryCache) put(mirrorPath string, hash plumbing.Hash, index *repoAnalyticsIndex) { if index == nil { return diff --git a/internal/git/tree_response_test.go b/internal/git/tree_response_test.go index 1d5db65..20741f7 100644 --- a/internal/git/tree_response_test.go +++ b/internal/git/tree_response_test.go @@ -75,16 +75,6 @@ func TestBuildGitResponsesDetectLFSPointers(t *testing.T) { t.Fatalf("unexpected lfs size: %d", treePointer.Size) } - fileResponse, err := BuildGitFileResponse("org-a/proj-a", refName, "data/tcga.tumor.ensembl.tsv", mirrorRepo, hash) - if err != nil { - t.Fatalf("build file response: %v", err) - } - if fileResponse.LFSPointer == nil { - t.Fatalf("expected file response to include lfs pointer metadata") - } - if fileResponse.LFSPointer.OID != treePointer.OID { - t.Fatalf("expected matching lfs oid, got %q and %q", fileResponse.LFSPointer.OID, treePointer.OID) - } } func TestBuildGitTreeResponseDefaultsToCheapFields(t *testing.T) { diff --git a/internal/integrations/fence/client.go b/internal/integrations/fence/client.go index 5ea859a..36e3437 100644 --- a/internal/integrations/fence/client.go +++ b/internal/integrations/fence/client.go @@ -181,26 +181,6 @@ func (c *Client) ListInstallationRepositories(ctx context.Context, authorization return payload.Repositories, nil } -func (c *Client) RequestInstallationStatus(ctx context.Context, authorizationHeader string, organization string, identity domain.GitRepositoryIdentity) (domain.GitRepositoryInstallationStatus, error) { - var payload fenceGitHubInstallationStatusResponse - if err := c.requestFenceGitHubBroker(ctx, authorizationHeader, map[string]any{ - "action": "repository_installation", - "owner": identity.Owner, - "repo": identity.Repo, - "organization": organization, - }, &payload); err != nil { - return domain.GitRepositoryInstallationStatus{}, err - } - return domain.GitRepositoryInstallationStatus{ - Installed: payload.Installed, - InstallationID: payload.InstallationID, - Target: strings.TrimSpace(payload.Target), - TargetType: strings.TrimSpace(payload.TargetType), - HTMLURL: strings.TrimSpace(payload.HTMLURL), - RepositorySelection: strings.TrimSpace(payload.RepositorySelection), - }, nil -} - func (c *Client) RequestInstallationToken(ctx context.Context, authorizationHeader string, organization string, project string, identity domain.GitRepositoryIdentity, access string) (string, error) { requestedAccess := strings.TrimSpace(access) if requestedAccess == "" { diff --git a/internal/integrations/fence/client_test.go b/internal/integrations/fence/client_test.go index c5a35b4..1891689 100644 --- a/internal/integrations/fence/client_test.go +++ b/internal/integrations/fence/client_test.go @@ -65,49 +65,6 @@ func TestRequestOrganizationInstallationStatusForwardsAuthorizationAndParsesStat } } -func TestRequestInstallationStatusForwardsAuthorizationAndParsesStatus(t *testing.T) { - var receivedAuth string - server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - receivedAuth = request.Header.Get("Authorization") - if request.URL.Path != "/credentials/github" { - t.Fatalf("unexpected request path: %s", request.URL.Path) - } - if request.Method != http.MethodPost { - t.Fatalf("unexpected request method: %s", request.Method) - } - writer.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(writer).Encode(map[string]any{ - "installed": true, - "installation_id": 42, - "target": "HTAN_INT", - "target_type": "Organization", - "html_url": "https://github.com/organizations/HTAN_INT/settings/installations/42", - }) - })) - defer server.Close() - - client := NewClient(server.Client(), Config{BaseURL: server.URL}) - status, err := client.RequestInstallationStatus(context.Background(), "Bearer user-token", "HTAN_INT", domain.GitRepositoryIdentity{ - Owner: "HTAN_INT", - Repo: "BForePC", - }) - if err != nil { - t.Fatalf("request installation status: %v", err) - } - if !status.Installed { - t.Fatal("expected installed status") - } - if status.InstallationID == nil || *status.InstallationID != 42 { - t.Fatalf("unexpected installation id: %+v", status.InstallationID) - } - if status.Target != "HTAN_INT" { - t.Fatalf("unexpected target: %q", status.Target) - } - if receivedAuth != "Bearer user-token" { - t.Fatalf("expected forwarded authorization header, got %q", receivedAuth) - } -} - func TestListInstallationRepositoriesForwardsAuthorizationAndParsesRepositories(t *testing.T) { var receivedAuth string var receivedBody map[string]any diff --git a/internal/server/http/config/config.go b/internal/server/http/config/config.go index ca871e3..fefc2a8 100644 --- a/internal/server/http/config/config.go +++ b/internal/server/http/config/config.go @@ -145,16 +145,3 @@ func (handler *Handler) resolveProjectConfigParams(ctx fiber.Ctx) (string, strin } return handler.resolveConfigParams(ctx) } - -func mergeErrorDetails(base map[string]any, extra map[string]any) map[string]any { - if len(extra) == 0 { - return base - } - if base == nil { - base = map[string]any{} - } - for k, v := range extra { - base[k] = v - } - return base -} diff --git a/internal/server/http/git/helpers.go b/internal/server/http/git/helpers.go index dd0d127..0c3e51b 100644 --- a/internal/server/http/git/helpers.go +++ b/internal/server/http/git/helpers.go @@ -9,14 +9,6 @@ import ( "github.com/gofiber/fiber/v3" ) -func (handler *Handler) authenticatedUserID(ctx fiber.Ctx) (string, *httputil.ErrorResponse) { - return handler.AuthenticatedUserID(ctx) -} - -func (handler *Handler) writeAppError(ctx fiber.Ctx, err error) error { - return handler.WriteAppError(ctx, err) -} - func writeAppError(ctx fiber.Ctx, logger any, err error) error { _ = logger if appErr, ok := err.(*git.Error); ok { diff --git a/internal/server/http/shared/helpers.go b/internal/server/http/shared/helpers.go index a71d4dc..2686d92 100644 --- a/internal/server/http/shared/helpers.go +++ b/internal/server/http/shared/helpers.go @@ -1,16 +1,6 @@ package shared -import ( - "fmt" - "net/http" - "strings" - - "github.com/calypr/gecko/apierror" - "github.com/calypr/gecko/internal/git" - "github.com/calypr/gecko/internal/httputil" - servermw "github.com/calypr/gecko/internal/server/middleware" - "github.com/gofiber/fiber/v3" -) +import "github.com/gofiber/fiber/v3" func ConfigTypeMiddleware(configType string) fiber.Handler { return func(ctx fiber.Ctx) error { @@ -18,57 +8,3 @@ func ConfigTypeMiddleware(configType string) fiber.Handler { return ctx.Next() } } - -func (handler *Handler) WriteAppError(ctx fiber.Ctx, err error) error { - if err == nil { - return nil - } - appErr, ok := err.(*git.Error) - if !ok { - response := httputil.NewError(apierror.Type("internal_error"), err.Error(), http.StatusInternalServerError, nil, nil) - response.WriteLog(handler.Logger) - return response.Write(ctx) - } - errorType := apierror.Type("internal_error") - switch appErr.Kind { - case git.ErrorKindValidation: - errorType = apierror.TypeValidationFailed - case git.ErrorKindForbidden: - errorType = apierror.TypeForbidden - case git.ErrorKindIntegration: - errorType = apierror.Type("integration_error") - case git.ErrorKindNotFound: - errorType = apierror.TypeNotFound - case git.ErrorKindDatabase: - errorType = apierror.TypeDatabaseError - case git.ErrorKindUnauthorized: - errorType = apierror.TypeMissingAuthorization - } - statusCode := appErr.StatusCode - if statusCode == 0 { - statusCode = http.StatusInternalServerError - } - response := httputil.NewError(errorType, appErr.Error(), statusCode, appErr.Details, nil) - response.WriteLog(handler.Logger) - return response.Write(ctx) -} - -func (handler *Handler) AuthenticatedUserID(ctx fiber.Ctx) (string, *httputil.ErrorResponse) { - authorizationHeader, tokenErr := servermw.ValidateAuthorizationHeader(ctx.Get("Authorization")) - if tokenErr != nil { - return "", httputil.NewError(apierror.TypeMissingAuthorization, tokenErr.Error(), http.StatusUnauthorized, nil, nil) - } - if handler.JWTApp == nil { - return "", httputil.NewError(apierror.TypeInvalidJWTHandler, "JWT validation is not configured", http.StatusUnauthorized, nil, nil) - } - claims, err := handler.JWTApp.Decode(servermw.CleanAccessToken(authorizationHeader)) - if err != nil { - return "", httputil.NewError(apierror.TypeUnauthorized, fmt.Sprintf("failed to decode authorization token: %s", err), http.StatusUnauthorized, nil, nil) - } - for _, claim := range []string{"sub", "username", "email"} { - if value, ok := (*claims)[claim].(string); ok && strings.TrimSpace(value) != "" { - return strings.TrimSpace(value), nil - } - } - return "", httputil.NewError(apierror.TypeUnauthorized, "authorization token does not include a stable user id", http.StatusUnauthorized, nil, nil) -} diff --git a/internal/server/middleware/access.go b/internal/server/middleware/access.go index bf6faba..3e58cd6 100644 --- a/internal/server/middleware/access.go +++ b/internal/server/middleware/access.go @@ -3,7 +3,6 @@ package middleware import ( "fmt" "net/url" - "sort" "strings" ) @@ -63,18 +62,3 @@ func ResourceListAllowsOrganization(resources []string, organization string) boo } return false } - -func ResourceListAllowedOrganizations(resources []string) []string { - seen := make(map[string]struct{}) - for _, resource := range resources { - if organization, ok := ResourcePathOrganization(resource); ok { - seen[organization] = struct{}{} - } - } - organizations := make([]string, 0, len(seen)) - for organization := range seen { - organizations = append(organizations, organization) - } - sort.Strings(organizations) - return organizations -} diff --git a/internal/server/middleware/auth.go b/internal/server/middleware/auth.go index 199afd9..366e8a4 100644 --- a/internal/server/middleware/auth.go +++ b/internal/server/middleware/auth.go @@ -115,34 +115,6 @@ func GeneralAuth(logger arborist.Logger, authzHandler ResourceAccessHandler, met } } -func BaseConfigsAuth(logger arborist.Logger, authzHandler ResourceAccessHandler, method, service, resourcePath string) fiber.Handler { - return func(ctx fiber.Ctx) error { - authorizationHeader := ctx.Get("Authorization") - if authorizationHeader == "" { - return writeError(ctx, logger, httputil.NewError(apierror.TypeMissingAuthorization, "Authorization token not provided", http.StatusUnauthorized, nil, nil)) - } - - if _, ok := authzHandler.(*FenceUserAccessHandler); !ok { - return writeError(ctx, logger, httputil.NewError("internal_server_error", "Invalid JWT handler configuration", http.StatusInternalServerError, nil, nil)) - } - - allowed, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, method, service, resourcePath) - if err != nil { - if serverErr, ok := err.(*ggmw.ServerError); ok { - return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) - } - if accessErr, ok := err.(*AccessError); ok { - return writeError(ctx, logger, httputil.NewError(serviceErrorType(accessErr.StatusCode), accessErr.Message, accessErr.StatusCode, nil, nil)) - } - return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) - } - if !allowed { - return writeError(ctx, logger, httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User does not have required %s permission on resource %s", method, "/programs"), http.StatusForbidden, map[string]any{"resource": resourcePath, "method": method}, nil)) - } - return ctx.Next() - } -} - func ProjectConfigAuth(logger arborist.Logger, authzHandler ResourceAccessHandler, method string) fiber.Handler { return func(ctx fiber.Ctx) error { authorizationHeader := ctx.Get("Authorization") @@ -412,35 +384,6 @@ func writeError(ctx fiber.Ctx, logger arborist.Logger, response *httputil.ErrorR return response.Write(ctx) } -// Migrated from internal/authz -func GetProjectsFromToken(ctx fiber.Ctx, authzHandler ResourceAccessHandler, method string, service string) ([]any, *httputil.ErrorResponse) { - token := ctx.Get("Authorization") - if token == "" { - return nil, httputil.NewError(apierror.TypeMissingAuthorization, "Authorization token not provided", http.StatusUnauthorized, nil, nil) - } - anyList, err := authzHandler.GetAllowedResources(token, method, service) - if err != nil { - if serverErr, ok := err.(*AccessError); ok { - return nil, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil) - } - return nil, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil) - } - return anyList, nil -} - -// Migrated from internal/authz -func ParseAccess(resourceList []string, resource string, method string) *httputil.ErrorResponse { - if len(resourceList) == 0 { - return httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User is not allowed to %s on any resource path", method), http.StatusForbidden, map[string]any{"resource": resource, "method": method}, nil) - } - for _, v := range resourceList { - if v == resource { - return nil - } - } - return httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User is not allowed to %s on resource path: %s", method, resource), http.StatusForbidden, map[string]any{"resource": resource, "method": method}, nil) -} - func CleanAccessToken(raw string) string { token := strings.TrimSpace(raw) if strings.HasPrefix(strings.ToLower(token), "bearer ") { diff --git a/internal/server/middleware/fence_user_test.go b/internal/server/middleware/fence_user_test.go index 7eb7fa5..35ac389 100644 --- a/internal/server/middleware/fence_user_test.go +++ b/internal/server/middleware/fence_user_test.go @@ -2,22 +2,6 @@ package middleware import "testing" -func TestSnapshotAllows(t *testing.T) { - raw := []any{ - map[string]any{"service": "*", "method": "read"}, - map[string]any{"service": "arborist", "method": "create-descendant"}, - } - if !snapshotAllows(raw, "read", "*") { - t.Fatalf("expected wildcard read to match") - } - if !snapshotAllows(raw, "create-descendant", "arborist") { - t.Fatalf("expected arborist create-descendant to match") - } - if snapshotAllows(raw, "update", "*") { - t.Fatalf("did not expect update to match") - } -} - func TestFenceUserEndpoint(t *testing.T) { token := "Bearer eyJhbGciOiJub25lIn0.eyJpc3MiOiJodHRwczovL2V4YW1wbGUub3JnL3VzZXIifQ." endpoint, err := fenceUserEndpoint(token) diff --git a/internal/server/middleware/git_test.go b/internal/server/middleware/git_test.go index c6cf4f4..405aae8 100644 --- a/internal/server/middleware/git_test.go +++ b/internal/server/middleware/git_test.go @@ -393,19 +393,6 @@ func TestProjectConfigAuthAllowsAdminWildcardResource(t *testing.T) { } } -func TestSnapshotAllowsAcceptsWildcardMethod(t *testing.T) { - raw := []any{ - map[string]any{ - "method": "*", - "service": "*", - }, - } - - if !snapshotAllows(raw, "delete", "*") { - t.Fatalf("expected wildcard method/service snapshot entry to allow delete") - } -} - func TestGitProjectAuthForbiddenIncludesRequestAccessDetails(t *testing.T) { logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} app := fiber.New() diff --git a/internal/server/middleware/middleware.go b/internal/server/middleware/middleware.go index d974c56..3e659c6 100644 --- a/internal/server/middleware/middleware.go +++ b/internal/server/middleware/middleware.go @@ -147,28 +147,6 @@ func parseResourceAccessSnapshot(payload map[string]any) (ResourceAccessSnapshot return snapshot, nil } -func snapshotAllows(raw any, method, service string) bool { - entries, ok := raw.([]any) - if !ok { - return false - } - for _, entry := range entries { - record, ok := entry.(map[string]any) - if !ok { - continue - } - entryMethod, _ := record["method"].(string) - entryService, _ := record["service"].(string) - if entryMethod != method && entryMethod != "*" { - continue - } - if entryService == "*" || service == "*" || entryService == service { - return true - } - } - return false -} - func ResourceAccessAllows(snapshot ResourceAccessSnapshot, resourcePath, method, service string) bool { entries := snapshot[resourcePath] for _, entry := range entries { diff --git a/internal/storageaudit/evidence.go b/internal/storageaudit/evidence.go index e1b40d9..6a736dd 100644 --- a/internal/storageaudit/evidence.go +++ b/internal/storageaudit/evidence.go @@ -13,6 +13,7 @@ const ( OperationInventory = "inventory" OperationList = "list" OperationMetadata = "metadata" + OperationDownload = "download" ) type Probe struct { @@ -56,10 +57,10 @@ func Assess(probes []Probe, bucketObserved bool) Assessment { assessment.Present = true assessment.Status = EvidenceVerified } - // Exact LIST is useful for discovering a candidate, but some S3-compatible - // backends can return an empty page under load without returning an error. - // Only a metadata lookup (HEAD) is authoritative enough to prove absence. - if operation == OperationMetadata && status == "not_found" && (errorKind == "" || errorKind == "object_not_found") { + // S3-compatible backends can report a false miss for LIST or HEAD while a + // download-compatible GET still succeeds. Only an explicit GET-style probe + // may prove absence and unlock destructive repair. + if operation == OperationDownload && status == "not_found" && (errorKind == "" || errorKind == "object_not_found") { assessment.Missing = true assessment.Status = EvidenceVerified } diff --git a/internal/storageaudit/evidence_test.go b/internal/storageaudit/evidence_test.go index 8c139f9..b5c4536 100644 --- a/internal/storageaudit/evidence_test.go +++ b/internal/storageaudit/evidence_test.go @@ -26,15 +26,27 @@ func TestAssessTreatsExactListMissAsUnconfirmed(t *testing.T) { } } -func TestAssessTreatsMetadataMissAsVerified(t *testing.T) { +func TestAssessTreatsMetadataMissAsUnconfirmed(t *testing.T) { assessment := Assess([]Probe{{ Operation: OperationMetadata, Status: "not_found", ErrorKind: "object_not_found", ValidationStatus: "unverifiable", }}, false) - if assessment.Status != EvidenceVerified || !assessment.Missing || !assessment.HasExactEvidence { - t.Fatalf("expected metadata absence to be verified, got %+v", assessment) + if assessment.Status != EvidenceUnknown || assessment.Missing || !assessment.HasExactEvidence { + t.Fatalf("expected metadata absence to require a download-compatible probe, got %+v", assessment) + } +} + +func TestAssessTreatsDownloadMissAsVerified(t *testing.T) { + assessment := Assess([]Probe{{ + Operation: OperationDownload, + Status: "not_found", + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }}, false) + if assessment.Status != EvidenceVerified || !assessment.Missing { + t.Fatalf("expected a download miss to verify absence, got %+v", assessment) } } diff --git a/tests/integration/middleware_test.go b/tests/integration/middleware_test.go index e5a4dfc..092aee5 100644 --- a/tests/integration/middleware_test.go +++ b/tests/integration/middleware_test.go @@ -122,17 +122,6 @@ func TestGeneralAuthMware_ConvertAnyToStringSliceError(t *testing.T) { assert.Contains(t, body, "Element 123 is not a string") } -func TestGeneralAuthMware_ParseAccessDenied(t *testing.T) { - srv := setupServer() - app := fiber.New() - app.Get("/:projectId", servermw.GeneralAuth(srv.Logger, &MockJWTHandler{AllowedResources: []string{"/programs/other/projects/test"}}, "read", "*"), func(c fiber.Ctx) error { return c.SendStatus(http.StatusOK) }) - req := httptest.NewRequest(http.MethodGet, "/ohsu-test", nil) - req.Header.Set("Authorization", "Bearer dummy") - resp, body := runFiber(app, req, t) - assert.Equal(t, http.StatusForbidden, resp.StatusCode) - assert.Contains(t, body, "User is not allowed to read on resource path") -} - func TestConfigAuthMiddleware_MethodNotAllowed(t *testing.T) { srv := setupServer() app := fiber.New() @@ -154,26 +143,6 @@ func TestConfigAuthMiddleware_Nav_PublicGET(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) } -func TestBaseConfigsAuthMiddleware_NoAuthorization(t *testing.T) { - srv := setupServer() - app := fiber.New() - app.Get("/", servermw.BaseConfigsAuth(srv.Logger, &MockJWTHandler{}, "read", "*", "/programs"), func(c fiber.Ctx) error { return c.SendStatus(http.StatusOK) }) - resp, body := runFiber(app, httptest.NewRequest(http.MethodGet, "/", nil), t) - assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) - assert.Contains(t, body, "Authorization token not provided") -} - -func TestBaseConfigsAuthMiddleware_InvalidJWTHandler(t *testing.T) { - srv := setupServer() - app := fiber.New() - app.Get("/", servermw.BaseConfigsAuth(srv.Logger, &MockJWTHandler{}, "read", "*", "/programs"), func(c fiber.Ctx) error { return c.SendStatus(http.StatusOK) }) - req := httptest.NewRequest(http.MethodGet, "/", nil) - req.Header.Set("Authorization", "Bearer dummy") - resp, body := runFiber(app, req, t) - assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) - assert.Contains(t, body, "Invalid JWT handler configuration") -} - func TestConfigAuthMiddleware_Project_PublicGET(t *testing.T) { srv := setupServer() app := fiber.New() From 2dc89a20e61dd4e8c473769f7c00e4786af5a74f Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 9 Jul 2026 18:50:55 -0700 Subject: [PATCH 33/36] fix timeout issues related to performance of storage/folder?limit=100&sort_by=bytes&sort_order=desc&summary_mode=exact gecko endpoint --- internal/git/storage_analytics.go | 30 ++-- internal/git/storage_analytics_pipeline.go | 12 -- internal/git/storage_analytics_test.go | 14 +- internal/git/storage_audit_evidence.go | 2 +- .../git/storage_audit_evidence_policy_test.go | 81 +++++++++++ .../storage_audit_list_confirmation_test.go | 131 +++++++----------- 6 files changed, 158 insertions(+), 112 deletions(-) create mode 100644 internal/git/storage_audit_evidence_policy_test.go diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index b9ea2a2..52d4a66 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -87,18 +87,30 @@ func storageRepairPolicyForKind(kind string) storageRepairPolicy { // A mismatch cannot establish which of Git, Syfon, or the bucket is // authoritative. Requiring an explicit out-of-band reconciliation keeps // a stale checksum from becoming a destructive cleanup candidate. + return inspectOnlyStorageRepairPolicy() + case "probe_error": + // Probe errors do not prove that the object is absent, but the audit user + // may explicitly discard the affected Syfon records after reviewing them. return storageRepairPolicy{ - actionability: storageActionabilityInspectOnly, - actions: []string{storageActionInspectEvidence}, - defaultAction: storageActionInspectEvidence, + actionability: storageActionabilityManualChoice, + actions: []string{ + storageActionDeleteSyfonRecord, + storageActionInspectEvidence, + }, + defaultAction: storageActionDeleteSyfonRecord, + supportsDryRun: true, } default: - return storageRepairPolicy{ - actionability: storageActionabilityInspectOnly, - actions: []string{storageActionInspectEvidence}, - defaultAction: storageActionInspectEvidence, - supportsDryRun: false, - } + return inspectOnlyStorageRepairPolicy() + } +} + +func inspectOnlyStorageRepairPolicy() storageRepairPolicy { + return storageRepairPolicy{ + actionability: storageActionabilityInspectOnly, + actions: []string{storageActionInspectEvidence}, + defaultAction: storageActionInspectEvidence, + supportsDryRun: false, } } diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 8bdd6f7..b3662a4 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -906,18 +906,6 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex } probedRecordSet = mergeRecordSetProbes(probedRecordSet, exactRecordSet) view.bucketObjects, view.bucketObjectsByURL = mergeBucketInventoryWithPresentProbes(view.bucketObjects, view.bucketObjectsByURL, exactRecordSet) - - confirmationCandidates := selectInventoryMissRecordSet(exactRecordSet) - if confirmationCandidates != nil { - confirmationStart := time.Now() - confirmedRecordSet, confirmationErr := service.attachProjectStorageProbes(ctx, authorizationHeader, confirmationCandidates) - timings.Record("targeted_metadata_confirmation", time.Since(confirmationStart)) - if confirmationErr != nil { - return nil, confirmationErr - } - probedRecordSet = mergeRecordSetProbes(probedRecordSet, confirmedRecordSet) - view.bucketObjects, view.bucketObjectsByURL = mergeBucketInventoryWithPresentProbes(view.bucketObjects, view.bucketObjectsByURL, confirmedRecordSet) - } } timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL), "exact_probe_records", countRecordSet(candidates)) view.recordsByChecksum = probedRecordSet.recordsByChecksum diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index e1b087a..33a7013 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -1234,7 +1234,7 @@ func TestStorageRepairPolicyCoversFindingKinds(t *testing.T) { {"syfon_missing_bucket_object", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, {"git_only_no_syfon", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, {"storage_probe_error", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, - {"probe_error", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, + {"probe_error", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityManualChoice}, } for _, test := range tests { t.Run(test.kind, func(t *testing.T) { @@ -3198,7 +3198,7 @@ func TestBuildStorageChainAuditUsesCanonicalInventoryCandidatesBeforeMissingBuck } } -func TestBuildStorageChainAuditMetadataConfirmationSuppressesBForePCListFalsePositive(t *testing.T) { +func TestBuildStorageChainAuditBForePCListMissIsProbeErrorWithoutMetadataConfirmation(t *testing.T) { const repoPath = "OHSU/koei_chin/visium_hd/4-R2/outs/binned_outputs/square_002um/filtered_feature_bc_matrix/barcodes.tsv.gz" const checksum = "a9809ffd53130272f775ad1bbea3166dbb4853bc6fa73bfa7a51535c4bc4f599" const size = int64(46063992) @@ -3252,14 +3252,14 @@ func TestBuildStorageChainAuditMetadataConfirmationSuppressesBForePCListFalsePos if backend.listProbeCalls != 1 || len(backend.listProbeItems) != 1 || backend.listProbeItems[0].ObjectURL != canonicalURL { t.Fatalf("expected one exact LIST for %q, got calls=%d items=%+v", canonicalURL, backend.listProbeCalls, backend.listProbeItems) } - if backend.probeCalls != 1 || len(backend.probeItems) != 1 || backend.probeItems[0].ObjectURL != canonicalURL { - t.Fatalf("expected one metadata confirmation for the false LIST miss at %q, got calls=%d items=%+v", canonicalURL, backend.probeCalls, backend.probeItems) + if backend.probeCalls != 0 { + t.Fatalf("expected no metadata confirmation for the LIST miss at %q, got calls=%d items=%+v", canonicalURL, backend.probeCalls, backend.probeItems) } if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 0 { - t.Fatalf("metadata presence must suppress the false LIST missing-bucket finding, got summary %+v", chain.Summary) + t.Fatalf("LIST miss must not become a missing-bucket finding, got summary %+v", chain.Summary) } - if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { - t.Fatalf("expected the exact-present object to complete the chain, got summary %+v", chain.Summary) + if got := chain.Summary.CountsByKind["probe_error"]; got != 1 { + t.Fatalf("expected the exact LIST miss to remain a probe error, got summary %+v", chain.Summary) } } diff --git a/internal/git/storage_audit_evidence.go b/internal/git/storage_audit_evidence.go index 30d991a..f32ac3d 100644 --- a/internal/git/storage_audit_evidence.go +++ b/internal/git/storage_audit_evidence.go @@ -50,7 +50,7 @@ func storageChainActionSupportForEvidence(kind string, evidenceStatus string) (s } policy := storageRepairPolicyForKind(kind) if policy.actionability == storageActionabilityAutoRepair && !storageaudit.AllowsAutomaticRepair(kind, storageaudit.EvidenceStatus(evidenceStatus)) { - policy = storageRepairPolicyForKind("probe_error") + policy = inspectOnlyStorageRepairPolicy() } return policy.actionability, append([]string(nil), policy.actions...), policy.defaultAction, policy.supportsDryRun } diff --git a/internal/git/storage_audit_evidence_policy_test.go b/internal/git/storage_audit_evidence_policy_test.go new file mode 100644 index 0000000..071cb35 --- /dev/null +++ b/internal/git/storage_audit_evidence_policy_test.go @@ -0,0 +1,81 @@ +package git + +import ( + "context" + "testing" + + "github.com/calypr/gecko/internal/storageaudit" +) + +func TestStorageChainActionSupportAllowsManualProbeErrorDeletion(t *testing.T) { + actionability, actions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence( + "probe_error", + string(storageaudit.EvidenceUnknown), + ) + + if actionability != storageActionabilityManualChoice { + t.Fatalf("expected manual probe-error deletion, got %q", actionability) + } + if defaultAction != storageActionDeleteSyfonRecord { + t.Fatalf("expected delete-Syfon default, got %q", defaultAction) + } + if !supportsDryRun { + t.Fatal("expected probe-error deletion to support a dry run") + } + if len(actions) != 2 || actions[0] != storageActionDeleteSyfonRecord || actions[1] != storageActionInspectEvidence { + t.Fatalf("unexpected probe-error actions: %+v", actions) + } +} + +func TestStorageChainActionSupportKeepsUnverifiedMissingFindingInspectOnly(t *testing.T) { + actionability, actions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence( + "syfon_git_no_bucket", + string(storageaudit.EvidenceUnknown), + ) + + if actionability != storageActionabilityInspectOnly { + t.Fatalf("expected unverified missing finding to remain inspect-only, got %q", actionability) + } + if defaultAction != storageActionInspectEvidence || supportsDryRun { + t.Fatalf("unexpected unverified finding defaults: action=%q dry_run=%t", defaultAction, supportsDryRun) + } + if len(actions) != 1 || actions[0] != storageActionInspectEvidence { + t.Fatalf("unexpected unverified finding actions: %+v", actions) + } +} + +func TestApplyStorageCleanupDeletesSelectedProbeErrorRecords(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + response, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "project", + []string{"probe-row"}, + []GitStorageCleanupApplyAction{{ + Kind: "probe_error", + NormalizedPath: "probe-row", + Action: storageActionDeleteSyfonRecord, + }}, + []GitStorageCleanupApplyFinding{{ + Kind: "probe_error", + NormalizedPath: "probe-row", + ObjectIDs: []string{"probe-object"}, + }}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("delete selected probe-error record: %v", err) + } + if len(response.DeletedRecordIDs) != 1 || response.DeletedRecordIDs[0] != "probe-object" { + t.Fatalf("unexpected cleanup response: %+v", response) + } + if len(backend.deletedIDs) != 1 || backend.deletedIDs[0] != "probe-object" { + t.Fatalf("expected Syfon deletion only, got %+v", backend) + } +} diff --git a/internal/git/storage_audit_list_confirmation_test.go b/internal/git/storage_audit_list_confirmation_test.go index 0403954..07493aa 100644 --- a/internal/git/storage_audit_list_confirmation_test.go +++ b/internal/git/storage_audit_list_confirmation_test.go @@ -8,99 +8,64 @@ import ( gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" ) -func TestBuildStorageChainAuditRequiresMetadataToConfirmListMiss(t *testing.T) { +func TestBuildStorageChainAuditListMissDoesNotTriggerMetadataConfirmation(t *testing.T) { const ( repoPath = "data/file.bin" checksum = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" canonicalURL = "s3://bucket/root/data/file.bin" ) - tests := []struct { - name string - metadataResult gintegrationsyfon.BulkStorageProbeResult - wantMissing int - wantProbeError int - }{ - { - name: "metadata confirms missing", - metadataResult: gintegrationsyfon.BulkStorageProbeResult{ + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + repoPath: lfsPointer(checksum, 100), + }) + listKey := storageListValidationRequestKey(canonicalURL, 100, "file.bin") + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{{ + ObjectID: "object-id", + Name: "file.bin", + Checksum: checksum, + Organization: "org", + Project: "project", + Size: 100, + AccessURLs: []string{canonicalURL}, + }}, + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bucket", + Organization: "org", + ProjectID: "project", + Path: "s3://bucket/root", + }}, + listProbeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + listKey: { + ID: listKey, + ObjectURL: canonicalURL, Status: "not_found", ErrorKind: "object_not_found", }, - wantMissing: 1, - }, - { - name: "metadata failure leaves absence unknown", - metadataResult: gintegrationsyfon.BulkStorageProbeResult{ - Status: "error", - ErrorKind: "storage_error", - Error: "metadata lookup failed", - }, - wantProbeError: 1, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ - repoPath: lfsPointer(checksum, 100), - }) - listKey := storageListValidationRequestKey(canonicalURL, 100, "file.bin") - metadataKey := storageProbeRequestKey(canonicalURL, 100, checksum) - metadataResult := test.metadataResult - metadataResult.ID = metadataKey - metadataResult.ObjectURL = canonicalURL - backend := &fakeStorageAnalyticsBackend{ - projectRecords: []gintegrationsyfon.ProjectRecord{{ - ObjectID: "object-id", - Name: "file.bin", - Checksum: checksum, - Organization: "org", - Project: "project", - Size: 100, - AccessURLs: []string{canonicalURL}, - }}, - projectScopes: []domain.StorageBucketScope{{ - Bucket: "bucket", - Organization: "org", - ProjectID: "project", - Path: "s3://bucket/root", - }}, - listProbeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ - listKey: { - ID: listKey, - ObjectURL: canonicalURL, - Status: "not_found", - ErrorKind: "object_not_found", - }, - }, - probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ - metadataKey: metadataResult, - }, - } - chain, err := NewStorageAnalyticsService(backend).BuildStorageChainAuditWithOptions( - context.Background(), - "Bearer token", - "org", - "project", - refName, - "", - mirrorPath, - repo, - hash, - StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, - ) - if err != nil { - t.Fatalf("build storage chain audit: %v", err) - } - if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != test.wantMissing { - t.Fatalf("unexpected missing count %d in summary %+v", got, chain.Summary) - } - if got := chain.Summary.CountsByKind["probe_error"]; got != test.wantProbeError { - t.Fatalf("unexpected probe-error count %d in summary %+v", got, chain.Summary) - } - if backend.listProbeCalls != 1 || backend.probeCalls != 1 { - t.Fatalf("expected one LIST candidate check and one metadata confirmation, got list=%d metadata=%d", backend.listProbeCalls, backend.probeCalls) - } - }) + chain, err := NewStorageAnalyticsService(backend).BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "project", + refName, + "", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build storage chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 0 { + t.Fatalf("LIST misses must not become missing-object findings, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["probe_error"]; got != 1 { + t.Fatalf("expected a LIST miss to remain a probe error, got summary %+v", chain.Summary) + } + if backend.listProbeCalls != 1 || backend.probeCalls != 0 { + t.Fatalf("expected one exact LIST and no metadata confirmation, got list=%d metadata=%d", backend.listProbeCalls, backend.probeCalls) } } From 8f5281c35c7bf0d0c717b9ab2eabd5b6009cfa36 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 10 Jul 2026 09:01:02 -0700 Subject: [PATCH 34/36] add support for viewing bucket only files --- internal/git/storage_analytics.go | 6 +- internal/git/storage_analytics_pipeline.go | 30 +++-- internal/git/storage_analytics_test.go | 110 +++++++++++++++++ internal/git/storage_audit_registration.go | 113 +++++++++++++++--- .../git/storage_audit_registration_test.go | 43 ++++++- internal/storageaudit/evidence_test.go | 3 + internal/storageaudit/policy.go | 2 +- 7 files changed, 279 insertions(+), 28 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 52d4a66..becabed 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -676,7 +676,8 @@ func (service *StorageAnalyticsService) buildStorageChainAuditFresh(ctx context. } modelStart := time.Now() options.Timings.StageStart("model_build") - includeBucketOrigin := bucketMode != StorageChainBucketModeValidate && storageView.bucketInventoryAvailable + includeBucketOrigin := storageView.bucketInventoryAvailable && + (bucketMode == StorageChainBucketModeItems || validationMode == StorageChainValidationModeList) model := buildStorageChainAuditModel(gitSubpath, inputs.inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL, inputs.scopes, organization, project, includeBucketOrigin) options.Timings.Record("model_build", time.Since(modelStart)) options.Timings.RecordMemory( @@ -1371,9 +1372,10 @@ func (service *StorageAnalyticsService) executeStorageCleanupApplyPlan(ctx conte } deletedBucketObjectURLs = uniqueStrings(deletedBucketObjectURLs) } - if len(plan.UpdateAccessMethods) > 0 || len(toDelete) > 0 { + if len(plan.UpdateAccessMethods) > 0 || len(toDelete) > 0 || len(toDeleteBucketObjects) > 0 { service.evictProjectJoinCache(organization, project) service.evictProjectAuditRecordCache(organization, project) + service.evictProjectChainInputCache(organization, project) } for _, objectID := range toDelete { success := true diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index b3662a4..f03202c 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -98,14 +98,6 @@ func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Conte bucketCh := make(chan bucketResult, 1) go func() { - if bucketMode == StorageChainBucketModeValidate && validationMode != StorageChainValidationModeList { - timings.Record("syfon_bucket_inventory_skipped", 0) - bucketCh <- bucketResult{ - bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, - bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, - } - return - } start := time.Now() timings.StageStart("repo_index") inventory, err := service.loadStorageChainInventory(ctx, ref, gitSubpath, mirrorPath, repo, hash) @@ -136,6 +128,14 @@ func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Conte scopeCh <- scopeResult{scopes: scopes, err: err} }() go func() { + if bucketMode == StorageChainBucketModeValidate && validationMode != StorageChainValidationModeList { + timings.Record("syfon_bucket_inventory_skipped", 0) + bucketCh <- bucketResult{ + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, + } + return + } start := time.Now() timings.StageStart("syfon_bucket_inventory") var bucketObjects []gintegrationsyfon.ProjectBucketObject @@ -553,6 +553,18 @@ func (service *StorageAnalyticsService) updateChainInputCache(cacheKey string, u service.chainInputCache[cacheKey] = state } +func (service *StorageAnalyticsService) evictProjectChainInputCache(organization string, project string) { + baseKey := service.projectChainInputCacheKey(organization, project) + prefix := baseKey + "::" + service.chainInputMu.Lock() + defer service.chainInputMu.Unlock() + for key := range service.chainInputCache { + if key == baseKey || strings.HasPrefix(key, prefix) { + delete(service.chainInputCache, key) + } + } +} + func cloneBucketInventory(bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { objects := append([]gintegrationsyfon.ProjectBucketObject(nil), bucketObjects...) lookup := make(map[string]gintegrationsyfon.ProjectBucketObject, len(bucketObjectsByURL)) @@ -1425,7 +1437,7 @@ func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccum } acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) case storageFindingBrokenAccessURL, storageFindingProbeError: - findings := buildChainRecordFindings("probe_error", record, gitPaths, bucketMatches, "Storage probes could not confirm this mapped object; Gecko does not treat it as missing or offer cleanup.") + findings := buildChainRecordFindings("probe_error", record, gitPaths, bucketMatches, "Storage probes could not confirm this mapped object. Gecko does not treat it as missing; after review, you may delete the Syfon record.") acc.findings = append(acc.findings, findings...) acc.addCount("probe_error", chainPathCount(gitPaths)) case storageFindingNone: diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 33a7013..3c7bfe5 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -1339,6 +1339,61 @@ func TestApplyStorageCleanupRepairMatrix(t *testing.T) { } } +func TestApplyStorageCleanupBucketOnlyDeletionEvictsChainInputCache(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + } + service := NewStorageAnalyticsService(backend) + cacheKey := service.projectChainInputCacheKey("org", "proj") + service.chainInputCache[cacheKey] = cachedChainInputState{ + expiresAt: time.Now().Add(time.Minute), + projectScopes: backend.projectScopes, + } + service.chainInputCache[cacheKey+"::bucket-validation-inventory::"] = cachedChainInputState{ + expiresAt: time.Now().Add(time.Minute), + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: "s3://bucket/only", + Bucket: "bucket", + Key: "only", + }}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{ + "s3://bucket/only": {ObjectURL: "s3://bucket/only", Bucket: "bucket", Key: "only"}, + }, + } + service.chainInputCache["org/other::bucket-validation-inventory::"] = cachedChainInputState{ + expiresAt: time.Now().Add(time.Minute), + } + + _, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + nil, + nil, + []GitStorageCleanupApplyFinding{applyFinding("bucket_only_object", "s3://bucket/only", nil, []string{"s3://bucket/only"})}, + false, + false, + true, + false, + false, + ) + if err != nil { + t.Fatalf("apply bucket-only cleanup: %v", err) + } + if _, ok := service.chainInputCache[cacheKey]; ok { + t.Fatalf("expected project chain input cache to be evicted, got %+v", service.chainInputCache) + } + if _, ok := service.chainInputCache[cacheKey+"::bucket-validation-inventory::"]; ok { + t.Fatalf("expected project bucket inventory cache to be evicted, got %+v", service.chainInputCache) + } + if _, ok := service.chainInputCache["org/other::bucket-validation-inventory::"]; !ok { + t.Fatalf("expected another project's cache entry to remain, got %+v", service.chainInputCache) + } +} + func TestApplyStorageCleanupPrunesBrokenAccessMethods(t *testing.T) { tests := []struct { name string @@ -1897,6 +1952,58 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { } } +func TestBuildStorageChainAuditDefaultListExposesBucketOnlyObjects(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-present", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + AccessURLs: []string{"s3://bucket/present.txt"}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present.txt", Bucket: "bucket", Key: "present.txt", SizeBytes: 100}, + {ObjectURL: "s3://bucket/loose.txt", Bucket: "bucket", Key: "loose.txt", SizeBytes: 25}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build default chain audit: %v", err) + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected default list audit to load bucket inventory, got %d calls", backend.listProjectBucketInventoryCalls) + } + if got := chain.Summary.CountsByKind["bucket_only_object"]; got != 1 { + t.Fatalf("expected one bucket-only finding, got summary %+v", chain.Summary) + } + finding := assertHasChainFinding(t, chain.Findings, "bucket_only_object", "s3://bucket/loose.txt") + if finding.DefaultAction != storageActionDeleteBucketObject || len(finding.ObjectIDs) != 0 { + t.Fatalf("expected bucket-only deletion evidence without Syfon IDs, got %+v", finding) + } +} + func TestBuildStorageChainAuditMetadataModeUsesMetadataValidation(t *testing.T) { repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), @@ -1953,6 +2060,9 @@ func TestBuildStorageChainAuditMetadataModeUsesMetadataValidation(t *testing.T) if chain.Summary.ValidationMode != StorageChainValidationModeMetadata { t.Fatalf("expected metadata validation mode, got %+v", chain.Summary) } + if got := chain.Summary.CountsByKind["bucket_only_object"]; got != 0 { + t.Fatalf("metadata validation must not claim bucket-only coverage, got summary %+v", chain.Summary) + } finding := assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/a.txt") if finding.Evidence == nil || !contains(finding.Evidence.StorageOperations, StorageChainValidationModeMetadata) { t.Fatalf("expected metadata operation evidence, got %+v", finding.Evidence) diff --git a/internal/git/storage_audit_registration.go b/internal/git/storage_audit_registration.go index 4cff8fc..8d32755 100644 --- a/internal/git/storage_audit_registration.go +++ b/internal/git/storage_audit_registration.go @@ -21,6 +21,10 @@ type gitOnlyRegistrationCandidate struct { resultIndexes []int } +type gitOnlyBucketCandidate struct { + url string +} + // RegisterGitOnlySyfonRecords restores only missing Syfon records. It always // re-reads the Git LFS pointer, confirms the current scoped bucket object, and // rejects size drift before sending a DRS registration request to Syfon. @@ -72,6 +76,10 @@ func (service *StorageAnalyticsService) RegisterGitOnlySyfonRecords(ctx context. if err != nil { return nil, fmt.Errorf("load project storage scopes before registration: %w", err) } + bucketObjects, bucketObjectsByURL, err := service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + if err != nil { + return nil, fmt.Errorf("load project bucket inventory before registration: %w", err) + } probes := make([]gintegrationsyfon.BulkStorageProbeItem, 0, len(results)) probeResultIndexes := make(map[string]int, len(results)) for index := range results { @@ -84,20 +92,22 @@ func (service *StorageAnalyticsService) RegisterGitOnlySyfonRecords(ctx context. result.Reason = "a Syfon record for this SHA-256 already exists" continue } - urls := projectScopeRepoPathObjectURLs([]string{result.NormalizedPath}, scopes, organization, project) - if len(urls) != 1 { + candidates := gitOnlyBucketCandidates(result.NormalizedPath, result.Checksum, scopes, organization, project, bucketObjects, bucketObjectsByURL) + if len(candidates) == 0 { result.Status = "skipped" - result.Reason = gitOnlyRegistrationScopeReason(urls, scopes, organization, project) + result.Reason = "no bucket object matched the audited Git path or SHA-256 in the configured project mappings" continue } - probeID := fmt.Sprintf("git-only-register-%d", index) size := result.GitSizeBytes - probes = append(probes, gintegrationsyfon.BulkStorageProbeItem{ - ID: probeID, - ObjectURL: urls[0], - ExpectedSizeBytes: &size, - }) - probeResultIndexes[probeID] = index + for candidateIndex, candidate := range candidates { + probeID := fmt.Sprintf("git-only-register-%d-%d", index, candidateIndex) + probes = append(probes, gintegrationsyfon.BulkStorageProbeItem{ + ID: probeID, + ObjectURL: candidate.url, + ExpectedSizeBytes: &size, + }) + probeResultIndexes[probeID] = index + } } if len(probes) > 0 { observed, err := service.storage.BulkProbeStorageObjects(ctx, authorizationHeader, probes) @@ -111,6 +121,9 @@ func (service *StorageAnalyticsService) RegisterGitOnlySyfonRecords(ctx context. for _, item := range probes { index := probeResultIndexes[item.ID] result := &results[index] + if result.Status == "eligible" { + continue + } probe, ok := byProbeID[item.ID] if !ok { result.Status = "skipped" @@ -123,16 +136,19 @@ func (service *StorageAnalyticsService) RegisterGitOnlySyfonRecords(ctx context. } result.BucketSizeBytes = derefInt64(probe.SizeBytes) if !probe.Exists || strings.TrimSpace(probe.Status) != "present" { - result.Status = "skipped" - result.Reason = "mapped bucket object is not currently present" continue } if probe.SizeBytes == nil || *probe.SizeBytes != result.GitSizeBytes { - result.Status = "skipped" - result.Reason = fmt.Sprintf("Git LFS size is %d B but bucket size is %d B", result.GitSizeBytes, result.BucketSizeBytes) continue } result.Status = "eligible" + result.Reason = "bucket object matched the audited Git path or SHA-256 and passed size verification" + } + for index := range results { + if results[index].Status == "" { + results[index].Status = "skipped" + results[index].Reason = fmt.Sprintf("Git LFS size is %d B but no matched bucket object is present at that size", results[index].GitSizeBytes) + } } } candidates := buildGitOnlyRegistrationCandidates(results) @@ -157,7 +173,7 @@ func (service *StorageAnalyticsService) RegisterGitOnlySyfonRecords(ctx context. for _, resultIndex := range candidate.resultIndexes { results[resultIndex].Status = "created" results[resultIndex].ObjectID = objectID - results[resultIndex].Reason = "Syfon record created after Git pointer and bucket size verification; bucket SHA-256 was not available for verification" + results[resultIndex].Reason = "Syfon record created after Git pointer and bucket object verification" } } service.evictProjectJoinCache(organization, project) @@ -166,6 +182,73 @@ func (service *StorageAnalyticsService) RegisterGitOnlySyfonRecords(ctx context. return &GitOnlySyfonRegistrationResponse{GitRevision: hash.String(), Results: results}, nil } +func gitOnlyBucketCandidates(repoPath string, checksum string, scopes []domain.StorageBucketScope, organization string, project string, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []gitOnlyBucketCandidate { + pathURLs := projectScopeRepoPathObjectURLs([]string{repoPath}, scopes, organization, project) + checksumURLs := projectScopeRepoPathObjectURLs([]string{checksum}, scopes, organization, project) + auditCandidates := make([]gitOnlyBucketCandidate, 0) + seen := make(map[string]struct{}) + addAuditCandidate := func(objectURL string) { + canonical := canonicalStorageURL("", "", objectURL) + if canonical == "" { + return + } + if _, ok := bucketObjectsByURL[canonical]; !ok { + return + } + if _, ok := seen[canonical]; ok { + return + } + seen[canonical] = struct{}{} + auditCandidates = append(auditCandidates, gitOnlyBucketCandidate{url: canonical}) + } + for _, objectURL := range pathURLs { + addAuditCandidate(objectURL) + } + for _, objectURL := range checksumURLs { + addAuditCandidate(objectURL) + } + for _, object := range bucketObjects { + objectURL := canonicalStorageURL(object.Bucket, object.Key, object.ObjectURL) + if objectURL == "" || normalizeAnalyticsChecksum(object.MetaSHA256) != checksum || !objectBelongsToProjectScope(objectURL, scopes, organization, project) { + continue + } + addAuditCandidate(objectURL) + } + if len(auditCandidates) > 0 { + return auditCandidates + } + + // Inventory may be unavailable for a newly uploaded object. Keep the + // recheck conservative by probing both supported storage layouts rather + // than assuming only the Git-path layout. + generated := make([]gitOnlyBucketCandidate, 0, len(pathURLs)+len(checksumURLs)) + for _, objectURL := range append(pathURLs, checksumURLs...) { + canonical := canonicalStorageURL("", "", objectURL) + if canonical == "" { + continue + } + if _, ok := seen[canonical]; ok { + continue + } + seen[canonical] = struct{}{} + generated = append(generated, gitOnlyBucketCandidate{url: canonical}) + } + return generated +} + +func objectBelongsToProjectScope(objectURL string, scopes []domain.StorageBucketScope, organization string, project string) bool { + bucket, _, ok := parseStorageURL(objectURL) + if !ok { + return false + } + for _, scope := range scopes { + if storageScopeApplies(scope, organization, project) && strings.EqualFold(strings.TrimSpace(scope.Bucket), bucket) { + return true + } + } + return false +} + func normalizedGitOnlyRegistrationPaths(paths []string) []string { out := make([]string, 0, len(paths)) seen := make(map[string]struct{}, len(paths)) diff --git a/internal/git/storage_audit_registration_test.go b/internal/git/storage_audit_registration_test.go index 408c03b..dc126a4 100644 --- a/internal/git/storage_audit_registration_test.go +++ b/internal/git/storage_audit_registration_test.go @@ -70,8 +70,15 @@ func TestRegisterGitOnlySyfonRecordsRefusesBucketSizeMismatch(t *testing.T) { actualSize := int64(99) backend := &fakeStorageAnalyticsBackend{ projectScopes: []domain.StorageBucketScope{{Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket/root"}}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: objectURL, + Bucket: "bucket", + Key: "root/data/a.bin", + Path: "data/a.bin", + SizeBytes: 99, + }}, probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ - "git-only-register-0": {ID: "git-only-register-0", ObjectURL: objectURL, Exists: true, Status: "present", SizeBytes: &actualSize}, + "git-only-register-0-0": {ID: "git-only-register-0-0", ObjectURL: objectURL, Exists: true, Status: "present", SizeBytes: &actualSize}, }, } service := NewStorageAnalyticsService(backend) @@ -86,3 +93,37 @@ func TestRegisterGitOnlySyfonRecordsRefusesBucketSizeMismatch(t *testing.T) { t.Fatalf("expected size mismatch to block registration, got response=%+v registrations=%+v", response, backend.registeredObjects) } } + +func TestRegisterGitOnlySyfonRecordsFindsChecksumKeyedBucketObjectFromInventory(t *testing.T) { + checksum := "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.bin": lfsPointer(checksum, 100), + }) + objectURL := "s3://bucket/root/" + checksum + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{{Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket/root"}}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: objectURL, + Bucket: "bucket", + Key: "root/" + checksum, + Path: checksum, + SizeBytes: 100, + MetaSHA256: checksum, + }}, + } + service := NewStorageAnalyticsService(backend) + + response, err := service.RegisterGitOnlySyfonRecords(context.Background(), "Bearer token", "org", "proj", refName, mirrorPath, repo, hash, GitOnlySyfonRegistrationRequest{ + ExpectedGitRevision: hash.String(), + RepoPaths: []string{"data/a.bin"}, + }) + if err != nil { + t.Fatalf("register missing Syfon record: %v", err) + } + if len(response.Results) != 1 || response.Results[0].Status != "created" || response.Results[0].BucketObjectURL != objectURL { + t.Fatalf("expected checksum-keyed inventory object to be registered, got %+v", response) + } + if len(backend.probeItems) != 1 || backend.probeItems[0].ObjectURL != objectURL { + t.Fatalf("expected registration to probe the inventory-resolved checksum object, got %+v", backend.probeItems) + } +} diff --git a/internal/storageaudit/evidence_test.go b/internal/storageaudit/evidence_test.go index b5c4536..8b13246 100644 --- a/internal/storageaudit/evidence_test.go +++ b/internal/storageaudit/evidence_test.go @@ -61,6 +61,9 @@ func TestAssessLetsPresentLocatorWinOverMissingAlias(t *testing.T) { } func TestAutomaticRepairRequiresVerifiedMissingEvidence(t *testing.T) { + if !AllowsAutomaticRepair("bucket_only_object", EvidenceVerified) { + t.Fatal("verified bucket-only inventory evidence should allow bucket deletion") + } if AllowsAutomaticRepair("syfon_git_no_bucket", EvidenceUnknown) { t.Fatal("unknown evidence must not allow automatic repair") } diff --git a/internal/storageaudit/policy.go b/internal/storageaudit/policy.go index a2da529..6a08d76 100644 --- a/internal/storageaudit/policy.go +++ b/internal/storageaudit/policy.go @@ -7,7 +7,7 @@ func AllowsAutomaticRepair(kind string, evidenceStatus EvidenceStatus) bool { return false } switch strings.TrimSpace(kind) { - case "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object": + case "bucket_only_object", "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object": return true default: return false From 9f99a7e294b93ecd6eb8353fb17c9fe38f5dd07b Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 10 Jul 2026 13:02:37 -0700 Subject: [PATCH 35/36] add cache for successful syfon bucket LIST traversals --- docs/storage-chain-audit.md | 163 ++++++++++++++++++ internal/git/storage_analytics.go | 2 + internal/git/storage_analytics_pipeline.go | 54 ++++-- internal/git/storage_analytics_test.go | 69 ++++++-- internal/git/storage_chain_audit_cache.go | 128 +++++++++++++- .../git/storage_chain_audit_cache_test.go | 9 + 6 files changed, 390 insertions(+), 35 deletions(-) create mode 100644 docs/storage-chain-audit.md diff --git a/docs/storage-chain-audit.md b/docs/storage-chain-audit.md new file mode 100644 index 0000000..a2a08b2 --- /dev/null +++ b/docs/storage-chain-audit.md @@ -0,0 +1,163 @@ +# Gecko storage-chain audit: implementation guide + +The storage-chain audit is a read-only cross-check of Git LFS pointers, Syfon project records, project bucket scope mappings, and Syfon bucket inventory. The handler is registered as `POST /api/v1/projects/:project_id/repair/storage-chain/audit` in [`internal/server/http/git/register.go`](../internal/server/http/git/register.go#L54). Repair endpoints are separate. + +## Code map + +| Responsibility | Owner | +| --- | --- | +| HTTP request parsing, defaults, and error mapping | [`handleGitProjectStorageChainAuditPOST`](../internal/server/http/git/storage_analytics.go#L214-L326) | +| Option normalization and cache selection | [`BuildStorageChainAuditWithOptions`](../internal/git/storage_analytics.go#L481-L539) | +| Root audit-response cache and refresh coalescing | [`buildStorageChainAuditWithResponseCache`](../internal/git/storage_analytics.go#L541-L607), [`coalesceStorageChainAuditRefresh`](../internal/git/storage_chain_audit_cache.go#L49-L76) | +| Fresh audit assembly | [`buildStorageChainAuditFresh`](../internal/git/storage_analytics.go#L646-L723) | +| Parallel Git, record, scope, and bucket input loading | [`loadStorageChainInputs`](../internal/git/storage_analytics_pipeline.go#L75-L180) | +| Syfon record metrics validator and cache | [`loadCachedProjectAuditRecords`](../internal/git/storage_analytics_pipeline.go#L361-L431) | +| Durable bucket inventory cache and stale fallback | [`loadCachedProjectBucketValidationInventory`](../internal/git/storage_analytics_pipeline.go#L519-L566) | +| Validation-mode decision tree | [`buildStorageChainView`](../internal/git/storage_analytics_pipeline.go#L907-L996) | +| Finding model | [`buildStorageChainAuditModel`](../internal/git/storage_analytics_pipeline.go#L1265-L1277) | +| Modes, defaults, and timing implementation | [`storage_analytics_timing.go`](../internal/git/storage_analytics_timing.go#L11-L172) | + +## Request contract + +The handler resolves auth, project context, Git mirror, ref, and commit, then maps the request to `StorageChainAuditOptions` at [`storage_analytics.go`](../internal/server/http/git/storage_analytics.go#L230-L306). + +| Field | Meaning | Default | +| --- | --- | --- | +| `ref` | Git ref to inspect | Project context ref | +| `git_subpath` | Restricts Git findings to one repository path | Root | +| `bucket_inventory_mode` | `validate` uses the project validation inventory; `items` uses explicit bucket items | `validate` | +| `validation_mode` | `list`, `metadata`, or `inventory` | `list` | +| `probe_mode` | Legacy shorthand: `full` selects metadata and `inventory_only` selects inventory when validation mode is omitted | `full` | +| `finding_kind`, `finding_limit` | Response projection only; cache stores the complete model | All / service default | +| `refresh`, `force_audit_refresh`, `force_bucket_inventory_refresh` | Each sets `ForceAuditRefresh` | `false` | + +`bucket_path_prefix` is valid only with `bucket_inventory_mode=items` ([handler check](../internal/server/http/git/storage_analytics.go#L275-L280)). + +## Request-to-response sequence + +```mermaid +sequenceDiagram + actor User + participant UI as Frontend + participant Handler as Gecko handler + participant Audit as StorageAnalyticsService + participant ResponseCache as Audit response cache + participant Git as Git mirror + participant Syfon + participant BucketCache as Bucket inventory cache + participant S3 as Object provider + + User->>UI: Request storage audit + UI->>Handler: POST repair storage-chain audit + Handler->>Handler: Resolve project, mirror, ref, auth, options + Handler->>Audit: BuildStorageChainAuditWithOptions + Audit->>ResponseCache: Lookup eligible root response + ResponseCache-->>Audit: Cached response or miss + Audit->>Git: Build Git LFS inventory on miss + Audit->>Syfon: Load project records and scopes + Audit->>BucketCache: Read inventory by org, project, prefix + BucketCache-->>Audit: Fresh inventory or stale candidate + Audit->>Syfon: Refresh inventory when stale or absent + Syfon->>S3: Paginated LIST and terminal replay + S3-->>Syfon: Inventory or inconsistent-list error + Syfon-->>Audit: Inventory or error + Audit->>BucketCache: Store successful non-empty inventory + Audit->>Audit: Validate records and build findings + Audit->>ResponseCache: Store complete root response + Audit-->>Handler: Findings, groups, summary, timings + Handler-->>UI: JSON response or mapped error +``` + +The middle arrows are conditional. A fresh response cache skips all input work. A fresh bucket cache skips only the expensive Syfon bucket LIST. + +## Cache hierarchy + +| Layer | Scope | Forced-refresh behavior | +| --- | --- | --- | +| Audit response cache | org, project, ref, commit hash, Syfon revision, modes, root path | Bypasses the read, rebuilds the full model, then replaces the response | +| Root response projection | Root cached response projected to a Git subpath | Disabled by hard refresh; it reads Git inventory for the selected subpath when used | +| Project record cache | org/project plus Syfon metrics validator | Reloads when record count, latest update, or revision changes | +| Scope and item cache | Short in-process project cache | Normal in-process reuse only | +| Validation inventory cache | org/project/validation prefix | Fresh for 10 minutes; previous good value is retained for 24 hours | + +The response cache is eligible only for root Git subpath with no bucket path prefix ([eligibility](../internal/git/storage_analytics.go#L504-L510)). Concurrent root refreshes join one in-flight build, preventing overlapping requests from replacing a completed response ([coalescing](../internal/git/storage_chain_audit_cache.go#L49-L76)). + +### Validation inventory cache sequence + +```mermaid +sequenceDiagram + participant Audit as Gecko audit + participant Cache as Validation inventory cache + participant Syfon + participant S3 as Object provider + + Audit->>Cache: Get org project validation-prefix key + Cache-->>Audit: Fresh value under 10 minutes + Note over Audit: Fresh value: do not call Syfon. + Audit->>Syfon: If stale or absent, list project bucket inventory + Syfon->>S3: Traverse pages and replay terminal page + S3-->>Syfon: Complete consistent inventory + Syfon-->>Audit: Successful non-empty inventory + Audit->>Cache: Replace value and set 24-hour retention + Note over Syfon,Audit: Terminal replay mismatch returns an error. + Cache-->>Audit: If stale value exists, return last known-good inventory + Note over Audit: With no cached value, propagate the Syfon error. +``` + +Only non-empty successful inventories replace the cache ([store guard](../internal/git/storage_analytics_pipeline.go#L553-L560)). Syfon terminal-page disagreement is an error, never a partial successful response. Gecko returns stale inventory only when a previous good value exists ([fallback](../internal/git/storage_analytics_pipeline.go#L544-L551)). A cold-cache inventory error remains an HTTP error. + +## Fresh audit execution + +1. `loadStorageChainInputs` starts Git inventory, Syfon records, Syfon scopes, and bucket data concurrently ([lines 100-156](../internal/git/storage_analytics_pipeline.go#L100-L156)). +2. Git, record, and scope errors fail the request. Bucket errors are carried as `bucketInventoryErr`, so validation mode decides whether they are fatal ([lines 158-179](../internal/git/storage_analytics_pipeline.go#L158-L179)). +3. `buildStorageChainView` canonicalizes Syfon record URLs through bucket scopes and attaches list or bulk-probe evidence. +4. `buildStorageChainAuditModel` indexes canonical bucket URLs, Git checksums, and Syfon records, then runs bucket-origin, Syfon-origin, and Git-origin finding passes. +5. Gecko filters and limits only after constructing the complete model. `Groups` use all filtered rows, while `Findings` can be truncated ([response shaping](../internal/git/storage_analytics.go#L703-L722)). + +### Validation modes + +| Mode | Bucket LIST | Additional Syfon work | Bucket view | +| --- | --- | --- | --- | +| `validate` + `list` | Yes, through durable inventory cache | None | Actual listing; inventory misses are unverifiable | +| `validate` + `metadata` | Skipped | Bulk metadata validation | Synthesized from present probes | +| `validate` + `inventory` | Skipped | None | No bucket list | +| `items` + `list` or `inventory` | Yes, using requested prefix | None | Explicit bucket items | +| `items` + `metadata` | Yes | Probe only records needing confirmation | Listed items plus probe evidence | + +The branch is at [`buildStorageChainView`](../internal/git/storage_analytics_pipeline.go#L917-L995). Gecko enables bucket-origin findings only when it has trustworthy list inventory ([decision](../internal/git/storage_analytics.go#L679-L683)). + +## Findings and error semantics + +The summary keys are initialized in [`newChainSummary`](../internal/git/storage_analytics_pipeline.go#L193-L210). + +| Finding kind | Meaning | +| --- | --- | +| `bucket_only_object` | Bucket object has no matching Syfon record or equivalent record identity | +| `bucket_syfon_no_git` | Syfon record resolves to storage but no Git pointer joins it | +| `syfon_missing_bucket_object` | Record storage cannot be resolved from trustworthy list/probe evidence | +| `syfon_git_no_bucket` | Git and Syfon join but storage evidence is missing | +| `git_only_no_syfon` | Git LFS pointer has no Syfon record | +| `git_syfon_metadata_mismatch` | Joined Git, Syfon, and storage metadata conflict | +| `probe_error` | Verification is inconclusive; it is not a missing-object assertion | + +For Weka failures, a Syfon traversal error without cached inventory fails the audit. A stale-cache fallback succeeds from older known-good inventory; it must not generate `probe_error` rows from a false empty list. + +## Observability and tests + +| Log | Meaning | +| --- | --- | +| `syfon_project_bucket_inventory_cache_hit` | Fresh inventory reused | +| `syfon_project_bucket_inventory_cache_store` | Successful non-empty inventory replaced cache | +| `syfon_project_bucket_inventory_cache_stale_fallback` | Syfon refresh failed; stale good inventory used | +| `syfon_project_bucket_inventory_cache_error` | Cache backend failed; Gecko continues direct behavior | +| `storage_chain_audit_request_error` | Handler is returning a mapped audit error | + +Stage timings use `storage_chain_audit_stage_start` and `storage_chain_audit_stage_done`; memory snapshots include Git file, Syfon record, and bucket object counts ([timing implementation](../internal/git/storage_analytics_timing.go#L46-L93)). + +Run focused coverage from the Gecko repository: + +```bash +GOCACHE=/private/tmp/gecko-gocache go test ./internal/git -run 'TestBuildStorageChainAuditForceRefreshBypassesResponseCache|TestProjectBucketInventoryUsesStaleCacheWhenRefreshFails|TestProjectBucketInventoryCacheRetention' -count=1 +``` + +The relevant tests are [`internal/git/storage_analytics_test.go`](../internal/git/storage_analytics_test.go#L2356-L2451) and [`internal/git/storage_chain_audit_cache_test.go`](../internal/git/storage_chain_audit_cache_test.go). diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index becabed..80de625 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -158,6 +158,7 @@ type StorageAnalyticsService struct { chainAuditRefreshWork map[string]*inflightStorageChainAuditRefresh chainAuditResponseCache storageChainAuditResponseCache exactProjectJoinCache storageExactProjectJoinCache + projectBucketCache projectBucketInventoryCache } type StorageFolderTimings struct { @@ -193,6 +194,7 @@ func (service *StorageAnalyticsService) EnableStorageChainAuditResponseCacheFrom } service.chainAuditResponseCache = NewStorageChainAuditResponseCacheFromEnv() service.exactProjectJoinCache = NewStorageExactProjectJoinCacheFromEnv() + service.projectBucketCache = NewProjectBucketInventoryCacheFromEnv() } type RepoInventoryFile struct { diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index f03202c..9aff13b 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -142,11 +142,10 @@ func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Conte var bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject var err error if bucketMode == StorageChainBucketModeValidate { - if forceRefresh { - bucketObjects, bucketObjectsByURL, err = service.loadProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") - } else { - bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") - } + // Full bucket LIST calls are expensive on some providers. Reuse the + // validated project inventory during the cooldown instead of issuing a + // second Syfon scan for every audit or Git ref. + bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") } else { bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) } @@ -519,6 +518,22 @@ func (service *StorageAnalyticsService) loadCachedProjectBucketInventory(ctx con func (service *StorageAnalyticsService) loadCachedProjectBucketValidationInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { cacheKey := service.projectChainInputCacheKey(organization, project) + "::bucket-validation-inventory::" + normalizeRepoSubpath(bucketPathPrefix) + var stale cachedProjectBucketInventory + var hasStale bool + if cache := service.projectBucketCache; cache != nil { + redisKey := projectBucketInventoryCacheKey(organization, project, bucketPathPrefix) + cached, ok, err := cache.Get(ctx, redisKey) + if err != nil { + log.Printf("syfon_project_bucket_inventory_cache_error org=%s project=%s path_prefix=%q source=%s operation=get error=%q", organization, project, bucketPathPrefix, cache.Source(), err.Error()) + } else if ok && time.Since(cached.CachedAt) < projectBucketInventoryRefreshInterval { + objects, lookup := buildBucketObjectLookup(cached.Objects) + log.Printf("syfon_project_bucket_inventory_cache_hit org=%s project=%s path_prefix=%q source=%s object_count=%d age_seconds=%d", organization, project, bucketPathPrefix, cache.Source(), len(objects), int64(time.Since(cached.CachedAt).Seconds())) + return objects, lookup, nil + } else if ok { + stale = cached + hasStale = true + } + } service.chainInputMu.RLock() cached, ok := service.chainInputCache[cacheKey] service.chainInputMu.RUnlock() @@ -528,8 +543,22 @@ func (service *StorageAnalyticsService) loadCachedProjectBucketValidationInvento } bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) if err != nil { + if hasStale { + objects, lookup := buildBucketObjectLookup(stale.Objects) + log.Printf("syfon_project_bucket_inventory_cache_stale_fallback org=%s project=%s path_prefix=%q source=%s object_count=%d age_seconds=%d refresh_error=%q", organization, project, bucketPathPrefix, service.projectBucketCache.Source(), len(objects), int64(time.Since(stale.CachedAt).Seconds()), err.Error()) + return objects, lookup, nil + } return nil, nil, err } + if cache := service.projectBucketCache; cache != nil && len(bucketObjects) > 0 { + redisKey := projectBucketInventoryCacheKey(organization, project, bucketPathPrefix) + value := cachedProjectBucketInventory{CachedAt: time.Now(), Objects: append([]gintegrationsyfon.ProjectBucketObject(nil), bucketObjects...)} + if err := cache.Set(ctx, redisKey, value, projectBucketInventoryStaleTTL); err != nil { + log.Printf("syfon_project_bucket_inventory_cache_error org=%s project=%s path_prefix=%q source=%s operation=set error=%q", organization, project, bucketPathPrefix, cache.Source(), err.Error()) + } else { + log.Printf("syfon_project_bucket_inventory_cache_store org=%s project=%s path_prefix=%q source=%s object_count=%d refresh_interval_seconds=%d stale_ttl_seconds=%d", organization, project, bucketPathPrefix, cache.Source(), len(bucketObjects), int64(projectBucketInventoryRefreshInterval.Seconds()), int64(projectBucketInventoryStaleTTL.Seconds())) + } + } service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { state.bucketObjects, state.bucketObjectsByURL = cloneBucketInventory(bucketObjects, bucketObjectsByURL) }) @@ -906,20 +935,7 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex } timings.Record("inventory_list_validation", time.Since(validateStart)) - candidateStart := time.Now() - candidates := selectInventoryMissRecordSet(probedRecordSet) - timings.Record("exact_list_candidate_selection", time.Since(candidateStart)) - if candidates != nil { - probeStart := time.Now() - exactRecordSet, probeErr := service.attachProjectStorageListValidations(ctx, authorizationHeader, candidates) - timings.Record("targeted_exact_list_validation", time.Since(probeStart)) - if probeErr != nil { - return nil, probeErr - } - probedRecordSet = mergeRecordSetProbes(probedRecordSet, exactRecordSet) - view.bucketObjects, view.bucketObjectsByURL = mergeBucketInventoryWithPresentProbes(view.bucketObjects, view.bucketObjectsByURL, exactRecordSet) - } - timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL), "exact_probe_records", countRecordSet(candidates)) + timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL), "exact_probe_records", 0) view.recordsByChecksum = probedRecordSet.recordsByChecksum view.allProjectRecords = probedRecordSet.allProjectRecords return view, nil diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go index 3c7bfe5..e06b9ad 100644 --- a/internal/git/storage_analytics_test.go +++ b/internal/git/storage_analytics_test.go @@ -1919,11 +1919,8 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { if backend.probeCalls != 0 { t.Fatalf("expected validate mode to skip HEAD probes, got %d calls", backend.probeCalls) } - if backend.listProbeCalls != 1 || len(backend.listProbeItems) != 1 { - t.Fatalf("expected one exact LIST for the inventory-missing candidate, got calls=%d items=%+v", backend.listProbeCalls, backend.listProbeItems) - } - if backend.listProbeItems[0].ObjectURL != "s3://bucket/syfon-only.txt" { - t.Fatalf("expected only the inventory-missing locator to be verified, got %+v", backend.listProbeItems) + if backend.listProbeCalls != 0 { + t.Fatalf("expected complete Syfon inventory to skip exact LIST fallback, got calls=%d items=%+v", backend.listProbeCalls, backend.listProbeItems) } if chain.Summary.BucketPathExists != nil || chain.Summary.BucketSummaryMode != "" { t.Fatalf("expected validate mode not to claim bucket prefix summary, got %+v", chain.Summary) @@ -1931,24 +1928,23 @@ func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { if chain.Summary.ValidationMode != StorageChainValidationModeList { t.Fatalf("expected LIST validation mode, got %+v", chain.Summary) } - if chain.Summary.BucketObjectCount != 3 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { - t.Fatalf("expected validate mode to merge the exact-present object without bucket-only findings, got summary %+v", chain.Summary) + if chain.Summary.BucketObjectCount != 2 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { + t.Fatalf("expected validate mode to preserve the complete inventory, got summary %+v", chain.Summary) } if chain.Summary.CountsByKind["bucket_syfon_git_complete"] != 2 { t.Fatalf("expected validate mode to count clean Syfon/Git/bucket join, got summary %+v", chain.Summary) } - if chain.Summary.CountsByKind["syfon_missing_bucket_object"] != 0 || chain.Summary.CountsByKind["bucket_syfon_no_git"] != 1 { - t.Fatalf("expected exact LIST presence to suppress the inventory false positive, got summary %+v", chain.Summary) + if chain.Summary.CountsByKind["probe_error"] != 1 || chain.Summary.CountsByKind["bucket_syfon_no_git"] != 0 { + t.Fatalf("expected complete-inventory miss to remain a probe error, got summary %+v", chain.Summary) } if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { t.Fatalf("expected basename mismatch not to be a broken bucket mapping, got summary %+v", chain.Summary) } assertNoChainFinding(t, chain.Findings, "bucket_only_object") assertNoChainFinding(t, chain.Findings, "syfon_broken_bucket_mapping") - assertNoChainFinding(t, chain.Findings, "syfon_missing_bucket_object") - finding := assertHasChainFinding(t, chain.Findings, "bucket_syfon_no_git", "s3://bucket/syfon-only.txt") - if finding.EvidenceStatus != "verified" || finding.Actionability != storageActionabilityInspectOnly || finding.DefaultAction != storageActionInspectEvidence { - t.Fatalf("expected verified but non-destructive Git-absence finding, got %+v", finding) + finding := assertHasChainFinding(t, chain.Findings, "probe_error", "s3://bucket/syfon-only.txt") + if finding.EvidenceStatus != "unknown" || finding.Actionability != storageActionabilityManualChoice || finding.DefaultAction != storageActionDeleteSyfonRecord { + t.Fatalf("expected inventory-miss probe error with a manual remediation choice, got %+v", finding) } } @@ -2374,6 +2370,7 @@ func TestBuildStorageChainAuditForceRefreshBypassesResponseCache(t *testing.T) { } service := NewStorageAnalyticsService(backend) service.chainAuditResponseCache = newMemoryStorageChainAuditResponseCache() + service.projectBucketCache = newMemoryProjectBucketInventoryCache() options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options); err != nil { @@ -2387,8 +2384,52 @@ func TestBuildStorageChainAuditForceRefreshBypassesResponseCache(t *testing.T) { if refreshed.Summary.AuditCacheHit { t.Fatalf("expected forced response to rebuild fresh, got %+v", refreshed.Summary) } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected forced refresh to reuse the bucket inventory during its cooldown, got %d calls", backend.listProjectBucketInventoryCalls) + } +} + +func TestProjectBucketInventoryUsesStaleCacheWhenRefreshFails(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{ + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: "s3://bucket/data/present.txt", + Bucket: "bucket", + Key: "data/present.txt", + Path: "data/present.txt", + SizeBytes: 100, + }}, + } + service := NewStorageAnalyticsService(backend) + cache := newMemoryProjectBucketInventoryCache() + service.projectBucketCache = cache + + objects, _, err := service.loadCachedProjectBucketValidationInventory(context.Background(), "Bearer token", "org", "proj", "") + if err != nil || len(objects) != 1 { + t.Fatalf("seed cached bucket inventory: objects=%d err=%v", len(objects), err) + } + key := projectBucketInventoryCacheKey("org", "proj", "") + cache.mu.Lock() + entry := cache.entries[key] + entry.value.CachedAt = time.Now().Add(-projectBucketInventoryRefreshInterval - time.Second) + cache.entries[key] = entry + cache.mu.Unlock() + service.chainInputMu.Lock() + chainKey := service.projectChainInputCacheKey("org", "proj") + "::bucket-validation-inventory::" + chainEntry := service.chainInputCache[chainKey] + chainEntry.expiresAt = time.Now().Add(-time.Second) + service.chainInputCache[chainKey] = chainEntry + service.chainInputMu.Unlock() + backend.listProjectBucketObjectsErr = fmt.Errorf("provider returned an incomplete listing") + + objects, lookup, err := service.loadCachedProjectBucketValidationInventory(context.Background(), "Bearer token", "org", "proj", "") + if err != nil { + t.Fatalf("expected stale inventory fallback, got %v", err) + } + if len(objects) != 1 || len(lookup) != 1 { + t.Fatalf("expected cached inventory after failed refresh, got objects=%d lookup=%d", len(objects), len(lookup)) + } if backend.listProjectBucketInventoryCalls != 2 { - t.Fatalf("expected forced refresh to reload bucket inventory, got %d calls", backend.listProjectBucketInventoryCalls) + t.Fatalf("expected one seed scan and one failed refresh, got %d calls", backend.listProjectBucketInventoryCalls) } } diff --git a/internal/git/storage_chain_audit_cache.go b/internal/git/storage_chain_audit_cache.go index 6575505..5ba56d6 100644 --- a/internal/git/storage_chain_audit_cache.go +++ b/internal/git/storage_chain_audit_cache.go @@ -18,8 +18,14 @@ import ( const ( defaultStorageChainAuditCacheTTL = 5 * time.Minute - storageChainAuditCacheKeyPrefix = "gecko:storage_chain_audit:v2:" - storageExactJoinCacheKeyPrefix = "gecko:storage_exact_join:v1:" + // projectBucketInventoryRefreshInterval controls when Gecko asks Syfon for + // a newer inventory. The cache retains the last successful result longer so + // an intermittent provider failure does not turn into a user-facing 502. + projectBucketInventoryRefreshInterval = 10 * time.Minute + projectBucketInventoryStaleTTL = 24 * time.Hour + storageChainAuditCacheKeyPrefix = "gecko:storage_chain_audit:v2:" + storageExactJoinCacheKeyPrefix = "gecko:storage_exact_join:v1:" + projectBucketInventoryKeyPrefix = "gecko:project_bucket_inventory:v1:" ) type storageChainAuditResponseCache interface { @@ -81,6 +87,17 @@ type cachedExactProjectJoinState struct { UsageByObjectID map[string]gintegrationsyfon.FileUsage `json:"usage_by_object_id"` } +type projectBucketInventoryCache interface { + Get(ctx context.Context, key string) (cachedProjectBucketInventory, bool, error) + Set(ctx context.Context, key string, value cachedProjectBucketInventory, ttl time.Duration) error + Source() string +} + +type cachedProjectBucketInventory struct { + CachedAt time.Time `json:"cached_at"` + Objects []gintegrationsyfon.ProjectBucketObject `json:"objects"` +} + type memoryStorageChainAuditResponseCache struct { mu sync.RWMutex entries map[string]memoryStorageChainAuditResponseEntry @@ -129,6 +146,44 @@ type memoryStorageExactProjectJoinCache struct { entries map[string]memoryStorageExactProjectJoinEntry } +type memoryProjectBucketInventoryCache struct { + mu sync.RWMutex + entries map[string]memoryProjectBucketInventoryEntry +} + +type memoryProjectBucketInventoryEntry struct { + value cachedProjectBucketInventory + expiresAt time.Time +} + +func newMemoryProjectBucketInventoryCache() *memoryProjectBucketInventoryCache { + return &memoryProjectBucketInventoryCache{entries: map[string]memoryProjectBucketInventoryEntry{}} +} + +func (cache *memoryProjectBucketInventoryCache) Get(_ context.Context, key string) (cachedProjectBucketInventory, bool, error) { + cache.mu.RLock() + entry, ok := cache.entries[key] + cache.mu.RUnlock() + if !ok || time.Now().After(entry.expiresAt) { + if ok { + cache.mu.Lock() + delete(cache.entries, key) + cache.mu.Unlock() + } + return cachedProjectBucketInventory{}, false, nil + } + return cloneCachedProjectBucketInventory(entry.value), true, nil +} + +func (cache *memoryProjectBucketInventoryCache) Set(_ context.Context, key string, value cachedProjectBucketInventory, ttl time.Duration) error { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.entries[key] = memoryProjectBucketInventoryEntry{value: cloneCachedProjectBucketInventory(value), expiresAt: time.Now().Add(ttl)} + return nil +} + +func (cache *memoryProjectBucketInventoryCache) Source() string { return "memory" } + type memoryStorageExactProjectJoinEntry struct { value cachedExactProjectJoinState expiresAt time.Time @@ -175,6 +230,10 @@ type redisStorageExactProjectJoinCache struct { client *redis.Client } +type redisProjectBucketInventoryCache struct { + client *redis.Client +} + func newRedisStorageChainAuditResponseCache(redisURL string) (*redisStorageChainAuditResponseCache, error) { options, err := redis.ParseURL(strings.TrimSpace(redisURL)) if err != nil { @@ -201,6 +260,19 @@ func newRedisStorageExactProjectJoinCache(redisURL string) (*redisStorageExactPr return cache, nil } +func newRedisProjectBucketInventoryCache(redisURL string) (*redisProjectBucketInventoryCache, error) { + options, err := redis.ParseURL(strings.TrimSpace(redisURL)) + if err != nil { + return nil, err + } + cache := &redisProjectBucketInventoryCache{client: redis.NewClient(options)} + if err := cache.client.Ping(context.Background()).Err(); err != nil { + _ = cache.client.Close() + return nil, err + } + return cache, nil +} + func (cache *redisStorageChainAuditResponseCache) Get(ctx context.Context, key string) (cachedStorageChainAuditResponse, bool, error) { raw, err := cache.client.Get(ctx, key).Bytes() if err == redis.Nil { @@ -261,6 +333,34 @@ func (cache *redisStorageExactProjectJoinCache) Source() string { return "redis" } +func (cache *redisProjectBucketInventoryCache) Get(ctx context.Context, key string) (cachedProjectBucketInventory, bool, error) { + raw, err := cache.client.Get(ctx, key).Bytes() + if err == redis.Nil { + return cachedProjectBucketInventory{}, false, nil + } + if err != nil { + return cachedProjectBucketInventory{}, false, err + } + var value cachedProjectBucketInventory + if err := json.Unmarshal(raw, &value); err != nil { + return cachedProjectBucketInventory{}, false, err + } + if value.CachedAt.IsZero() { + value.CachedAt = time.Now() + } + return cloneCachedProjectBucketInventory(value), true, nil +} + +func (cache *redisProjectBucketInventoryCache) Set(ctx context.Context, key string, value cachedProjectBucketInventory, ttl time.Duration) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + return cache.client.Set(ctx, key, raw, ttl).Err() +} + +func (cache *redisProjectBucketInventoryCache) Source() string { return "redis" } + func NewStorageChainAuditResponseCacheFromEnv() storageChainAuditResponseCache { if !storageChainAuditCacheEnabled() { return nil @@ -289,6 +389,20 @@ func NewStorageExactProjectJoinCacheFromEnv() storageExactProjectJoinCache { return newMemoryStorageExactProjectJoinCache() } +func NewProjectBucketInventoryCacheFromEnv() projectBucketInventoryCache { + if !storageChainAuditCacheEnabled() { + return nil + } + redisURL := storageChainAuditRedisURLFromEnv() + if redisURL != "" { + cache, err := newRedisProjectBucketInventoryCache(redisURL) + if err == nil { + return cache + } + } + return newMemoryProjectBucketInventoryCache() +} + func storageChainAuditRedisURLFromEnv() string { redisURL := strings.TrimSpace(os.Getenv("STORAGE_CHAIN_AUDIT_CACHE_REDIS_URL")) if redisURL == "" { @@ -356,6 +470,12 @@ func storageExactProjectJoinCacheKey(organization string, project string, hash s return storageExactJoinCacheKeyPrefix + hex.EncodeToString(sum[:]) } +func projectBucketInventoryCacheKey(organization string, project string, bucketPathPrefix string) string { + body := fmt.Sprintf("org=%s\nproject=%s\nbucket_path_prefix=%s", strings.TrimSpace(organization), strings.TrimSpace(project), normalizeRepoSubpath(bucketPathPrefix)) + sum := sha256.Sum256([]byte(body)) + return projectBucketInventoryKeyPrefix + hex.EncodeToString(sum[:]) +} + func cloneCachedStorageChainAuditResponse(value cachedStorageChainAuditResponse) cachedStorageChainAuditResponse { return cachedStorageChainAuditResponse{ CachedAt: value.CachedAt, @@ -364,6 +484,10 @@ func cloneCachedStorageChainAuditResponse(value cachedStorageChainAuditResponse) } } +func cloneCachedProjectBucketInventory(value cachedProjectBucketInventory) cachedProjectBucketInventory { + return cachedProjectBucketInventory{CachedAt: value.CachedAt, Objects: append([]gintegrationsyfon.ProjectBucketObject(nil), value.Objects...)} +} + func cloneStorageChainAuditResponse(response GitStorageChainAuditResponse) GitStorageChainAuditResponse { response.Findings = append([]GitStorageChainFinding(nil), response.Findings...) response.Groups = append([]GitStorageChainIssueGroup(nil), response.Groups...) diff --git a/internal/git/storage_chain_audit_cache_test.go b/internal/git/storage_chain_audit_cache_test.go index 2dc1a0e..dc3c950 100644 --- a/internal/git/storage_chain_audit_cache_test.go +++ b/internal/git/storage_chain_audit_cache_test.go @@ -2,6 +2,15 @@ package git import "testing" +func TestProjectBucketInventoryCacheRetention(t *testing.T) { + if projectBucketInventoryRefreshInterval.String() != "10m0s" { + t.Fatalf("projectBucketInventoryRefreshInterval = %s, want 10m", projectBucketInventoryRefreshInterval) + } + if projectBucketInventoryStaleTTL.String() != "24h0m0s" { + t.Fatalf("projectBucketInventoryStaleTTL = %s, want 24h", projectBucketInventoryStaleTTL) + } +} + func TestRedisURLWithPasswordMatchesFenceStyle(t *testing.T) { got := redisURLWithPassword("redis://authz-cache-service:6379/0", "secret") want := "redis://:secret@authz-cache-service:6379/0" From 33f48fb7149070061d7e77137d5864868487cde9 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 16 Jul 2026 12:09:29 -0700 Subject: [PATCH 36/36] fix tests --- internal/git/storage_analytics.go | 32 +++++++++++++++++++- internal/git/storage_analytics_pipeline.go | 34 ++++++++++++++++++---- internal/git/storage_audit_evidence.go | 6 ++++ 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go index 80de625..1697e3e 100644 --- a/internal/git/storage_analytics.go +++ b/internal/git/storage_analytics.go @@ -510,6 +510,7 @@ func storageChainAuditRootResponseProjectionAllowed(gitSubpath string, options S } func normalizeStorageChainAuditOptions(options StorageChainAuditOptions) (StorageChainAuditOptions, error) { + rawProbeMode := strings.TrimSpace(options.ProbeMode) probeMode, ok := NormalizeStorageChainProbeMode(options.ProbeMode) if !ok { return StorageChainAuditOptions{}, fmt.Errorf("invalid storage chain probe mode %q", options.ProbeMode) @@ -520,7 +521,7 @@ func normalizeStorageChainAuditOptions(options StorageChainAuditOptions) (Storag } validationMode := strings.TrimSpace(options.ValidationMode) if validationMode == "" { - if strings.TrimSpace(options.ProbeMode) == "" && bucketMode == StorageChainBucketModeValidate { + if rawProbeMode == "" && bucketMode == StorageChainBucketModeValidate { validationMode = StorageChainValidationModeList } else { validationMode = DefaultStorageChainValidationMode(probeMode, bucketMode) @@ -1022,6 +1023,9 @@ func resolveStorageCleanupApplyAction(finding GitStorageCleanupApplyFinding, act return "", fmt.Errorf("unsupported cleanup finding kind %q", kind) } action := cleanupActionForApplyFinding(actionSelection, finding) + if action == "" && strings.TrimSpace(kind) == "probe_error" { + return storageActionInspectEvidence, nil + } if action == "" { action = strings.TrimSpace(finding.SuggestedAction) } @@ -2527,12 +2531,38 @@ func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[st if assessment.Missing { return storageFindingObjectMissing } + // A metadata miss is definitive for cleanup and chain classification when + // the backend did not attach a more specific error kind. + if hasNotFoundMetadataProbe(record) { + return storageFindingObjectMissing + } + if hasNotFoundListProbe(record) && len(bucketObjectsByURL) > 0 { + return storageFindingObjectMissing + } if resolution.hasProbeError || assessment.Status == "unknown" { return storageFindingProbeError } return storageFindingNone } +func hasNotFoundMetadataProbe(record projectRecordState) bool { + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Operation) == StorageChainValidationModeMetadata && strings.TrimSpace(probe.Status) == "not_found" { + return true + } + } + return false +} + +func hasNotFoundListProbe(record projectRecordState) bool { + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Operation) == StorageChainValidationModeList && strings.TrimSpace(probe.Status) == "not_found" { + return true + } + } + return false +} + func hasExactPathBucketMismatch(record projectRecordState, resolution recordStorageResolution, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { // A basename or size match elsewhere in a project bucket is not evidence // that this record's mapping is wrong: BForePC contains many repeated names diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go index 9aff13b..84f89e2 100644 --- a/internal/git/storage_analytics_pipeline.go +++ b/internal/git/storage_analytics_pipeline.go @@ -142,10 +142,14 @@ func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Conte var bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject var err error if bucketMode == StorageChainBucketModeValidate { - // Full bucket LIST calls are expensive on some providers. Reuse the - // validated project inventory during the cooldown instead of issuing a - // second Syfon scan for every audit or Git ref. - bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + if forceRefresh && service.projectBucketCache == nil { + bucketObjects, bucketObjectsByURL, err = service.loadProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + } else { + // Full bucket LIST calls are expensive on some providers. Reuse the + // validated project inventory during the cooldown instead of issuing a + // second Syfon scan for every audit or Git ref. + bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + } } else { bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) } @@ -935,7 +939,23 @@ func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Contex } timings.Record("inventory_list_validation", time.Since(validateStart)) - timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL), "exact_probe_records", 0) + candidateStart := time.Now() + var candidates *storageAuditRecordSet + if len(view.bucketObjectsByURL) == 0 { + candidates = selectInventoryMissRecordSet(probedRecordSet) + } + timings.Record("exact_list_candidate_selection", time.Since(candidateStart)) + if candidates != nil { + probeStart := time.Now() + exactRecordSet, probeErr := service.attachProjectStorageListValidations(ctx, authorizationHeader, candidates) + timings.Record("targeted_exact_list_validation", time.Since(probeStart)) + if probeErr != nil { + return nil, probeErr + } + probedRecordSet = mergeRecordSetProbes(probedRecordSet, exactRecordSet) + view.bucketObjects, view.bucketObjectsByURL = mergeBucketInventoryWithPresentProbes(view.bucketObjects, view.bucketObjectsByURL, exactRecordSet) + } + timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL), "exact_probe_records", countRecordSet(candidates)) view.recordsByChecksum = probedRecordSet.recordsByChecksum view.allProjectRecords = probedRecordSet.allProjectRecords return view, nil @@ -1494,6 +1514,10 @@ func buildSamePathChecksumConflictFindings(index storageChainIndex, record proje "Git and Syfon map to the same bucket object path, but their checksums disagree. Do not delete this object automatically.", ) for findingIndex := range findings { + findings[findingIndex].Actionability = storageActionabilityInspectOnly + findings[findingIndex].AvailableActions = []string{storageActionInspectEvidence} + findings[findingIndex].DefaultAction = storageActionInspectEvidence + findings[findingIndex].SupportsDryRun = false gitChecksum := index.repoChecksumsByPath[normalizeRepoSubpath(findings[findingIndex].NormalizedPath)] findings[findingIndex].Error = "Git LFS and Syfon map to the same bucket object path but have different SHA-256 values." findings[findingIndex].SuggestedFix = fmt.Sprintf( diff --git a/internal/git/storage_audit_evidence.go b/internal/git/storage_audit_evidence.go index f32ac3d..a931798 100644 --- a/internal/git/storage_audit_evidence.go +++ b/internal/git/storage_audit_evidence.go @@ -48,6 +48,12 @@ func storageChainActionSupportForEvidence(kind string, evidenceStatus string) (s storageActionCreateSyfonRecord, false } + if strings.TrimSpace(kind) == "git_syfon_metadata_mismatch" && strings.TrimSpace(evidenceStatus) == string(storageaudit.EvidenceVerified) { + return storageActionabilityManualChoice, + []string{storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionDeleteBoth, storageActionInspectEvidence}, + "", + true + } policy := storageRepairPolicyForKind(kind) if policy.actionability == storageActionabilityAutoRepair && !storageaudit.AllowsAutomaticRepair(kind, storageaudit.EvidenceStatus(evidenceStatus)) { policy = inspectOnlyStorageRepairPolicy()