diff --git a/config/presentationConfig.go b/config/presentationConfig.go new file mode 100644 index 0000000..116398d --- /dev/null +++ b/config/presentationConfig.go @@ -0,0 +1,18 @@ +package config + +import "fmt" + +type PresentationConfig struct { + PresentationConfig string `json:"presentationConfig"` +} + +func (p PresentationConfig) IsZero() bool { + return p.PresentationConfig == "" +} + +func (p *PresentationConfig) Validate() error { + if p == nil { + return fmt.Errorf("presentation config is required") + } + return nil +} diff --git a/config/presentationConfig_test.go b/config/presentationConfig_test.go new file mode 100644 index 0000000..2fee096 --- /dev/null +++ b/config/presentationConfig_test.go @@ -0,0 +1,28 @@ +package config + +import "testing" + +func TestPresentationConfigValidateAllowsRawHTML(t *testing.T) { + cfg := &PresentationConfig{ + PresentationConfig: `
link

Hello

`, + } + + if err := cfg.Validate(); err != nil { + t.Fatalf("validate failed: %v", err) + } + + if cfg.PresentationConfig != `
link

Hello

` { + t.Fatalf("unexpected raw HTML: %q", cfg.PresentationConfig) + } +} + +func TestPresentationConfigValidateAllowsEmptyHTML(t *testing.T) { + cfg := &PresentationConfig{} + + if err := cfg.Validate(); err != nil { + t.Fatalf("validate failed: %v", err) + } + if cfg.PresentationConfig != "" { + t.Fatalf("expected empty presentationConfig, got %q", cfg.PresentationConfig) + } +} diff --git a/docs/docs.go b/docs/docs.go index f26d636..7dbcb41 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -215,56 +215,213 @@ const docTemplate = `{ } } }, - "/dir/{projectId}": { + "/git/projects/{orgTitle}/{projectTitle}/presentationConfig": { "get": { - "description": "Retrieve directory details for the given project ID and Directory path", + "description": "Retrieve the raw presentation HTML configured for a project. Returns empty content when no presentation has been configured.", "produces": [ "application/json" ], "tags": [ - "Directory" + "Git" ], - "summary": "Retrieve directory information for a project", + "summary": "Get project presentation HTML", "parameters": [ { "type": "string", - "description": "Project ID (format: program-project)", - "name": "projectId", + "description": "Organization title", + "name": "orgTitle", "in": "path", "required": true }, { "type": "string", - "description": "Directory Path (e.g., /data/my-dir)", - "name": "directory_path", - "in": "query", + "description": "Project title", + "name": "projectTitle", + "in": "path", "required": true } ], "responses": { "200": { - "description": "Directory information", + "description": "Presentation configuration", "schema": { - "type": "object", - "additionalProperties": true + "$ref": "#/definitions/config.PresentationConfig" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "500": { + "description": "Server error", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + } + } + }, + "post": { + "description": "Persist raw presentation HTML for a project.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Git" + ], + "summary": "Create project presentation HTML", + "parameters": [ + { + "type": "string", + "description": "Organization title", + "name": "orgTitle", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Project title", + "name": "projectTitle", + "in": "path", + "required": true + }, + { + "description": "Presentation HTML payload", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + } + ], + "responses": { + "200": { + "description": "Persisted presentation configuration", + "schema": { + "$ref": "#/definitions/config.PresentationConfig" } }, "400": { - "description": "Invalid request body or Directory path", + "description": "Invalid request body", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } }, "403": { - "description": "User is not allowed on any resource path", + "description": "Forbidden", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } }, "500": { "description": "Server error", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + } + } + }, + "put": { + "description": "Persist raw presentation HTML for a project.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Git" + ], + "summary": "Update project presentation HTML", + "parameters": [ + { + "type": "string", + "description": "Organization title", + "name": "orgTitle", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Project title", + "name": "projectTitle", + "in": "path", + "required": true + }, + { + "description": "Presentation HTML payload", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + } + ], + "responses": { + "200": { + "description": "Persisted presentation configuration", + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + }, + "400": { + "description": "Invalid request body", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "500": { + "description": "Server error", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } } } @@ -1043,6 +1200,14 @@ const docTemplate = `{ } } }, + "config.PresentationConfig": { + "type": "object", + "properties": { + "presentationConfig": { + "type": "string" + } + } + }, "config.ConfigItem": { "type": "object", "properties": { @@ -1728,7 +1893,3 @@ var SwaggerInfo = &swag.Spec{ LeftDelim: "{{", RightDelim: "}}", } - -func init() { - swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) -} diff --git a/docs/pr-feature-project-config.md b/docs/pr-feature-project-config.md new file mode 100644 index 0000000..a88be6f --- /dev/null +++ b/docs/pr-feature-project-config.md @@ -0,0 +1,565 @@ +# PR Notes: `feature/project-config` + +## What This Branch Actually Does + +This branch is not a small project-config patch. It changes Gecko from a relatively flat config and vector service into a multi-surface backend that now owns: + +- typed project configuration records +- project-oriented config CRUD routes +- Git-backed project repository state and read APIs +- GitHub/Fence integration points for installation and token brokering +- organization-level Git connect/reconcile flows +- upload session persistence and PR-oriented file submission flows +- thumbnail storage for projects +- a full internal package reorganization +- CI and runtime changes required by the new Git subsystem + +The branch is large because it combines feature delivery with a substantial internal re-layout. The biggest review question is not any one handler. It is whether the new package boundaries, auth model, data model, and route surface still line up cleanly with the CALYPR frontend and deployment stack. + +## Scope by Area + +`git diff --dirstat` against `main` shows the branch is concentrated in these areas: + +- `internal/git/` ~10% +- `internal/server/http/git/` ~9% +- `internal/server/middleware/` ~8% +- `gecko/` legacy removal/replacement ~8% +- `internal/server/http/config/` ~7% +- `internal/db/` ~6% +- `config/` ~5% +- `internal/thumbnail/` ~5% +- `tests/integration/` ~4% + +That matches the real shape of the work: this is mostly a Git/project backend branch, plus the refactor needed to support it. + +## Architecture Reorganization + +### Before + +The old service logic lived mostly in the top-level `gecko/` package with mixed concerns: + +- HTTP handlers +- middleware +- DB logic +- vector logic +- config logic +- response helpers + +### After + +The branch moves Gecko toward a clearer package layout: + +- `config/` + - typed config models and validation +- `internal/db/` + - config and Git persistence +- `internal/git/` + - repository domain logic, reconciliation, setup, sync, upload workflows +- `internal/integrations/fence/` + - Fence broker client for GitHub App operations +- `internal/integrations/github/` + - GitHub API client wrapper built on `go-github` +- `internal/server/http/` + - route families split by surface: + - `config` + - `directory` + - `git` + - `health` + - `vector` + - `shared` +- `internal/server/middleware/` + - request auth, access checks, logging, resource helpers +- `internal/thumbnail/` + - thumbnail storage and validation +- `internal/httputil/` + - shared JSON/error response utilities +- `internal/logging/` + - service logging wrapper/helpers +- `internal/vectoradapter/` + - Qdrant request/response translation + +This is the right direction structurally. The tradeoff is that the branch mixes architectural reorganization with product behavior changes, so code review needs to separate “moved” from “changed.” + +## Runtime / Bootstrap Changes + +`main.go` now builds Gecko as a composable server with optional integrations: + +- PostgreSQL +- Git service +- thumbnail store +- Qdrant +- Grip +- JWKS-backed JWT validation + +The important runtime additions are: + +- `--git-data-dir` / `GIT_DATA_DIR` +- `--fence-base-url` / `FENCE_BASE_URL` +- `--github-api-base-url` / `GITHUB_API_BASE_URL` + +Once DB connectivity is available, Gecko now also constructs: + +- `git.NewGitService(...)` +- `thumbnail.NewFilesystemStore(gitDataDir)` + +That makes `gitDataDir` a hard runtime dependency for the Git-enabled server path. + +### CI impact + +Because Gecko now requires a Git data directory, CI had to be updated. The branch modifies: + +- `.github/workflows/tests.yaml` + +to launch Gecko with: + +```bash +-git-data-dir /tmp/gecko-git +``` + +Without that, the service exits before health checks come up. + +## Config Model Changes + +### Project config becomes first-class + +This branch adds a typed `ProjectConfig` model in: + +- `config/projectConfig.go` + +Supported fields: + +- `title` +- `contact_email` +- `src_repo` +- `org_title` +- `description` +- `project_title` + +Validation includes: + +- required-field checks +- email validation +- repository URL normalization into a GitHub-style `host/owner/repo` form + +This is a real shift from using generic config blobs everywhere. Project metadata now has a stronger contract and normalization behavior. + +### Explorer config compatibility + +This branch also carries forward explorer config compatibility work, including the richer `fileActions` shape: + +```json +{ + "extensions": {"ext": ["action"]}, + "actions": {"action": "/path"} +} +``` + +That matters because the frontend expects this richer form, and Gecko has to unmarshal it correctly for CALYPR configs to load. + +## Database / Persistence Changes + +The database layer is no longer only about simple `config_schema.` JSON tables. + +### Existing typed config tables + +The init path now ensures config tables for: + +- `explorer` +- `nav` +- `file_summary` +- `project` +- `projects` + +### New Git-related state tables + +`internal/db/EnsureGitProjectStateTable` expands the DB footprint significantly. This branch adds persistence for: + +1. `config_schema.git_project_state` + - repository identity + - installation metadata + - mirror path + - sync/default branch/error state + +2. `config_schema.git_organization_state` + - organization installation/configuration status + - target metadata + - timestamps and last error + +3. `config_schema.git_upload_session` + - upload/PR submission session metadata + +4. `config_schema.git_upload_session_file` + - per-file status inside an upload session + +5. `config_schema.git_pending_repository` + - repositories discovered but not yet reconciled into project config + - supports both webhook-originated and user-scoped pending records + +6. `config_schema.git_setup_session` + - setup snapshot used to compare repository sets before/after install/connect flows + +This is one of the biggest branch changes. Gecko is no longer stateless around Git operations; it is now persisting lifecycle and reconciliation state explicitly. + +## HTTP Surface Changes + +The route surface is much broader than before. + +### Top-level registration + +The new entrypoint is: + +- `internal/server/http/register.go` + +Registered route families: + +- `/health` +- `/Dir...` +- `/config...` +- `/git...` +- vector routes +- swagger JSON route + +### Config routes + +`internal/server/http/config/register.go` now exposes: + +#### Generic config routes + +- `GET /config/types` +- `GET /config/list` + +Typed config groups: + +- `/config/explorer` +- `/config/nav` +- `/config/file_summary` +- `/config/project` + +Per-type operations include combinations of: + +- `GET /list` +- `GET /:configId` +- `PUT /:configId` +- `DELETE /:configId` + +#### Project config routes + +This branch adds a dedicated project config surface: + +- `GET /config/projects` +- `GET /config/projects/list` +- `GET /config/projects/summary` +- `GET /config/projects/:orgTitle/:projectTitle` +- `PUT /config/projects/:orgTitle/:projectTitle` +- `DELETE /config/projects/:orgTitle/:projectTitle` +- `DELETE /config/projects/:orgTitle` + +This is materially different from the old “config row by arbitrary key” model. The route shape now matches organization/project semantics directly. + +### Git routes + +`internal/server/http/git/register.go` is one of the largest new route families in the service. + +#### Organization-level Git routes + +- `GET /git/projects` +- `GET /git/organizations/status` +- `POST /git/organizations/reconcile` +- `POST /git/organizations/:orgTitle/init-connect` +- `POST /git/organizations/:orgTitle/connect` +- `GET /git/organizations/:orgTitle/status` +- `POST /git/organizations/:orgTitle/reconcile` + +These routes cover installation status, connect flows, and organization-wide repository reconciliation. + +#### Project-level Git read routes + +- `GET /git/projects/:orgTitle/:projectTitle` +- `GET /git/projects/:orgTitle/:projectTitle/refs` +- `GET /git/projects/:orgTitle/:projectTitle/tree` +- `GET /git/projects/:orgTitle/:projectTitle/tree/*` +- `GET /git/projects/:orgTitle/:projectTitle/file/*` +- `GET /git/projects/:orgTitle/:projectTitle/download/*` +- `GET /git/projects/:orgTitle/:projectTitle/thumbnail` + +These expose Gecko as a repository-backed project read service, not just a config API. + +#### Known issue: inconsistent Git LFS classification on file detail responses + +As of June 15, 2026, the frontend has a defensive fix for Git LFS-backed files in the git-style file browser because some `GET /git/projects/:orgTitle/:projectTitle/file/*` responses appear to omit `lfs_pointer` even when the fetched file content is a valid Git LFS pointer. + +Current impact: + +- the Gecko contract is intended to mark LFS-backed files with `lfs_pointer` +- some file detail responses may still look like ordinary GitHub-backed text/blob responses +- when that happens, a naive client can show the raw GitHub LFS pointer content or link users to GitHub, which is not the desired UX + +Current mitigation: + +- the frontend now detects Git LFS pointer text directly in the fetched raw content +- if detected, it suppresses GitHub preview/link actions and routes the user to the Syfon download flow instead + +Follow-up: + +- Gecko should be investigated later to determine why some `/file/*` responses are not consistently populated with `lfs_pointer` +- once Gecko is made consistent, clients can treat `lfs_pointer` as the authoritative classification signal again + +#### Project-level Git write/workflow routes + +- `PUT /git/projects/:orgTitle/:projectTitle/setup` +- `PUT /git/projects/:orgTitle/:projectTitle/storage` +- `PUT /git/projects/:orgTitle/:projectTitle/thumbnail` +- `DELETE /git/projects/:orgTitle/:projectTitle/thumbnail` +- `POST /git/projects/:orgTitle/:projectTitle/edit-connect` +- `POST /git/projects/:orgTitle/:projectTitle/update` +- `POST /git/projects/:orgTitle/:projectTitle/uploads/session` +- `GET /git/projects/:orgTitle/:projectTitle/uploads/session/:sessionID` +- `POST /git/projects/:orgTitle/:projectTitle/uploads/session/:sessionID/files` +- `POST /git/projects/:orgTitle/:projectTitle/uploads/session/:sessionID/finalize` + +This is a major expansion of product surface. Gecko is now responsible for project repository setup, update, artifact flow staging, and PR-style finalization. + +## Auth and Access Model + +The middleware layer has been significantly expanded. + +### Resource path model + +The branch standardizes project authorization around Arborist-style paths: + +```text +/programs/{organization}/projects/{project} +``` + +Helper functions in `internal/server/middleware/access.go` normalize and check these resource paths. + +### Config auth model + +`ConfigAuth` now treats explorer configs differently from base/global config routes: + +- explorer config access is project-scoped +- non-explorer GET routes are broadly readable +- non-explorer write/delete routes require route-specific authorization + +### Project config auth model + +`ProjectConfigAuth` checks direct access on: + +```text +/programs/{org}/projects/{project} +``` + +and also allows certain broader admin-like resource paths such as: + +- `*` +- `/programs` +- `/programs/{org}` +- `/programs/{org}/projects` + +### Git auth model + +Git reads and organization reads are protected separately: + +- `GitProjectAuth` +- `GitOrganizationAuth` + +The important boundary in this branch is: + +- caller authorization still comes from the request `Authorization` token and Arborist/Fence checks +- Git remote access is not the same thing as caller auth and is handled separately through the Git integration flow + +Project write routes are intentionally split: + +- `PUT /git/projects/:orgTitle/:projectTitle/setup` should only require an `Authorization` header at the route layer +- setup then calls Arborist directly to determine whether the caller can read an existing project, create descendants under `/programs/{org}/projects`, manage owners on `/programs/{org}`, or bootstrap missing ownership resources +- Gecko should not pre-judge whether the org or project already exists before making that Arborist call +- follow-on mutation routes such as storage, thumbnail updates, connect edits, uploads, and update actions can use stricter project-scoped middleware because they operate after bootstrap + +That split is correct because Gecko does not have enough local state to distinguish "missing org/project" from "existing but unauthorized"; Arborist is the authority for that decision. + +## Fence / GitHub Integration Model + +This branch adds explicit integration clients instead of spreading ad hoc HTTP logic through handlers. + +### Fence integration + +`internal/integrations/fence/client.go` turns Fence into a GitHub App broker that Gecko calls for: + +- install URL requests +- organization installation status +- repository installation status +- installation repository listing +- installation token minting + +The request target is: + +- `POST {FENCE_BASE_URL}/credentials/github` + +with action-based payloads. + +This is architecturally important. Gecko is moving away from owning GitHub App secrets directly and toward asking Fence for short-lived GitHub access on demand. + +### GitHub integration + +`internal/integrations/github/client.go` uses: + +- `github.com/google/go-github/v87/github` + +for GitHub API metadata reads. + +At minimum, it currently centralizes repository metadata lookup: + +- default branch +- HTML URL + +This is the right direction and avoids more hand-written GitHub REST client code. + +## Git Service and Repository Semantics + +The branch adds a large `internal/git/` package that now owns: + +- repository identity/domain types +- setup and reconcile flows +- repository state persistence coordination +- update/sync operations +- upload workflows +- response shaping +- error mapping + +The key product shift is that Gecko is no longer just proxying config or metadata. It now maintains local repository state under a configured data directory and serves project Git views from there. + +Reviewers should pay special attention to: + +- when local repository state is created +- how `update` behaves when the local repo is missing vs already present +- how default branch information is sourced and persisted +- where Fence tokens are requested and how long they are retained in memory + +## Upload and Thumbnail Workflows + +Two entirely new concerns land in this branch. + +### Upload sessions + +Upload state is now explicit and persistent: + +- session creation +- file list replacement/storage +- session lookup +- finalize flow +- PR metadata persistence + +This means Gecko now participates in a staged contribution workflow rather than just reading repository state. + +### Thumbnails + +`internal/thumbnail/` adds filesystem-backed thumbnail storage plus validation. + +Route support includes: + +- `GET /git/projects/:orgTitle/:projectTitle/thumbnail` +- `PUT /git/projects/:orgTitle/:projectTitle/thumbnail` +- `DELETE /git/projects/:orgTitle/:projectTitle/thumbnail` + +That is a durable product-surface change and should be reviewed as such, not as a minor helper addition. + +## Legacy Code Removal + +A large part of the diff is deletion of the old flat handlers from `gecko/`, including legacy files such as: + +- `handleConfig.go` +- `handleDir.go` +- `handleVector.go` +- `middleware.go` +- `response.go` +- `server.go` + +This is not dead-code cleanup alone. These deletions are paired with replacements under `internal/server/http/...`, `internal/server/middleware/...`, and supporting packages. + +## Build / Tooling Changes + +This branch also touches: + +- `Dockerfile` +- `Makefile` +- `.dockerignore` +- `go.mod` +- `go.sum` +- swagger/docs artifacts + +Those are not side noise. They are part of the fallout from: + +- root build compatibility +- package reorganization +- new GitHub client dependency +- swagger generation drift + +Review should include a sanity pass on container build assumptions and root `go build .` behavior. + +## Testing Changes + +The branch adds or updates tests across multiple layers: + +- config tests +- project config tests +- middleware tests +- Git service tests +- upload tests +- Fence integration tests +- thumbnail tests +- integration tests + +Notable files include: + +- `config/explorerConfig_test.go` +- `config/projectConfig_test.go` +- `internal/git/service_test.go` +- `internal/git/upload_test.go` +- `internal/integrations/fence/client_test.go` +- `internal/server/middleware/git_test.go` +- `internal/thumbnail/store_test.go` +- `tests/integration/*` + +Given the branch size, the main review question is coverage shape rather than raw test count: + +- do route tests reflect current auth semantics? +- do config tests lock frontend/backend schema compatibility? +- do Git tests cover first-time setup, update, and persistence transitions? + +## Highest-Risk Areas + +If reviewing this branch for merge readiness, focus here first: + +1. **Auth correctness** + - project vs organization vs generic config auth + - read vs write route consistency + +2. **Route compatibility** + - does the route surface still match the frontend and revproxy expectations? + +3. **DB/state lifecycle** + - are new tables initialized everywhere Gecko runs? + - are deletes/updates cleaning up related Git state correctly? + +4. **Git/Fence boundary** + - does Gecko request the right thing from Fence? + - is Gecko still assuming too much GitHub App behavior locally? + +5. **Config schema compatibility** + - especially explorer config and project config alignment with the frontend + +6. **Startup behavior** + - new hard dependency on `git-data-dir` + - optional integrations degrading cleanly when unset + +## Bottom Line + +This branch should be read as a backend expansion and service re-platforming branch, not as a narrow project-config feature. + +The durable outcomes are: + +- Gecko now has a first-class project config model. +- Gecko now has a real Git-backed project API surface. +- Gecko now persists Git lifecycle state instead of treating repo operations as transient. +- Gecko now relies on Fence as the GitHub App broker boundary. +- Gecko’s internal organization is materially better, but the branch is broad enough that compatibility review has to be disciplined. diff --git a/docs/storage-chain-audit.md b/docs/storage-chain-audit.md new file mode 100644 index 0000000..a2a08b2 --- /dev/null +++ b/docs/storage-chain-audit.md @@ -0,0 +1,163 @@ +# Gecko storage-chain audit: implementation guide + +The storage-chain audit is a read-only cross-check of Git LFS pointers, Syfon project records, project bucket scope mappings, and Syfon bucket inventory. The handler is registered as `POST /api/v1/projects/:project_id/repair/storage-chain/audit` in [`internal/server/http/git/register.go`](../internal/server/http/git/register.go#L54). Repair endpoints are separate. + +## Code map + +| Responsibility | Owner | +| --- | --- | +| HTTP request parsing, defaults, and error mapping | [`handleGitProjectStorageChainAuditPOST`](../internal/server/http/git/storage_analytics.go#L214-L326) | +| Option normalization and cache selection | [`BuildStorageChainAuditWithOptions`](../internal/git/storage_analytics.go#L481-L539) | +| Root audit-response cache and refresh coalescing | [`buildStorageChainAuditWithResponseCache`](../internal/git/storage_analytics.go#L541-L607), [`coalesceStorageChainAuditRefresh`](../internal/git/storage_chain_audit_cache.go#L49-L76) | +| Fresh audit assembly | [`buildStorageChainAuditFresh`](../internal/git/storage_analytics.go#L646-L723) | +| Parallel Git, record, scope, and bucket input loading | [`loadStorageChainInputs`](../internal/git/storage_analytics_pipeline.go#L75-L180) | +| Syfon record metrics validator and cache | [`loadCachedProjectAuditRecords`](../internal/git/storage_analytics_pipeline.go#L361-L431) | +| Durable bucket inventory cache and stale fallback | [`loadCachedProjectBucketValidationInventory`](../internal/git/storage_analytics_pipeline.go#L519-L566) | +| Validation-mode decision tree | [`buildStorageChainView`](../internal/git/storage_analytics_pipeline.go#L907-L996) | +| Finding model | [`buildStorageChainAuditModel`](../internal/git/storage_analytics_pipeline.go#L1265-L1277) | +| Modes, defaults, and timing implementation | [`storage_analytics_timing.go`](../internal/git/storage_analytics_timing.go#L11-L172) | + +## Request contract + +The handler resolves auth, project context, Git mirror, ref, and commit, then maps the request to `StorageChainAuditOptions` at [`storage_analytics.go`](../internal/server/http/git/storage_analytics.go#L230-L306). + +| Field | Meaning | Default | +| --- | --- | --- | +| `ref` | Git ref to inspect | Project context ref | +| `git_subpath` | Restricts Git findings to one repository path | Root | +| `bucket_inventory_mode` | `validate` uses the project validation inventory; `items` uses explicit bucket items | `validate` | +| `validation_mode` | `list`, `metadata`, or `inventory` | `list` | +| `probe_mode` | Legacy shorthand: `full` selects metadata and `inventory_only` selects inventory when validation mode is omitted | `full` | +| `finding_kind`, `finding_limit` | Response projection only; cache stores the complete model | All / service default | +| `refresh`, `force_audit_refresh`, `force_bucket_inventory_refresh` | Each sets `ForceAuditRefresh` | `false` | + +`bucket_path_prefix` is valid only with `bucket_inventory_mode=items` ([handler check](../internal/server/http/git/storage_analytics.go#L275-L280)). + +## Request-to-response sequence + +```mermaid +sequenceDiagram + actor User + participant UI as Frontend + participant Handler as Gecko handler + participant Audit as StorageAnalyticsService + participant ResponseCache as Audit response cache + participant Git as Git mirror + participant Syfon + participant BucketCache as Bucket inventory cache + participant S3 as Object provider + + User->>UI: Request storage audit + UI->>Handler: POST repair storage-chain audit + Handler->>Handler: Resolve project, mirror, ref, auth, options + Handler->>Audit: BuildStorageChainAuditWithOptions + Audit->>ResponseCache: Lookup eligible root response + ResponseCache-->>Audit: Cached response or miss + Audit->>Git: Build Git LFS inventory on miss + Audit->>Syfon: Load project records and scopes + Audit->>BucketCache: Read inventory by org, project, prefix + BucketCache-->>Audit: Fresh inventory or stale candidate + Audit->>Syfon: Refresh inventory when stale or absent + Syfon->>S3: Paginated LIST and terminal replay + S3-->>Syfon: Inventory or inconsistent-list error + Syfon-->>Audit: Inventory or error + Audit->>BucketCache: Store successful non-empty inventory + Audit->>Audit: Validate records and build findings + Audit->>ResponseCache: Store complete root response + Audit-->>Handler: Findings, groups, summary, timings + Handler-->>UI: JSON response or mapped error +``` + +The middle arrows are conditional. A fresh response cache skips all input work. A fresh bucket cache skips only the expensive Syfon bucket LIST. + +## Cache hierarchy + +| Layer | Scope | Forced-refresh behavior | +| --- | --- | --- | +| Audit response cache | org, project, ref, commit hash, Syfon revision, modes, root path | Bypasses the read, rebuilds the full model, then replaces the response | +| Root response projection | Root cached response projected to a Git subpath | Disabled by hard refresh; it reads Git inventory for the selected subpath when used | +| Project record cache | org/project plus Syfon metrics validator | Reloads when record count, latest update, or revision changes | +| Scope and item cache | Short in-process project cache | Normal in-process reuse only | +| Validation inventory cache | org/project/validation prefix | Fresh for 10 minutes; previous good value is retained for 24 hours | + +The response cache is eligible only for root Git subpath with no bucket path prefix ([eligibility](../internal/git/storage_analytics.go#L504-L510)). Concurrent root refreshes join one in-flight build, preventing overlapping requests from replacing a completed response ([coalescing](../internal/git/storage_chain_audit_cache.go#L49-L76)). + +### Validation inventory cache sequence + +```mermaid +sequenceDiagram + participant Audit as Gecko audit + participant Cache as Validation inventory cache + participant Syfon + participant S3 as Object provider + + Audit->>Cache: Get org project validation-prefix key + Cache-->>Audit: Fresh value under 10 minutes + Note over Audit: Fresh value: do not call Syfon. + Audit->>Syfon: If stale or absent, list project bucket inventory + Syfon->>S3: Traverse pages and replay terminal page + S3-->>Syfon: Complete consistent inventory + Syfon-->>Audit: Successful non-empty inventory + Audit->>Cache: Replace value and set 24-hour retention + Note over Syfon,Audit: Terminal replay mismatch returns an error. + Cache-->>Audit: If stale value exists, return last known-good inventory + Note over Audit: With no cached value, propagate the Syfon error. +``` + +Only non-empty successful inventories replace the cache ([store guard](../internal/git/storage_analytics_pipeline.go#L553-L560)). Syfon terminal-page disagreement is an error, never a partial successful response. Gecko returns stale inventory only when a previous good value exists ([fallback](../internal/git/storage_analytics_pipeline.go#L544-L551)). A cold-cache inventory error remains an HTTP error. + +## Fresh audit execution + +1. `loadStorageChainInputs` starts Git inventory, Syfon records, Syfon scopes, and bucket data concurrently ([lines 100-156](../internal/git/storage_analytics_pipeline.go#L100-L156)). +2. Git, record, and scope errors fail the request. Bucket errors are carried as `bucketInventoryErr`, so validation mode decides whether they are fatal ([lines 158-179](../internal/git/storage_analytics_pipeline.go#L158-L179)). +3. `buildStorageChainView` canonicalizes Syfon record URLs through bucket scopes and attaches list or bulk-probe evidence. +4. `buildStorageChainAuditModel` indexes canonical bucket URLs, Git checksums, and Syfon records, then runs bucket-origin, Syfon-origin, and Git-origin finding passes. +5. Gecko filters and limits only after constructing the complete model. `Groups` use all filtered rows, while `Findings` can be truncated ([response shaping](../internal/git/storage_analytics.go#L703-L722)). + +### Validation modes + +| Mode | Bucket LIST | Additional Syfon work | Bucket view | +| --- | --- | --- | --- | +| `validate` + `list` | Yes, through durable inventory cache | None | Actual listing; inventory misses are unverifiable | +| `validate` + `metadata` | Skipped | Bulk metadata validation | Synthesized from present probes | +| `validate` + `inventory` | Skipped | None | No bucket list | +| `items` + `list` or `inventory` | Yes, using requested prefix | None | Explicit bucket items | +| `items` + `metadata` | Yes | Probe only records needing confirmation | Listed items plus probe evidence | + +The branch is at [`buildStorageChainView`](../internal/git/storage_analytics_pipeline.go#L917-L995). Gecko enables bucket-origin findings only when it has trustworthy list inventory ([decision](../internal/git/storage_analytics.go#L679-L683)). + +## Findings and error semantics + +The summary keys are initialized in [`newChainSummary`](../internal/git/storage_analytics_pipeline.go#L193-L210). + +| Finding kind | Meaning | +| --- | --- | +| `bucket_only_object` | Bucket object has no matching Syfon record or equivalent record identity | +| `bucket_syfon_no_git` | Syfon record resolves to storage but no Git pointer joins it | +| `syfon_missing_bucket_object` | Record storage cannot be resolved from trustworthy list/probe evidence | +| `syfon_git_no_bucket` | Git and Syfon join but storage evidence is missing | +| `git_only_no_syfon` | Git LFS pointer has no Syfon record | +| `git_syfon_metadata_mismatch` | Joined Git, Syfon, and storage metadata conflict | +| `probe_error` | Verification is inconclusive; it is not a missing-object assertion | + +For Weka failures, a Syfon traversal error without cached inventory fails the audit. A stale-cache fallback succeeds from older known-good inventory; it must not generate `probe_error` rows from a false empty list. + +## Observability and tests + +| Log | Meaning | +| --- | --- | +| `syfon_project_bucket_inventory_cache_hit` | Fresh inventory reused | +| `syfon_project_bucket_inventory_cache_store` | Successful non-empty inventory replaced cache | +| `syfon_project_bucket_inventory_cache_stale_fallback` | Syfon refresh failed; stale good inventory used | +| `syfon_project_bucket_inventory_cache_error` | Cache backend failed; Gecko continues direct behavior | +| `storage_chain_audit_request_error` | Handler is returning a mapped audit error | + +Stage timings use `storage_chain_audit_stage_start` and `storage_chain_audit_stage_done`; memory snapshots include Git file, Syfon record, and bucket object counts ([timing implementation](../internal/git/storage_analytics_timing.go#L46-L93)). + +Run focused coverage from the Gecko repository: + +```bash +GOCACHE=/private/tmp/gecko-gocache go test ./internal/git -run 'TestBuildStorageChainAuditForceRefreshBypassesResponseCache|TestProjectBucketInventoryUsesStaleCacheWhenRefreshFails|TestProjectBucketInventoryCacheRetention' -count=1 +``` + +The relevant tests are [`internal/git/storage_analytics_test.go`](../internal/git/storage_analytics_test.go#L2356-L2451) and [`internal/git/storage_chain_audit_cache_test.go`](../internal/git/storage_chain_audit_cache_test.go). diff --git a/docs/swagger.json b/docs/swagger.json index be01477..e54836a 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -209,56 +209,213 @@ } } }, - "/dir/{projectId}": { + "/git/projects/{orgTitle}/{projectTitle}/presentationConfig": { "get": { - "description": "Retrieve directory details for the given project ID and Directory path", + "description": "Retrieve the raw presentation HTML configured for a project. Returns empty content when no presentation has been configured.", "produces": [ "application/json" ], "tags": [ - "Directory" + "Git" ], - "summary": "Retrieve directory information for a project", + "summary": "Get project presentation HTML", "parameters": [ { "type": "string", - "description": "Project ID (format: program-project)", - "name": "projectId", + "description": "Organization title", + "name": "orgTitle", "in": "path", "required": true }, { "type": "string", - "description": "Directory Path (e.g., /data/my-dir)", - "name": "directory_path", - "in": "query", + "description": "Project title", + "name": "projectTitle", + "in": "path", "required": true } ], "responses": { "200": { - "description": "Directory information", + "description": "Presentation configuration", "schema": { - "type": "object", - "additionalProperties": true + "$ref": "#/definitions/config.PresentationConfig" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "500": { + "description": "Server error", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + } + } + }, + "post": { + "description": "Persist raw presentation HTML for a project.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Git" + ], + "summary": "Create project presentation HTML", + "parameters": [ + { + "type": "string", + "description": "Organization title", + "name": "orgTitle", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Project title", + "name": "projectTitle", + "in": "path", + "required": true + }, + { + "description": "Presentation HTML payload", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + } + ], + "responses": { + "200": { + "description": "Persisted presentation configuration", + "schema": { + "$ref": "#/definitions/config.PresentationConfig" } }, "400": { - "description": "Invalid request body or Directory path", + "description": "Invalid request body", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } }, "403": { - "description": "User is not allowed on any resource path", + "description": "Forbidden", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } }, "500": { "description": "Server error", "schema": { - "$ref": "#/definitions/gecko.ErrorResponse" + "$ref": "#/definitions/httputil.ErrorResponse" + } + } + } + }, + "put": { + "description": "Persist raw presentation HTML for a project.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Git" + ], + "summary": "Update project presentation HTML", + "parameters": [ + { + "type": "string", + "description": "Organization title", + "name": "orgTitle", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Project title", + "name": "projectTitle", + "in": "path", + "required": true + }, + { + "description": "Presentation HTML payload", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + } + ], + "responses": { + "200": { + "description": "Persisted presentation configuration", + "schema": { + "$ref": "#/definitions/config.PresentationConfig" + } + }, + "400": { + "description": "Invalid request body", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "401": { + "description": "Missing authorization", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "404": { + "description": "Project not found", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" + } + }, + "500": { + "description": "Server error", + "schema": { + "$ref": "#/definitions/httputil.ErrorResponse" } } } @@ -1037,6 +1194,14 @@ } } }, + "config.PresentationConfig": { + "type": "object", + "properties": { + "presentationConfig": { + "type": "string" + } + } + }, "config.ConfigItem": { "type": "object", "properties": { @@ -1707,4 +1872,4 @@ "in": "header" } } -} \ No newline at end of file +} diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 337884f..373462b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -193,6 +193,11 @@ definitions: sharedFilters: $ref: '#/definitions/config.SharedFiltersConfig' type: object + config.PresentationConfig: + properties: + presentationConfig: + type: string + type: object config.ConfigItem: properties: buttons: @@ -828,44 +833,148 @@ paths: summary: List all configuration IDs for a specific type tags: - Config - /dir/{projectId}: + /git/projects/{orgTitle}/{projectTitle}/presentationConfig: get: - description: Retrieve directory details for the given project ID and Directory - path + description: Retrieve the raw presentation HTML configured for a project. Returns empty content when no presentation has been configured. parameters: - - description: 'Project ID (format: program-project)' + - description: Organization title in: path - name: projectId + name: orgTitle required: true type: string - - description: Directory Path (e.g., /data/my-dir) - in: query - name: directory_path + - description: Project title + in: path + name: projectTitle required: true type: string produces: - application/json responses: "200": - description: Directory information + description: Presentation configuration schema: - additionalProperties: true - type: object + $ref: '#/definitions/config.PresentationConfig' + "401": + description: Missing authorization + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "404": + description: Project not found + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "500": + description: Server error + schema: + $ref: '#/definitions/httputil.ErrorResponse' + summary: Get project presentation HTML + tags: + - Git + post: + consumes: + - application/json + description: Persist raw presentation HTML for a project. + parameters: + - description: Organization title + in: path + name: orgTitle + required: true + type: string + - description: Project title + in: path + name: projectTitle + required: true + type: string + - description: Presentation HTML payload + in: body + name: body + required: true + schema: + $ref: '#/definitions/config.PresentationConfig' + produces: + - application/json + responses: + "200": + description: Persisted presentation configuration + schema: + $ref: '#/definitions/config.PresentationConfig' "400": - description: Invalid request body or Directory path + description: Invalid request body schema: - $ref: '#/definitions/gecko.ErrorResponse' + $ref: '#/definitions/httputil.ErrorResponse' + "401": + description: Missing authorization + schema: + $ref: '#/definitions/httputil.ErrorResponse' "403": - description: User is not allowed on any resource path + description: Forbidden schema: - $ref: '#/definitions/gecko.ErrorResponse' + $ref: '#/definitions/httputil.ErrorResponse' + "404": + description: Project not found + schema: + $ref: '#/definitions/httputil.ErrorResponse' "500": description: Server error schema: - $ref: '#/definitions/gecko.ErrorResponse' - summary: Retrieve directory information for a project + $ref: '#/definitions/httputil.ErrorResponse' + summary: Create project presentation HTML + tags: + - Git + put: + consumes: + - application/json + description: Persist raw presentation HTML for a project. + parameters: + - description: Organization title + in: path + name: orgTitle + required: true + type: string + - description: Project title + in: path + name: projectTitle + required: true + type: string + - description: Presentation HTML payload + in: body + name: body + required: true + schema: + $ref: '#/definitions/config.PresentationConfig' + produces: + - application/json + responses: + "200": + description: Persisted presentation configuration + schema: + $ref: '#/definitions/config.PresentationConfig' + "400": + description: Invalid request body + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "401": + description: Missing authorization + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "404": + description: Project not found + schema: + $ref: '#/definitions/httputil.ErrorResponse' + "500": + description: Server error + schema: + $ref: '#/definitions/httputil.ErrorResponse' + summary: Update project presentation HTML tags: - - Directory + - Git /health: get: description: Checks the database connection and returns the server status diff --git a/go.mod b/go.mod index 4a2b032..676e177 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 github.com/qdrant/go-client v1.18.1 + github.com/redis/go-redis/v9 v9.17.1 github.com/stretchr/testify v1.11.1 github.com/swaggo/swag v1.16.6 github.com/uc-cdis/arborist v0.0.0-20260324212054-708d91019bea @@ -45,10 +46,12 @@ require ( github.com/bytedance/sonic v1.15.1 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/calypr/syfon v0.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/cloudwego/base64x v0.1.7 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect diff --git a/go.sum b/go.sum index 6d6c2f6..8ff836d 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -215,6 +217,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/qdrant/go-client v1.18.1 h1:o/dDmSl6ONAlaAFtjdlzztcs3NH0tJY3l5C/z/Uu0bE= github.com/qdrant/go-client v1.18.1/go.mod h1:Xkfp+r89uNOgSbvilVAhCZ3wKI4G+hB/r9Zr2m4zifI= +github.com/redis/go-redis/v9 v9.17.1 h1:7tl732FjYPRT9H9aNfyTwKg9iTETjWjGKEJ2t/5iWTs= +github.com/redis/go-redis/v9 v9.17.1/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= diff --git a/internal/db/config.go b/internal/db/config.go index 36194cd..b9d0144 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -95,10 +95,6 @@ func ConfigPUTGeneric(db *sqlx.DB, configId string, configType string, data any) return ConfigPUTGenericContext(context.Background(), db, configId, configType, data) } -func ConfigPUTGenericTx(tx *sqlx.Tx, configId string, configType string, data any) error { - return ConfigPUTGenericTxContext(context.Background(), tx, configId, configType, data) -} - func ConfigPUTGenericContext(ctx context.Context, db *sqlx.DB, configId string, configType string, data any) error { if db == nil { return nil diff --git a/internal/db/git_pending.go b/internal/db/git_pending.go deleted file mode 100644 index e6e69a7..0000000 --- a/internal/db/git_pending.go +++ /dev/null @@ -1,246 +0,0 @@ -package db - -import ( - "database/sql" - "encoding/json" - "errors" - "fmt" - - "github.com/jmoiron/sqlx" -) - -func UpsertGitPendingRepository(db *sqlx.DB, pending GitPendingRepository) error { - if db == nil { - return nil - } - _, err := db.NamedExec(` - INSERT INTO config_schema.git_pending_repository ( - id, installation_id, setup_session_id, created_by_user_id, source, organization, repo_id, repo_name, repo_full_name, repo_html_url, repo_clone_url, repo_host, repo_owner, repo_path, added_at, updated_at, resolved_at, removed_at - ) VALUES ( - :id, :installation_id, :setup_session_id, :created_by_user_id, :source, :organization, :repo_id, :repo_name, :repo_full_name, :repo_html_url, :repo_clone_url, :repo_host, :repo_owner, :repo_path, :added_at, :updated_at, :resolved_at, :removed_at - ) - ON CONFLICT (id) DO UPDATE SET - id = EXCLUDED.id, - setup_session_id = EXCLUDED.setup_session_id, - created_by_user_id = EXCLUDED.created_by_user_id, - source = EXCLUDED.source, - organization = EXCLUDED.organization, - repo_name = EXCLUDED.repo_name, - repo_full_name = EXCLUDED.repo_full_name, - repo_html_url = EXCLUDED.repo_html_url, - repo_clone_url = EXCLUDED.repo_clone_url, - repo_host = EXCLUDED.repo_host, - repo_owner = EXCLUDED.repo_owner, - repo_path = EXCLUDED.repo_path, - added_at = EXCLUDED.added_at, - updated_at = EXCLUDED.updated_at, - resolved_at = EXCLUDED.resolved_at, - removed_at = EXCLUDED.removed_at - `, pending) - if err != nil { - return fmt.Errorf("upsert git pending repository: %w", err) - } - return nil -} - -func gitPendingRepositorySelectSQL() string { - return `SELECT id, installation_id, setup_session_id, created_by_user_id, source, organization, repo_id, repo_name, repo_full_name, repo_html_url, repo_clone_url, repo_host, repo_owner, repo_path, added_at, updated_at, resolved_at, removed_at FROM config_schema.git_pending_repository` -} - -func ListGitPendingRepositoriesByInstallation(db *sqlx.DB, installationID int64) ([]GitPendingRepository, error) { - if db == nil { - return []GitPendingRepository{}, nil - } - records := []GitPendingRepository{} - if err := db.Select(&records, gitPendingRepositorySelectSQL()+` - WHERE installation_id = $1 AND resolved_at IS NULL AND removed_at IS NULL - ORDER BY added_at, repo_full_name - `, installationID); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return []GitPendingRepository{}, nil - } - return nil, err - } - return records, nil -} - -func ListGitPendingRepositoriesByUser(db *sqlx.DB, userID string, installationID int64, setupSessionID string) ([]GitPendingRepository, error) { - if db == nil { - return []GitPendingRepository{}, nil - } - records := []GitPendingRepository{} - query := gitPendingRepositorySelectSQL() + ` - WHERE created_by_user_id = $1 - AND resolved_at IS NULL - AND removed_at IS NULL` - args := []any{userID} - if installationID > 0 { - args = append(args, installationID) - query += fmt.Sprintf(" AND installation_id = $%d", len(args)) - } - if setupSessionID != "" { - args = append(args, setupSessionID) - query += fmt.Sprintf(" AND setup_session_id = $%d", len(args)) - } - query += " ORDER BY added_at, repo_full_name" - if err := db.Select(&records, query, args...); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return []GitPendingRepository{}, nil - } - return nil, err - } - return records, nil -} - -func ListGitPendingRepositories(db *sqlx.DB) ([]GitPendingRepository, error) { - if db == nil { - return []GitPendingRepository{}, nil - } - records := []GitPendingRepository{} - if err := db.Select(&records, gitPendingRepositorySelectSQL()+` - WHERE resolved_at IS NULL AND removed_at IS NULL - ORDER BY added_at, repo_full_name - `); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return []GitPendingRepository{}, nil - } - return nil, err - } - return records, nil -} - -func GitPendingRepositoryByID(db *sqlx.DB, id string) (*GitPendingRepository, error) { - if db == nil || id == "" { - return nil, nil - } - var pending GitPendingRepository - if err := db.Get(&pending, gitPendingRepositorySelectSQL()+` WHERE id = $1`, id); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - return nil, fmt.Errorf("get git pending repository by id: %w", err) - } - return &pending, nil -} - -func ResolveGitPendingRepositoryByID(db *sqlx.DB, id string) error { - if db == nil || id == "" { - return nil - } - _, err := db.Exec(`UPDATE config_schema.git_pending_repository SET resolved_at = NOW(), updated_at = NOW() WHERE id = $1`, id) - if err != nil { - return fmt.Errorf("resolve git pending repository by id: %w", err) - } - return nil -} - -func ResolveGitPendingRepositoriesByRepo(db *sqlx.DB, installationID int64, repoHost string, repoOwner string, repoPath string) error { - if db == nil { - return nil - } - _, err := db.Exec(` - UPDATE config_schema.git_pending_repository - SET resolved_at = NOW(), updated_at = NOW() - WHERE installation_id = $1 - AND repo_host = $2 - AND repo_owner = $3 - AND repo_path = $4 - AND resolved_at IS NULL - AND removed_at IS NULL - `, installationID, repoHost, repoOwner, repoPath) - if err != nil { - return fmt.Errorf("resolve git pending repositories by repo: %w", err) - } - return nil -} - -func ResolveGitPendingRepositoriesByRepositoryIdentity(db *sqlx.DB, repoHost string, repoOwner string, repoPath string) error { - if db == nil { - return nil - } - _, err := db.Exec(` - UPDATE config_schema.git_pending_repository - SET resolved_at = NOW(), updated_at = NOW() - WHERE repo_host = $1 - AND repo_owner = $2 - AND repo_path = $3 - AND resolved_at IS NULL - AND removed_at IS NULL - `, repoHost, repoOwner, repoPath) - if err != nil { - return fmt.Errorf("resolve git pending repositories by repository identity: %w", err) - } - return nil -} - -func RemoveGitPendingRepository(db *sqlx.DB, installationID int64, repoID int64) error { - if db == nil { - return nil - } - _, err := db.Exec(` - UPDATE config_schema.git_pending_repository - SET removed_at = NOW(), updated_at = NOW() - WHERE installation_id = $1 AND repo_id = $2 AND removed_at IS NULL - `, installationID, repoID) - if err != nil { - return fmt.Errorf("remove git pending repository: %w", err) - } - return nil -} - -func UpsertGitSetupSession(db *sqlx.DB, session GitSetupSession) error { - if db == nil { - return nil - } - _, err := db.NamedExec(` - INSERT INTO config_schema.git_setup_session ( - id, created_by_user_id, organization, installation_id, before_repo_ids, created_at, updated_at, completed_at - ) VALUES ( - :id, :created_by_user_id, :organization, :installation_id, :before_repo_ids, :created_at, :updated_at, :completed_at - ) - ON CONFLICT (id) DO UPDATE SET - created_by_user_id = EXCLUDED.created_by_user_id, - organization = EXCLUDED.organization, - installation_id = EXCLUDED.installation_id, - before_repo_ids = EXCLUDED.before_repo_ids, - updated_at = EXCLUDED.updated_at, - completed_at = EXCLUDED.completed_at - `, session) - if err != nil { - return fmt.Errorf("upsert git setup session: %w", err) - } - return nil -} - -func GitSetupSessionByID(db *sqlx.DB, id string) (*GitSetupSession, error) { - if db == nil || id == "" { - return nil, nil - } - var session GitSetupSession - err := db.Get(&session, `SELECT id, created_by_user_id, organization, installation_id, before_repo_ids, created_at, updated_at, completed_at FROM config_schema.git_setup_session WHERE id = $1`, id) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - return nil, err - } - return &session, nil -} - -func EncodeRepoIDs(repoIDs []int64) string { - body, err := json.Marshal(repoIDs) - if err != nil { - return "[]" - } - return string(body) -} - -func DecodeRepoIDs(raw string) map[int64]struct{} { - repoIDs := []int64{} - _ = json.Unmarshal([]byte(raw), &repoIDs) - indexed := make(map[int64]struct{}, len(repoIDs)) - for _, repoID := range repoIDs { - indexed[repoID] = struct{}{} - } - return indexed -} diff --git a/internal/db/git_project.go b/internal/db/git_project.go index 39121ef..d8a8080 100644 --- a/internal/db/git_project.go +++ b/internal/db/git_project.go @@ -143,10 +143,6 @@ func UpsertGitProjectState(db *sqlx.DB, state GitProjectState) error { return UpsertGitProjectStateContext(context.Background(), db, state) } -func UpsertGitProjectStateTx(tx *sqlx.Tx, state GitProjectState) error { - return UpsertGitProjectStateTxContext(context.Background(), tx, state) -} - func UpsertGitProjectStateContext(ctx context.Context, db *sqlx.DB, state GitProjectState) error { if db == nil { return nil diff --git a/internal/git/domain/types.go b/internal/git/domain/types.go index 8804f0d..8886710 100644 --- a/internal/git/domain/types.go +++ b/internal/git/domain/types.go @@ -39,6 +39,13 @@ type StorageBucket struct { Resources []string } +type StorageBucketScope struct { + Bucket string + Organization string + ProjectID string + Path string +} + type StorageConfig struct { Bucket string Provider string diff --git a/internal/git/github_file_contents_test.go b/internal/git/github_file_contents_test.go new file mode 100644 index 0000000..6c7e159 --- /dev/null +++ b/internal/git/github_file_contents_test.go @@ -0,0 +1,114 @@ +package git + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/google/go-github/v87/github" +) + +type gitRoundTripFunc func(*http.Request) (*http.Response, error) + +func (fn gitRoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + +func TestGetGitHubFileContentsUsesInstallationToken(t *testing.T) { + attempts := 0 + client, err := github.NewClient( + github.WithHTTPClient(&http.Client{ + Transport: gitRoundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + if req.Header.Get("Authorization") != "Bearer install-token" { + t.Fatalf("expected installation token auth header, got %q", req.Header.Get("Authorization")) + } + return gitHubJSONResponse(http.StatusOK, `{"type":"file","name":"Practitioner.ndjson","path":"META/Practitioner.ndjson","sha":"abc","size":43,"content":"dmVyc2lvbg==","encoding":"base64"}`), nil + }), + }), + github.WithAuthToken("install-token"), + ) + if err != nil { + t.Fatalf("create github client: %v", err) + } + + metadata, _, err := getGitHubFileContents(context.Background(), client, GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }, "META/Practitioner.ndjson", &github.RepositoryContentGetOptions{Ref: "main"}, "https://api.github.com", true, githubAccessTokenFingerprint("install-token"), githubAccessTokenLength("install-token")) + if err != nil { + t.Fatalf("get github file contents: %v", err) + } + if attempts != 1 { + t.Fatalf("expected one github file contents request, got %d attempts", attempts) + } + if metadata.GetPath() != "META/Practitioner.ndjson" { + t.Fatalf("unexpected metadata path %q", metadata.GetPath()) + } +} + +func TestGetGitHubFileContentsDoesNotRetryEOF(t *testing.T) { + attempts := 0 + client, err := github.NewClient( + github.WithHTTPClient(&http.Client{ + Transport: gitRoundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + return nil, io.EOF + }), + }), + github.WithAuthToken("install-token"), + ) + if err != nil { + t.Fatalf("create github client: %v", err) + } + + _, _, err = getGitHubFileContents(context.Background(), client, GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }, "META/Practitioner.ndjson", &github.RepositoryContentGetOptions{Ref: "main"}, "https://api.github.com", true, githubAccessTokenFingerprint("install-token"), githubAccessTokenLength("install-token")) + if err == nil { + t.Fatal("expected EOF error") + } + if attempts != 1 { + t.Fatalf("expected no retry after EOF, got %d attempts", attempts) + } +} + +func TestGetGitHubFileContentsDoesNotRetryUnauthorized(t *testing.T) { + attempts := 0 + client, err := github.NewClient( + github.WithHTTPClient(&http.Client{ + Transport: gitRoundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + return gitHubJSONResponse(http.StatusUnauthorized, `{"message":"Bad credentials"}`), nil + }), + }), + github.WithAuthToken("bad-token"), + ) + if err != nil { + t.Fatalf("create github client: %v", err) + } + + _, _, err = getGitHubFileContents(context.Background(), client, GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }, "META/Practitioner.ndjson", &github.RepositoryContentGetOptions{Ref: "main"}, "https://api.github.com", true, githubAccessTokenFingerprint("bad-token"), githubAccessTokenLength("bad-token")) + if err == nil { + t.Fatal("expected unauthorized error") + } + if attempts != 1 { + t.Fatalf("expected no retry for unauthorized response, got %d attempts", attempts) + } +} + +func gitHubJSONResponse(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(body)), + } +} diff --git a/internal/git/github_logging.go b/internal/git/github_logging.go new file mode 100644 index 0000000..ddf7e30 --- /dev/null +++ b/internal/git/github_logging.go @@ -0,0 +1,20 @@ +package git + +import ( + "crypto/sha256" + "encoding/hex" + "strings" +) + +func githubAccessTokenFingerprint(accessToken string) string { + accessToken = strings.TrimSpace(accessToken) + if accessToken == "" { + return "" + } + sum := sha256.Sum256([]byte(accessToken)) + return hex.EncodeToString(sum[:])[:16] +} + +func githubAccessTokenLength(accessToken string) int { + return len(strings.TrimSpace(accessToken)) +} diff --git a/internal/git/reconcile.go b/internal/git/reconcile.go index b98497a..70712ee 100644 --- a/internal/git/reconcile.go +++ b/internal/git/reconcile.go @@ -240,6 +240,12 @@ func (service *ReconcileService) BuildOrganizationStatus(ctx context.Context, or } } installation := buildInstallationStatus(responsePayload, state, identity.Owner) + workflowStage := "" + if strings.TrimSpace(cfg.SrcRepo) == "" { + workflowStage = GitWorkflowStageAwaitingGitHubConnect + } else if installation.Installed { + workflowStage = GitWorkflowStageGitHubConnected + } integrations := ProjectIntegrationStatus{ GitHub: ProjectIntegrationCheck{ Pass: installation.Installed, @@ -247,11 +253,11 @@ func (service *ReconcileService) BuildOrganizationStatus(ctx context.Context, or Storage: deriveStorageIntegrationCheck(buckets, bucketsErr, parts[0], parts[1]), } if strings.TrimSpace(cfg.SrcRepo) == "" { - integrations.GitHub.Reason = "missing_repository_link" + integrations.GitHub.Reason = GitWorkflowStageAwaitingGitHubConnect if responsePayload.AppInstalled { - integrations.GitHub.Details = "GitHub App is installed for this organization, but this project is not linked to a repository yet" + integrations.GitHub.Details = "Project creation is complete. Finish the GitHub connect step to link this project to a repository." } else { - integrations.GitHub.Details = "No GitHub repository is linked to this project" + integrations.GitHub.Details = "Project creation is complete. Connect GitHub for this organization, then finish the GitHub connect step for this project." } } else if !installation.Installed { integrations.GitHub.Reason = "missing_github_connection" @@ -264,6 +270,7 @@ func (service *ReconcileService) BuildOrganizationStatus(ctx context.Context, or Project: parts[1], ResourcePath: ProgramProjectResourcePath(parts[0], parts[1]), Repository: identity, + WorkflowStage: workflowStage, Configured: configured, Integrations: integrations, Accessible: readable, diff --git a/internal/git/reconcile_test.go b/internal/git/reconcile_test.go new file mode 100644 index 0000000..6d104e7 --- /dev/null +++ b/internal/git/reconcile_test.go @@ -0,0 +1,97 @@ +package git + +import ( + "context" + "database/sql" + "encoding/json" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + appconfig "github.com/calypr/gecko/config" + geckodb "github.com/calypr/gecko/internal/db" + "github.com/jmoiron/sqlx" +) + +func TestBuildOrganizationStatusTreatsSetupOnlyProjectAsAwaitingGitHubConnect(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + + projectCfg := appconfig.ProjectConfig{ + Title: "proj-a", + OrgTitle: "TEST", + ProjectTitle: "proj-a", + Description: "setup only", + ContactEmail: "test@example.org", + SrcRepo: "", + } + projectContent, err := json.Marshal(projectCfg) + if err != nil { + t.Fatalf("marshal project config: %v", err) + } + mock.ExpectQuery(`SELECT name, content FROM config_schema\.projects WHERE name=\$1`). + WithArgs("TEST/proj-a"). + WillReturnRows(sqlmock.NewRows([]string{"name", "content"}).AddRow("TEST/proj-a", projectContent)) + + service := &ReconcileService{db: sqlx.NewDb(db, "sqlmock")} + organizationStates := map[string]geckodb.GitOrganizationState{ + "TEST": { + Organization: "TEST", + Installed: true, + RepositorySelection: nullableString("selected"), + UpdatedAt: time.Now(), + }, + } + buckets := map[string]StorageBucket{ + "bucket-a": { + Bucket: "bucket-a", + Resources: []string{"/programs/TEST/projects/proj-a"}, + }, + } + + status, err := service.BuildOrganizationStatus( + context.Background(), + "TEST", + []string{"TEST/proj-a"}, + map[string]geckodb.GitProjectState{}, + organizationStates, + []string{"/programs/TEST/projects/proj-a"}, + buckets, + nil, + ) + if err != nil { + t.Fatalf("build organization status: %v", err) + } + if len(status.Projects) != 1 { + t.Fatalf("expected one project, got %d", len(status.Projects)) + } + project := status.Projects[0] + if project.WorkflowStage != GitWorkflowStageAwaitingGitHubConnect { + t.Fatalf("expected workflow stage %q, got %q", GitWorkflowStageAwaitingGitHubConnect, project.WorkflowStage) + } + if project.Integrations.GitHub.Reason != GitWorkflowStageAwaitingGitHubConnect { + t.Fatalf("expected github reason %q, got %q", GitWorkflowStageAwaitingGitHubConnect, project.Integrations.GitHub.Reason) + } + if project.Integrations.GitHub.Pass { + t.Fatal("expected github integration to be incomplete") + } + if !project.Integrations.Storage.Pass { + t.Fatal("expected storage integration to pass") + } + if project.Configured { + t.Fatal("expected setup-only project to remain unconfigured") + } + if project.Installation.Installed { + t.Fatal("expected project installation to remain unbound before connect") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func nullableString(value string) sql.NullString { + return sql.NullString{String: value, Valid: value != ""} +} diff --git a/internal/git/repository.go b/internal/git/repository.go index 397afb1..a6a0b40 100644 --- a/internal/git/repository.go +++ b/internal/git/repository.go @@ -9,7 +9,6 @@ import ( "regexp" "strconv" "strings" - "time" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" @@ -141,33 +140,6 @@ func OpenRepository(path string) (*gogit.Repository, error) { return repo, nil } -func lookupGitPathLastModified(repo *gogit.Repository, from plumbing.Hash, path string) (*time.Time, error) { - normalizedPath := strings.Trim(strings.TrimSpace(path), "/") - if normalizedPath == "" { - return nil, nil - } - - iter, err := repo.Log(&gogit.LogOptions{ - From: from, - Order: gogit.LogOrderCommitterTime, - PathFilter: func(candidate string) bool { - trimmed := strings.Trim(strings.TrimSpace(candidate), "/") - return trimmed == normalizedPath || strings.HasPrefix(trimmed, normalizedPath+"/") - }, - }) - if err != nil { - return nil, err - } - defer iter.Close() - - commit, err := iter.Next() - if err != nil { - return nil, err - } - lastModifiedAt := commit.Committer.When.UTC() - return &lastModifiedAt, nil -} - func ParseGitLFSPointer(content []byte) *GitLFSPointerInfo { trimmed := strings.TrimSpace(string(content)) if trimmed == "" { diff --git a/internal/git/response.go b/internal/git/response.go index d3615cd..25cc24b 100644 --- a/internal/git/response.go +++ b/internal/git/response.go @@ -2,21 +2,25 @@ package git import ( "context" + "encoding/base64" + "encoding/json" "fmt" - "io" + "log" "net/http" "path/filepath" "sort" "strings" + "time" + servermw "github.com/calypr/gecko/internal/server/middleware" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/google/go-github/v87/github" - servermw "github.com/calypr/gecko/internal/server/middleware" ) -func BuildGitTreeResponse(projectID string, ref string, path string, repo *gogit.Repository, hash plumbing.Hash) (*GitProjectTreeResponse, error) { +func BuildGitTreeResponse(projectID string, ref string, path string, repo *gogit.Repository, hash plumbing.Hash, options GitTreeResponseOptions) (*GitProjectTreeResponse, error) { commit, err := repo.CommitObject(hash) if err != nil { return nil, fmt.Errorf("load commit for ref %s: %w", ref, err) @@ -43,19 +47,6 @@ func BuildGitTreeResponse(projectID string, ref string, path string, repo *gogit gitEntry.Type = "tree" } else { gitEntry.Type = "blob" - if file, err := tree.File(entry.Name); err == nil { - gitEntry.Size = file.Size - if reader, err := file.Reader(); err == nil { - contentBytes, readErr := io.ReadAll(io.LimitReader(reader, 2048)) - _ = reader.Close() - if readErr == nil { - gitEntry.LFSPointer = ParseGitLFSPointer(contentBytes) - } - } - } - } - if lastModifiedAt, err := lookupGitPathLastModified(repo, hash, entryPath); err == nil && lastModifiedAt != nil { - gitEntry.LastModifiedAt = lastModifiedAt } entries = append(entries, gitEntry) } @@ -65,7 +56,164 @@ func BuildGitTreeResponse(projectID string, ref string, path string, repo *gogit } return strings.ToLower(entries[i].Name) < strings.ToLower(entries[j].Name) }) - return &GitProjectTreeResponse{ProjectID: projectID, Ref: ref, Path: normalizedPath, Entries: entries}, nil + + entryCount := len(entries) + truncated := false + if options.Limit > 0 && len(entries) > options.Limit { + entries = entries[:options.Limit] + truncated = true + } + + enrichTreeEntries(tree, entries, options) + if options.IncludeLastModified { + lastModifiedByPath, err := lookupGitPathsLastModified(repo, hash, entries) + if err != nil { + return nil, fmt.Errorf("lookup git tree last modified times: %w", err) + } + for index := range entries { + if lastModifiedAt, ok := lastModifiedByPath[entries[index].Path]; ok { + entries[index].LastModifiedAt = &lastModifiedAt + } + } + } + + return &GitProjectTreeResponse{ + ProjectID: projectID, + Ref: ref, + Path: normalizedPath, + EntryCount: entryCount, + Truncated: truncated, + Entries: entries, + }, nil +} + +type gitManifestCursor struct { + Ref string `json:"ref"` + Path string `json:"path"` + Offset int `json:"offset"` +} + +func BuildGitManifestResponse(projectID string, ref string, path string, repo *gogit.Repository, hash plumbing.Hash, options GitManifestResponseOptions) (*GitProjectManifestResponse, error) { + commit, err := repo.CommitObject(hash) + if err != nil { + return nil, fmt.Errorf("load commit for ref %s: %w", ref, err) + } + tree, err := commit.Tree() + if err != nil { + return nil, fmt.Errorf("load git tree for ref %s: %w", ref, err) + } + normalizedPath := strings.Trim(strings.TrimSpace(path), "/") + if normalizedPath != "" { + tree, err = tree.Tree(normalizedPath) + if err != nil { + return nil, fmt.Errorf("load git tree path %s: %w", normalizedPath, err) + } + } + + offset, err := parseGitManifestCursor(options.Cursor, ref, normalizedPath) + if err != nil { + return nil, err + } + + entries := make([]GitTreeEntry, 0, len(tree.Entries)) + if err := walkGitManifestTree(normalizedPath, tree, options.FilesOnly, &entries); err != nil { + return nil, err + } + sort.Slice(entries, func(i, j int) bool { + if !options.FilesOnly && entries[i].Type != entries[j].Type { + return entries[i].Type == "tree" + } + return entries[i].Path < entries[j].Path + }) + if offset > len(entries) { + return nil, fmt.Errorf("invalid manifest cursor") + } + + end := len(entries) + if options.Limit > 0 && offset+options.Limit < end { + end = offset + options.Limit + } + pageEntries := append([]GitTreeEntry(nil), entries[offset:end]...) + hasMore := end < len(entries) + nextCursor := "" + if hasMore { + nextCursor, err = encodeGitManifestCursor(ref, normalizedPath, end) + if err != nil { + return nil, err + } + } + + return &GitProjectManifestResponse{ + ProjectID: projectID, + Ref: ref, + Path: normalizedPath, + EntryCount: len(pageEntries), + HasMore: hasMore, + NextCursor: nextCursor, + Entries: pageEntries, + }, nil +} + +func walkGitManifestTree(prefix string, tree *object.Tree, filesOnly bool, entries *[]GitTreeEntry) error { + for _, entry := range tree.Entries { + entryPath := entry.Name + if prefix != "" { + entryPath = prefix + "/" + entry.Name + } + entryType := "blob" + if entry.Mode == filemode.Dir { + entryType = "tree" + } + if !filesOnly || entryType == "blob" { + *entries = append(*entries, GitTreeEntry{ + Name: entry.Name, + Path: entryPath, + Type: entryType, + Hash: entry.Hash.String(), + }) + } + if entryType == "tree" { + childTree, err := tree.Tree(entry.Name) + if err != nil { + return fmt.Errorf("load nested git tree %s: %w", entryPath, err) + } + if err := walkGitManifestTree(entryPath, childTree, filesOnly, entries); err != nil { + return err + } + } + } + return nil +} + +func parseGitManifestCursor(raw string, ref string, path string) (int, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return 0, nil + } + decoded, err := base64.RawURLEncoding.DecodeString(raw) + if err != nil { + return 0, fmt.Errorf("invalid manifest cursor") + } + var payload gitManifestCursor + if err := json.Unmarshal(decoded, &payload); err != nil { + return 0, fmt.Errorf("invalid manifest cursor") + } + if payload.Ref != ref || payload.Path != path || payload.Offset < 0 { + return 0, fmt.Errorf("invalid manifest cursor") + } + return payload.Offset, nil +} + +func encodeGitManifestCursor(ref string, path string, offset int) (string, error) { + payload, err := json.Marshal(gitManifestCursor{ + Ref: ref, + Path: path, + Offset: offset, + }) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(payload), nil } func BuildGitRefsResponse(projectID string, defaultBranch string, repo *gogit.Repository) (*GitProjectRefsResponse, error) { @@ -121,47 +269,6 @@ func BuildGitRefsResponse(projectID string, defaultBranch string, repo *gogit.Re return &GitProjectRefsResponse{ProjectID: projectID, DefaultBranch: defaultBranch, Refs: refs}, nil } -func BuildGitFileResponse(projectID string, ref string, path string, repo *gogit.Repository, hash plumbing.Hash) (*GitProjectFileResponse, error) { - commit, err := repo.CommitObject(hash) - if err != nil { - return nil, fmt.Errorf("load commit for ref %s: %w", ref, err) - } - tree, err := commit.Tree() - if err != nil { - return nil, fmt.Errorf("load git tree for ref %s: %w", ref, err) - } - normalizedPath := strings.Trim(strings.TrimSpace(path), "/") - if normalizedPath == "" { - return nil, fmt.Errorf("file path is required") - } - file, err := tree.File(normalizedPath) - if err != nil { - return nil, fmt.Errorf("load git file %s: %w", normalizedPath, err) - } - reader, err := file.Reader() - if err != nil { - return nil, fmt.Errorf("open git file %s: %w", normalizedPath, err) - } - defer reader.Close() - const inlineLimit = 256 * 1024 - contentBytes, err := io.ReadAll(io.LimitReader(reader, inlineLimit+1)) - if err != nil { - return nil, fmt.Errorf("read git file content for %s: %w", normalizedPath, err) - } - if len(contentBytes) > inlineLimit { - contentBytes = contentBytes[:inlineLimit] - } - return &GitProjectFileResponse{ - ProjectID: projectID, - Ref: ref, - Path: normalizedPath, - Name: filepath.Base(normalizedPath), - Hash: file.Hash.String(), - Size: file.Size, - LFSPointer: ParseGitLFSPointer(contentBytes), - }, nil -} - func BuildGitHubFileResponse(projectID string, ref string, path string, metadata *github.RepositoryContent, contentBytes []byte) *GitProjectFileResponse { name := filepath.Base(strings.Trim(strings.TrimSpace(path), "/")) hash := "" @@ -215,7 +322,7 @@ func (service *GitService) GetGitHubFileMetadata(ctx context.Context, authorizat if strings.TrimSpace(ref) != "" { opts.Ref = strings.TrimSpace(ref) } - metadata, _, response, err := client.Repositories.GetContents(ctx, identity.Owner, identity.Repo, path, opts) + metadata, response, err := getGitHubFileContents(ctx, client, identity, path, opts, strings.TrimRight(service.config.GitHubAPIBase, "/"), strings.TrimSpace(accessToken) != "", githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken)) if err != nil { statusCode := http.StatusBadGateway if response != nil && response.StatusCode > 0 { @@ -250,3 +357,44 @@ func (service *GitService) GetGitHubFileMetadata(ctx context.Context, authorizat } return metadata, []byte(contentString), nil } + +func getGitHubFileContents(ctx context.Context, client *github.Client, identity GitRepositoryIdentity, path string, opts *github.RepositoryContentGetOptions, apiBase string, authConfigured bool, tokenFingerprint string, tokenLength int) (*github.RepositoryContent, *github.Response, error) { + ref := "" + if opts != nil { + ref = strings.TrimSpace(opts.Ref) + } + started := time.Now() + requestURL := githubContentsRequestURL(apiBase, identity, path, ref) + log.Printf("INFO: github_file_contents_request_start owner=%s repo=%s path=%q ref=%q request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d", identity.Owner, identity.Repo, path, ref, requestURL, authConfigured, tokenFingerprint, tokenLength) + metadata, _, response, err := client.Repositories.GetContents(ctx, identity.Owner, identity.Repo, path, opts) + statusCode := 0 + rateLimitRemaining := -1 + rateLimitReset := "" + if response != nil { + rateLimitRemaining = response.Rate.Remaining + if !response.Rate.Reset.Time.IsZero() { + rateLimitReset = response.Rate.Reset.Time.UTC().Format(time.RFC3339) + } + if response.Response != nil { + statusCode = response.Response.StatusCode + } + } + if err == nil { + log.Printf("INFO: github_file_contents_request_done owner=%s repo=%s path=%q ref=%q request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%d rate_limit_reset=%q duration_ms=%d", identity.Owner, identity.Repo, path, ref, requestURL, authConfigured, tokenFingerprint, tokenLength, statusCode, rateLimitRemaining, rateLimitReset, time.Since(started).Milliseconds()) + return metadata, response, nil + } + log.Printf("INFO: github_file_contents_request_done owner=%s repo=%s path=%q ref=%q request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%d rate_limit_reset=%q duration_ms=%d error_type=%T error=%q", identity.Owner, identity.Repo, path, ref, requestURL, authConfigured, tokenFingerprint, tokenLength, statusCode, rateLimitRemaining, rateLimitReset, time.Since(started).Milliseconds(), err, err.Error()) + return nil, response, err +} + +func githubContentsRequestURL(apiBase string, identity GitRepositoryIdentity, path string, ref string) string { + if strings.TrimSpace(apiBase) == "" { + apiBase = "https://api.github.com" + } + base := strings.TrimRight(apiBase, "/") + requestURL := fmt.Sprintf("%s/repos/%s/%s/contents/%s", base, identity.Owner, identity.Repo, strings.TrimLeft(path, "/")) + if ref != "" { + requestURL += "?ref=" + ref + } + return requestURL +} diff --git a/internal/git/service.go b/internal/git/service.go index 9ec8f54..5e03fcd 100644 --- a/internal/git/service.go +++ b/internal/git/service.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "log" "os" "path/filepath" "strings" @@ -64,6 +65,19 @@ func (service *GitService) RefreshProject(ctx context.Context, projectID string, if err := SyncRepositoryMirror(ctx, cloneURL, state.MirrorPath, &githttp.BasicAuth{Username: "x-access-token", Password: accessToken}); err != nil { return nil, state, err } + repo, err := OpenRepository(state.MirrorPath) + if err != nil { + return nil, state, fmt.Errorf("open refreshed git mirror: %w", err) + } + if !RepositoryIsEmpty(repo) { + refName, hash, err := ResolveGitReference(repo, repoMetadata.DefaultBranch, repoMetadata.DefaultBranch) + if err != nil { + return nil, state, fmt.Errorf("resolve refreshed git ref: %w", err) + } + if err := PersistRepoAnalyticsIndex(ctx, state.MirrorPath, repo, refName, hash); err != nil { + return nil, state, fmt.Errorf("persist repo analytics index: %w", err) + } + } updated := *state updated.InstallationTarget = sql.NullString{String: identity.Owner, Valid: identity.Owner != ""} updated.InstallationTargetType = sql.NullString{String: "Organization", Valid: identity.Owner != ""} @@ -75,6 +89,10 @@ func (service *GitService) RefreshProject(ctx context.Context, projectID string, } func (service *GitService) StatusFromState(projectID string, organization string, project string, cfg appconfig.ProjectConfig, identity GitRepositoryIdentity, state *geckodb.GitProjectState, orgState *geckodb.GitOrganizationState) GitProjectStatusResponse { + workflowStage := "" + if strings.TrimSpace(cfg.SrcRepo) == "" { + workflowStage = GitWorkflowStageAwaitingGitHubConnect + } response := GitProjectStatusResponse{ ProjectID: projectID, Organization: organization, @@ -83,6 +101,7 @@ func (service *GitService) StatusFromState(projectID string, organization string RequestAccessResourcePath: ProgramProjectResourcePath(organization, project), Config: cfg, Repository: identity, + WorkflowStage: workflowStage, InstallationState: GitInstallationNotConnected, SyncState: GitSyncNeverSynced, } @@ -96,10 +115,15 @@ func (service *GitService) StatusFromState(projectID string, organization string } } if state == nil { + if response.WorkflowStage == "" && response.OrganizationAppInstalled && response.OrganizationRepositorySelection == "all" { + response.WorkflowStage = GitWorkflowStageGitHubConnected + response.InstallationState = GitInstallationConnected + } return response } if state.InstallationID.Valid || state.InstallationTarget.Valid { response.InstallationState = GitInstallationConnected + response.WorkflowStage = GitWorkflowStageGitHubConnected } if state.InstallationID.Valid { installationID := state.InstallationID.Int64 @@ -193,23 +217,24 @@ func (service *GitService) RequestOrganizationInstallationStatus(ctx context.Con return service.fenceAPI.RequestOrganizationInstallationStatus(ctx, authorizationHeader, organization, owner) } -func (service *GitService) ListInstallationRepositories(ctx context.Context, authorizationHeader string, installationID int64) ([]GitHubInstallationRepository, error) { +func (service *GitService) ListInstallationRepositories(ctx context.Context, authorizationHeader string, organization string, owner string, installationID int64) ([]GitHubInstallationRepository, error) { if service.fenceAPI == nil { return nil, fmt.Errorf("fence client is not initialized") } - return service.fenceAPI.ListInstallationRepositories(ctx, authorizationHeader, installationID) -} - -func (service *GitService) RequestInstallationStatus(ctx context.Context, authorizationHeader string, organization string, identity GitRepositoryIdentity) (GitRepositoryInstallationStatus, error) { - if service.fenceAPI == nil { - return GitRepositoryInstallationStatus{}, fmt.Errorf("fence client is not initialized") - } - return service.fenceAPI.RequestInstallationStatus(ctx, authorizationHeader, organization, identity) + return service.fenceAPI.ListInstallationRepositories(ctx, authorizationHeader, organization, owner, installationID) } func (service *GitService) RequestInstallationToken(ctx context.Context, authorizationHeader string, organization string, project string, identity GitRepositoryIdentity, access string) (string, error) { if service.fenceAPI == nil { return "", fmt.Errorf("fence client is not initialized") } - return service.fenceAPI.RequestInstallationToken(ctx, authorizationHeader, organization, project, identity, access) + started := time.Now() + log.Printf("INFO: github_installation_token_request_start organization=%s project=%s owner=%s repo=%s access=%s", organization, project, identity.Owner, identity.Repo, access) + token, err := service.fenceAPI.RequestInstallationToken(ctx, authorizationHeader, organization, project, identity, access) + if err != nil { + log.Printf("INFO: github_installation_token_request_done organization=%s project=%s owner=%s repo=%s access=%s token_received=false duration_ms=%d error=%q", organization, project, identity.Owner, identity.Repo, access, time.Since(started).Milliseconds(), err.Error()) + return "", err + } + log.Printf("INFO: github_installation_token_request_done organization=%s project=%s owner=%s repo=%s access=%s token_received=%t token_fingerprint=%s token_length=%d duration_ms=%d", organization, project, identity.Owner, identity.Repo, access, strings.TrimSpace(token) != "", githubAccessTokenFingerprint(token), githubAccessTokenLength(token), time.Since(started).Milliseconds()) + return token, nil } diff --git a/internal/git/service_test.go b/internal/git/service_test.go index fc7f5e2..d332c4b 100644 --- a/internal/git/service_test.go +++ b/internal/git/service_test.go @@ -46,7 +46,7 @@ func TestSyncRepositoryMirrorPullsUpdatesAndReadsTree(t *testing.T) { if err != nil { t.Fatalf("resolve HEAD: %v", err) } - treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash) + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{}) if err != nil { t.Fatalf("build tree response: %v", err) } @@ -74,16 +74,6 @@ func TestSyncRepositoryMirrorPullsUpdatesAndReadsTree(t *testing.T) { if err != nil { t.Fatalf("resolve updated HEAD: %v", err) } - fileResponse, err := BuildGitFileResponse("org-a/proj-a", refName, "README.md", mirrorRepo, hash) - if err != nil { - t.Fatalf("build updated file response: %v", err) - } - if fileResponse.Name != "README.md" { - t.Fatalf("expected README.md file name, got %q", fileResponse.Name) - } - if fileResponse.Hash == "" { - t.Fatal("expected file hash to be populated") - } } func TestBuildGitRefsResponseIncludesRemoteBranches(t *testing.T) { @@ -164,26 +154,16 @@ func TestBuildGitRefsResponseIncludesRemoteBranches(t *testing.T) { t.Fatalf("expected benchmarking branch in refs response, got %+v", refsResponse.Refs) } - refName, hash, err := ResolveGitReference(mirrorRepo, "benchmarking", defaultBranch) + refName, _, err := ResolveGitReference(mirrorRepo, "benchmarking", defaultBranch) if err != nil { t.Fatalf("resolve benchmarking branch: %v", err) } if refName != "benchmarking" { t.Fatalf("expected resolved ref name benchmarking, got %q", refName) } - fileResponse, err := BuildGitFileResponse("org-a/proj-a", refName, "benchmark.txt", mirrorRepo, hash) - if err != nil { - t.Fatalf("build branch file response: %v", err) - } - if fileResponse.Name != "benchmark.txt" { - t.Fatalf("expected benchmark.txt file name, got %q", fileResponse.Name) - } - if fileResponse.Hash == "" { - t.Fatal("expected branch file hash to be populated") - } } -func TestBuildGitResponsesDetectLFSPointers(t *testing.T) { +func TestBuildGitManifestResponseRecursesAndPaginates(t *testing.T) { tempDir := t.TempDir() sourcePath := filepath.Join(tempDir, "source") repo, err := gogit.PlainInit(sourcePath, false) @@ -194,23 +174,25 @@ func TestBuildGitResponsesDetectLFSPointers(t *testing.T) { if err != nil { t.Fatalf("load worktree: %v", err) } - pointerContent := strings.Join([]string{ - "version https://git-lfs.github.com/spec/v1", - "oid sha256:0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83", - "size 3780184021", - "", - }, "\n") - if err := os.MkdirAll(filepath.Join(sourcePath, "data"), 0o755); err != nil { - t.Fatalf("create data dir: %v", err) - } - if err := os.WriteFile(filepath.Join(sourcePath, "data", "tcga.tumor.ensembl.tsv"), []byte(pointerContent), 0o644); err != nil { - t.Fatalf("write lfs pointer file: %v", err) - } - if _, err := worktree.Add("data/tcga.tumor.ensembl.tsv"); err != nil { - t.Fatalf("add lfs pointer file: %v", err) + for path, content := range map[string]string{ + "README.md": "hello", + "data/a.txt": "a", + "data/nested/b.txt": "b", + "data/nested/c/c.txt": "c", + } { + fullPath := filepath.Join(sourcePath, filepath.FromSlash(path)) + if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } + if err := os.WriteFile(fullPath, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + if _, err := worktree.Add(path); err != nil { + t.Fatalf("add %s: %v", path, err) + } } - if _, err := worktree.Commit("add lfs pointer", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { - t.Fatalf("commit lfs pointer file: %v", err) + if _, err := worktree.Commit("add files", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit files: %v", err) } mirrorPath := filepath.Join(tempDir, "mirror.git") @@ -225,32 +207,89 @@ func TestBuildGitResponsesDetectLFSPointers(t *testing.T) { if err != nil { t.Fatalf("resolve HEAD: %v", err) } - treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "data", mirrorRepo, hash) + + firstPage, err := BuildGitManifestResponse("org-a/proj-a", refName, "data", mirrorRepo, hash, GitManifestResponseOptions{ + FilesOnly: true, + Limit: 2, + }) if err != nil { - t.Fatalf("build tree response: %v", err) + t.Fatalf("build first manifest page: %v", err) + } + if firstPage.EntryCount != 2 { + t.Fatalf("expected two entries on first page, got %d", firstPage.EntryCount) + } + if !firstPage.HasMore || strings.TrimSpace(firstPage.NextCursor) == "" { + t.Fatalf("expected pagination cursor, got %+v", firstPage) + } + if got := []string{firstPage.Entries[0].Path, firstPage.Entries[1].Path}; strings.Join(got, ",") != "data/a.txt,data/nested/b.txt" { + t.Fatalf("unexpected first page paths: %v", got) + } + + secondPage, err := BuildGitManifestResponse("org-a/proj-a", refName, "data", mirrorRepo, hash, GitManifestResponseOptions{ + FilesOnly: true, + Limit: 2, + Cursor: firstPage.NextCursor, + }) + if err != nil { + t.Fatalf("build second manifest page: %v", err) + } + if secondPage.HasMore { + t.Fatalf("expected second page to be terminal, got %+v", secondPage) + } + if got := []string{secondPage.Entries[0].Path}; strings.Join(got, ",") != "data/nested/c/c.txt" { + t.Fatalf("unexpected second page paths: %v", got) + } +} + +func TestBuildGitManifestResponseIncludesDirectoriesWhenRequested(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) } - if len(treeResponse.Entries) != 1 { - t.Fatalf("expected one tree entry, got %+v", treeResponse.Entries) + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + fullPath := filepath.Join(sourcePath, "data", "nested", "b.txt") + if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil { + t.Fatalf("mkdir nested: %v", err) } - treePointer := treeResponse.Entries[0].LFSPointer - if treePointer == nil { - t.Fatalf("expected tree entry to be marked as lfs pointer, got %+v", treeResponse.Entries[0]) + if err := os.WriteFile(fullPath, []byte("b"), 0o644); err != nil { + t.Fatalf("write file: %v", err) } - if treePointer.OID != "0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83" { - t.Fatalf("unexpected lfs oid: %q", treePointer.OID) + if _, err := worktree.Add("data/nested/b.txt"); err != nil { + t.Fatalf("add file: %v", err) + } + if _, err := worktree.Commit("add file", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit file: %v", err) } - if treePointer.Size != 3780184021 { - t.Fatalf("unexpected lfs size: %d", treePointer.Size) + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) } - fileResponse, err := BuildGitFileResponse("org-a/proj-a", refName, "data/tcga.tumor.ensembl.tsv", mirrorRepo, hash) + manifest, err := BuildGitManifestResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitManifestResponseOptions{ + FilesOnly: false, + Limit: 20, + }) if err != nil { - t.Fatalf("build file response: %v", err) + t.Fatalf("build manifest: %v", err) } - if fileResponse.LFSPointer == nil { - t.Fatalf("expected file response to include lfs pointer metadata") + if len(manifest.Entries) != 3 { + t.Fatalf("expected tree+tree+blob entries, got %+v", manifest.Entries) } - if fileResponse.LFSPointer.OID != treePointer.OID { - t.Fatalf("expected matching lfs oid, got %q and %q", fileResponse.LFSPointer.OID, treePointer.OID) + if manifest.Entries[0].Type != "tree" || manifest.Entries[1].Type != "tree" || manifest.Entries[2].Type != "blob" { + t.Fatalf("unexpected manifest entry ordering: %+v", manifest.Entries) } } diff --git a/internal/git/setup.go b/internal/git/setup.go index 3df2140..cd65941 100644 --- a/internal/git/setup.go +++ b/internal/git/setup.go @@ -9,9 +9,11 @@ import ( "net/http" "net/url" "strings" + "time" "github.com/calypr/gecko/config" geckodb "github.com/calypr/gecko/internal/db" + "github.com/calypr/gecko/internal/httpclient" "github.com/calypr/gecko/internal/integrations/syfon" servermw "github.com/calypr/gecko/internal/server/middleware" "github.com/golang-jwt/jwt/v5" @@ -25,6 +27,8 @@ type SetupService struct { accessChecks *servermw.FenceUserAccessHandler } +var arboristHTTPClient = httpclient.NewServiceClient(30 * time.Second) + func NewSetupService(db *sqlx.DB, gitService *GitService, storage *syfon.Manager, accessChecks *servermw.FenceUserAccessHandler) *SetupService { return &SetupService{ db: db, @@ -228,7 +232,7 @@ func createAuthzOwnedDescendant(ctx context.Context, authorizationHeader string, } req.Header.Set("Authorization", authorizationHeader) req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := arboristHTTPClient.Do(req) if err != nil { return fmt.Errorf("request arborist descendant create: %w", err) } @@ -250,7 +254,7 @@ func DeleteAuthzResource(ctx context.Context, authorizationHeader, resourcePath return fmt.Errorf("build arborist resource delete request: %w", err) } req.Header.Set("Authorization", authorizationHeader) - resp, err := http.DefaultClient.Do(req) + resp, err := arboristHTTPClient.Do(req) if err != nil { return fmt.Errorf("request arborist resource delete: %w", err) } diff --git a/internal/git/storage_analytics.go b/internal/git/storage_analytics.go new file mode 100644 index 0000000..1697e3e --- /dev/null +++ b/internal/git/storage_analytics.go @@ -0,0 +1,3816 @@ +package git + +import ( + "context" + "fmt" + "log" + "net/http" + "path" + "sort" + "strings" + "sync" + "time" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +const cleanupInactiveDays = 30 +const projectJoinCacheTTL = 45 * time.Second +const chainInputCacheTTL = 45 * time.Second +const chainProjectRecordCacheMaxAge = 30 * time.Minute +const projectFileUsageBulkChunkSize = 5000 +const storageChainValidationDebugSampleLimit = 20 +const StorageFolderSummaryModeExact = "exact" +const StorageFolderSummarySourceGitIndex = "git_index" +const StorageFolderSummarySourceExactJoin = "exact_join" + +const ( + storageActionabilityAutoRepair = "auto_repair" + storageActionabilityManualChoice = "manual_choice" + storageActionabilityInspectOnly = "inspect_only" + + storageActionRemoveBrokenAccessURLs = "remove_broken_access_urls" + storageActionDeleteSyfonRecord = "delete_syfon_record" + storageActionDeleteBucketObject = "delete_bucket_object" + storageActionDeleteBoth = "delete_both" + storageActionInspectEvidence = "inspect_evidence" + storageActionCreateSyfonRecord = "create_syfon_record" +) + +type storageRepairPolicy struct { + actionability string + actions []string + defaultAction string + supportsDryRun bool +} + +func storageRepairPolicyForKind(kind string) storageRepairPolicy { + switch strings.TrimSpace(kind) { + case "bucket_only_object": + return autoRepairPolicy(storageActionDeleteBucketObject, storageActionInspectEvidence) + case "bucket_syfon_no_git": + // Absence from Git is not sufficient to delete storage automatically, but + // a user may explicitly remove a verified Bucket + Syfon orphan. + return storageRepairPolicy{ + actionability: storageActionabilityManualChoice, + actions: []string{ + storageActionDeleteBoth, + storageActionDeleteSyfonRecord, + storageActionDeleteBucketObject, + storageActionInspectEvidence, + }, + defaultAction: storageActionDeleteBoth, + supportsDryRun: true, + } + case "repo_orphan_live_object": + return autoRepairPolicy(storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionInspectEvidence) + case "repo_orphan_stale_record", "stale_duplicate_record", "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object": + return autoRepairPolicy(storageActionDeleteSyfonRecord, storageActionInspectEvidence) + case "broken_access_url_error", "broken_bucket_mapping", "syfon_broken_bucket_mapping": + return autoRepairPolicy(storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord, storageActionInspectEvidence) + case "storage_validation_mismatch": + return storageRepairPolicy{ + actionability: storageActionabilityManualChoice, + actions: []string{ + storageActionRemoveBrokenAccessURLs, + storageActionDeleteSyfonRecord, + storageActionDeleteBucketObject, + storageActionDeleteBoth, + storageActionInspectEvidence, + }, + supportsDryRun: true, + } + case "git_syfon_metadata_mismatch": + // A mismatch cannot establish which of Git, Syfon, or the bucket is + // authoritative. Requiring an explicit out-of-band reconciliation keeps + // a stale checksum from becoming a destructive cleanup candidate. + return inspectOnlyStorageRepairPolicy() + case "probe_error": + // Probe errors do not prove that the object is absent, but the audit user + // may explicitly discard the affected Syfon records after reviewing them. + return storageRepairPolicy{ + actionability: storageActionabilityManualChoice, + actions: []string{ + storageActionDeleteSyfonRecord, + storageActionInspectEvidence, + }, + defaultAction: storageActionDeleteSyfonRecord, + supportsDryRun: true, + } + default: + return inspectOnlyStorageRepairPolicy() + } +} + +func inspectOnlyStorageRepairPolicy() storageRepairPolicy { + return storageRepairPolicy{ + actionability: storageActionabilityInspectOnly, + actions: []string{storageActionInspectEvidence}, + defaultAction: storageActionInspectEvidence, + supportsDryRun: false, + } +} + +func autoRepairPolicy(defaultAction string, extraActions ...string) storageRepairPolicy { + actions := append([]string{defaultAction}, extraActions...) + return storageRepairPolicy{ + actionability: storageActionabilityAutoRepair, + actions: uniqueStrings(actions), + defaultAction: defaultAction, + supportsDryRun: true, + } +} + +type storageAnalyticsBackend interface { + ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) + ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) + ListProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) + ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectRecord, error) + GetProjectMetricsSummary(ctx context.Context, authorizationHeader string, organization string, project string) (*gintegrationsyfon.ProjectMetricsSummary, error) + ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) + BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]gintegrationsyfon.ProjectRecord, error) + ListProjectFileUsageByObjectIDs(ctx context.Context, authorizationHeader string, organization string, project string, objectIDs []string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) + ListProjectFileUsage(ctx context.Context, authorizationHeader string, organization string, project string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) + ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) + ListProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) + ListProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) + BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) + BulkListStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) + BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error + DeleteProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, objectURLs []string) ([]gintegrationsyfon.ProjectBucketDeleteResult, error) + BulkUpdateAccessMethods(ctx context.Context, authorizationHeader string, updates map[string][]gintegrationsyfon.ProjectAccessMethod) error + RegisterProjectObjects(ctx context.Context, authorizationHeader string, candidates []gintegrationsyfon.ProjectObjectRegistration) ([]gintegrationsyfon.ProjectObjectRegistrationResult, error) +} + +type StorageAnalyticsService struct { + storage storageAnalyticsBackend + projectJoinMu sync.RWMutex + projectJoinCache map[string]cachedProjectJoinState + projectJoinWork map[string]*inflightProjectJoinState + chainInputMu sync.RWMutex + chainInputCache map[string]cachedChainInputState + projectAuditCache map[string]cachedProjectAuditRecordState + projectAuditWork map[string]*inflightProjectAuditRecordState + chainAuditRefreshMu sync.Mutex + chainAuditRefreshWork map[string]*inflightStorageChainAuditRefresh + chainAuditResponseCache storageChainAuditResponseCache + exactProjectJoinCache storageExactProjectJoinCache + projectBucketCache projectBucketInventoryCache +} + +type StorageFolderTimings struct { + DebugPrefix string + Logf func(format string, args ...any) +} + +func (timings *StorageFolderTimings) Record(stage string, duration time.Duration) { + if timings == nil || timings.Logf == nil { + return + } + timings.Logf("storage_folder_stage %s stage=%s duration_ms=%d", timings.DebugPrefix, stage, duration.Milliseconds()) +} + +func NewStorageAnalyticsService(storage storageAnalyticsBackend) *StorageAnalyticsService { + if storage == nil { + return nil + } + return &StorageAnalyticsService{ + storage: storage, + projectJoinCache: map[string]cachedProjectJoinState{}, + projectJoinWork: map[string]*inflightProjectJoinState{}, + chainInputCache: map[string]cachedChainInputState{}, + projectAuditCache: map[string]cachedProjectAuditRecordState{}, + projectAuditWork: map[string]*inflightProjectAuditRecordState{}, + chainAuditRefreshWork: map[string]*inflightStorageChainAuditRefresh{}, + } +} + +func (service *StorageAnalyticsService) EnableStorageChainAuditResponseCacheFromEnv() { + if service == nil { + return + } + service.chainAuditResponseCache = NewStorageChainAuditResponseCacheFromEnv() + service.exactProjectJoinCache = NewStorageExactProjectJoinCacheFromEnv() + service.projectBucketCache = NewProjectBucketInventoryCacheFromEnv() +} + +type RepoInventoryFile struct { + RepoPath string + Name string + Checksum string + Size int64 +} + +type projectRecordState struct { + gintegrationsyfon.ProjectRecord + CanonicalAccessURLs []string + CanonicalAccessURLByRaw map[string]string + Usage gintegrationsyfon.FileUsage + AccessProbes []gintegrationsyfon.BulkStorageProbeResult +} + +type storageAggregate struct { + name string + path string + rowType string + fileCount int + recordCount int + totalBytes int64 + downloadCount int64 + lastDownload *time.Time + latestUpdate *time.Time + duplicateCount int +} + +type projectDiffAuditModel struct { + Findings []GitProjectDiffFinding + Summary GitProjectDiffSummary + PathPrefix string +} + +type cleanupFindingModel struct { + Public GitStorageCleanupFinding + DeleteObjectIDs []string + DeleteStorageData bool + DeleteBucketObjects []string + UpdateAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod + Manual bool +} + +type storageCleanupApplyPlan struct { + DeleteObjectIDs []string + DeleteStorageObjectIDs []string + DeleteBucketObjects []string + UpdateAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod + UpdatedRecordIDs []string + RepoDeletePaths []string + ManualPaths []string + SkippedPaths []string + Verifications []storageCleanupVerification +} + +type cleanupAuditModel struct { + Findings []cleanupFindingModel + PublicFindings []GitStorageCleanupFinding + Summary GitStorageCleanupAuditSummary + ExpectedPathCount int + IncludesRepoManifest bool + PathPrefix string +} + +type chainAuditModel struct { + Findings []GitStorageChainFinding + Summary GitStorageChainAuditSummary + PathPrefix string +} + +type cachedProjectJoinState struct { + expiresAt time.Time + recordsByChecksum map[string][]projectRecordState + usageByObjectID map[string]gintegrationsyfon.FileUsage +} + +type inflightProjectJoinState struct { + done chan struct{} + recordsByChecksum map[string][]projectRecordState + usageByObjectID map[string]gintegrationsyfon.FileUsage + err error +} + +type cachedChainInputState struct { + expiresAt time.Time + projectRecords []gintegrationsyfon.ProjectRecord + projectScopes []domain.StorageBucketScope + bucketSummary *gintegrationsyfon.ProjectBucketSummary + bucketObjects []gintegrationsyfon.ProjectBucketObject + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject +} + +type cachedProjectAuditRecordState struct { + records []gintegrationsyfon.ProjectRecord + validator projectAuditRecordValidator + cachedAt time.Time +} + +type inflightProjectAuditRecordState struct { + done chan struct{} + records []gintegrationsyfon.ProjectRecord + validator projectAuditRecordValidator + err error +} + +type projectAuditRecordValidator struct { + RecordCount int + RecordLatestUpdatedTime string + RecordRevision string +} + +func (service *StorageAnalyticsService) BuildStorageSummary(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitStorageSummaryResponse, error) { + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, false) + if err != nil { + return nil, err + } + directory, err := repoDirectoryAggregate(index, gitSubpath) + if err != nil { + return nil, err + } + summaryAgg := summarizeSubtree(gitSubpath, inventory, recordsByChecksum, usageByObjectID, directory.DirectChildCount) + return &GitStorageSummaryResponse{ + Path: summaryAgg.path, + FileCount: summaryAgg.fileCount, + RecordCount: summaryAgg.recordCount, + DirectChildCount: directory.DirectChildCount, + TotalBytes: summaryAgg.totalBytes, + DownloadCount: summaryAgg.downloadCount, + LastDownloadTime: formatOptionalTime(summaryAgg.lastDownload), + LatestUpdateTime: formatOptionalTime(summaryAgg.latestUpdate), + DuplicatePathCount: summaryAgg.duplicateCount, + }, nil +} + +func (service *StorageAnalyticsService) BuildStorageChildren(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string, cursor string) (*GitStorageChildrenResponse, error) { + index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) + if err != nil { + return nil, err + } + directory, err := repoDirectoryAggregate(index, gitSubpath) + if err != nil { + return nil, err + } + aggregates := cloneDirectoryChildren(directory.Children) + sortStorageAggregates(aggregates, sortBy, sortOrder) + page, err := storageChildrenPageForRequest(aggregates, hash, gitSubpath, sortBy, sortOrder, limit, cursor) + if err != nil { + return nil, err + } + inventory := filterInventoryForStorageChildren(index.sidecar.Files, page.items) + enriched, err := service.enrichStorageChildrenPage(ctx, authorizationHeader, organization, project, gitSubpath, inventory, page.items) + if err != nil { + return nil, err + } + return &GitStorageChildrenResponse{ + Items: storageChildrenItemsFromAggregates(enriched), + HasMore: page.hasMore, + NextCursor: page.nextCursor, + }, nil +} + +func (service *StorageAnalyticsService) BuildStorageFolder(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, limit int, sortBy string, sortOrder string, cursor string, summaryMode string, forceRefresh bool, timings *StorageFolderTimings) (*GitStorageFolderResponse, error) { + if strings.EqualFold(strings.TrimSpace(summaryMode), StorageFolderSummaryModeExact) { + exactStart := time.Now() + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, forceRefresh) + timings.Record("exact_join", time.Since(exactStart)) + if err != nil { + return nil, err + } + directoryStart := time.Now() + directory, err := repoDirectoryAggregate(index, gitSubpath) + timings.Record("directory_aggregate", time.Since(directoryStart)) + if err != nil { + return nil, err + } + summaryAgg := summarizeSubtree(gitSubpath, inventory, recordsByChecksum, usageByObjectID, directory.DirectChildCount) + pageStart := time.Now() + aggregates := cloneDirectoryChildren(directory.Children) + sortStorageAggregates(aggregates, sortBy, sortOrder) + page, err := storageChildrenPageForRequest(aggregates, hash, gitSubpath, sortBy, sortOrder, limit, cursor) + timings.Record("child_pagination", time.Since(pageStart)) + if err != nil { + return nil, err + } + enrichStart := time.Now() + pageInventory := filterInventoryForStorageChildren(inventory, page.items) + enriched := aggregateImmediateChildren(gitSubpath, pageInventory, recordsByChecksum, usageByObjectID, page.items) + timings.Record("enrich_children_page", time.Since(enrichStart)) + return &GitStorageFolderResponse{ + Summary: GitStorageSummaryResponse{ + Path: summaryAgg.path, + Source: StorageFolderSummarySourceExactJoin, + FileCount: summaryAgg.fileCount, + RecordCount: summaryAgg.recordCount, + DirectChildCount: directory.DirectChildCount, + TotalBytes: summaryAgg.totalBytes, + DownloadCount: summaryAgg.downloadCount, + LastDownloadTime: formatOptionalTime(summaryAgg.lastDownload), + LatestUpdateTime: formatOptionalTime(summaryAgg.latestUpdate), + DuplicatePathCount: summaryAgg.duplicateCount, + }, + Children: GitStorageChildrenResponse{ + Items: storageChildrenItemsFromAggregates(enriched), + HasMore: page.hasMore, + NextCursor: page.nextCursor, + }, + }, nil + } + + indexStart := time.Now() + index, err := loadOrBuildRepoAnalyticsIndexWithTimings(ctx, mirrorPath, ref, repo, hash, timings) + timings.Record("load_repo_index", time.Since(indexStart)) + if err != nil { + return nil, err + } + directoryStart := time.Now() + directory, err := repoDirectoryServingIndex(index, gitSubpath) + timings.Record("directory_aggregate", time.Since(directoryStart)) + if err != nil { + return nil, err + } + pageStart := time.Now() + children, err := storageChildrenResponseForServingIndex(directory, hash, gitSubpath, sortBy, sortOrder, limit, cursor) + timings.Record("child_pagination", time.Since(pageStart)) + if err != nil { + return nil, err + } + timings.Record("git_index_remote_enrichment", 0) + normalizedPath := normalizeRepoSubpath(gitSubpath) + return &GitStorageFolderResponse{ + Summary: GitStorageSummaryResponse{ + Path: normalizedPath, + Source: StorageFolderSummarySourceGitIndex, + FileCount: directory.directory.FileCount, + DirectChildCount: directory.directory.DirectChildCount, + TotalBytes: directory.directory.TotalBytes, + }, + Children: children, + }, nil +} + +func (service *StorageAnalyticsService) BuildProjectDiffAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*GitProjectDiffAuditResponse, error) { + _, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, false) + if err != nil { + return nil, err + } + allProjectRecords, err := service.listProjectRecordStates(ctx, authorizationHeader, organization, project, usageByObjectID) + if err != nil { + return nil, err + } + model := buildProjectDiffAuditModel(gitSubpath, inventory, recordsByChecksum, allProjectRecords) + return &GitProjectDiffAuditResponse{ + Findings: model.Findings, + Summary: model.Summary, + PathPrefix: model.PathPrefix, + }, nil +} + +func (service *StorageAnalyticsService) BuildStorageCleanupAudit(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, selectedRepoPaths []string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, checkStorage bool) (*GitStorageCleanupAuditResponse, *cleanupAuditModel, error) { + baseInputs, err := service.loadStorageAuditBaseInputs(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash) + if err != nil { + return nil, nil, err + } + recordSet, err := service.loadScopedProjectRecords(ctx, authorizationHeader, organization, project, baseInputs) + if err != nil { + return nil, nil, err + } + storageView, err := service.loadStorageAuditStorageView(ctx, authorizationHeader, organization, project, recordSet, checkStorage, checkStorage) + if err != nil { + return nil, nil, err + } + model := buildCleanupAuditModel(gitSubpath, baseInputs.inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL, selectedRepoPaths, checkStorage) + return &GitStorageCleanupAuditResponse{ + Findings: model.PublicFindings, + Summary: model.Summary, + ExpectedPathCount: model.ExpectedPathCount, + IncludesRepoManifest: model.IncludesRepoManifest, + PathPrefix: model.PathPrefix, + }, model, nil +} + +func (service *StorageAnalyticsService) BuildStorageChainAuditWithOptions(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, error) { + normalized, err := normalizeStorageChainAuditOptions(options) + if err != nil { + return nil, err + } + if service.chainAuditResponseCache != nil && storageChainAuditResponseCacheAllowed(gitSubpath, normalized) { + return service.buildStorageChainAuditWithResponseCache(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, normalized) + } + if service.chainAuditResponseCache != nil && storageChainAuditRootResponseProjectionAllowed(gitSubpath, normalized) { + response, ok, err := service.projectStorageChainAuditFromRootResponseCache(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, normalized) + if err != nil { + return nil, err + } + if ok { + return response, nil + } + } + if service.chainAuditResponseCache != nil && normalized.Timings != nil { + normalized.Timings.Record("audit_response_cache_bypass_non_root", 0) + } + return service.buildStorageChainAuditFresh(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, normalized) +} + +func storageChainAuditResponseCacheAllowed(gitSubpath string, options StorageChainAuditOptions) bool { + return normalizeRepoSubpath(gitSubpath) == "" && normalizeRepoSubpath(options.BucketPathPrefix) == "" +} + +func storageChainAuditRootResponseProjectionAllowed(gitSubpath string, options StorageChainAuditOptions) bool { + return normalizeRepoSubpath(gitSubpath) != "" && normalizeRepoSubpath(options.BucketPathPrefix) == "" && !options.ForceAuditRefresh +} + +func normalizeStorageChainAuditOptions(options StorageChainAuditOptions) (StorageChainAuditOptions, error) { + rawProbeMode := strings.TrimSpace(options.ProbeMode) + probeMode, ok := NormalizeStorageChainProbeMode(options.ProbeMode) + if !ok { + return StorageChainAuditOptions{}, fmt.Errorf("invalid storage chain probe mode %q", options.ProbeMode) + } + bucketMode, ok := NormalizeStorageChainBucketInventoryMode(options.BucketInventoryMode) + if !ok { + return StorageChainAuditOptions{}, fmt.Errorf("invalid storage chain bucket inventory mode %q", options.BucketInventoryMode) + } + validationMode := strings.TrimSpace(options.ValidationMode) + if validationMode == "" { + if rawProbeMode == "" && bucketMode == StorageChainBucketModeValidate { + validationMode = StorageChainValidationModeList + } else { + validationMode = DefaultStorageChainValidationMode(probeMode, bucketMode) + } + } + validationMode, ok = NormalizeStorageChainValidationMode(validationMode) + if !ok { + return StorageChainAuditOptions{}, fmt.Errorf("invalid storage chain validation mode %q", options.ValidationMode) + } + options.ProbeMode = probeMode + options.ValidationMode = validationMode + options.BucketInventoryMode = bucketMode + options.BucketPathPrefix = normalizeRepoSubpath(options.BucketPathPrefix) + options.FindingKind = strings.TrimSpace(options.FindingKind) + return options, nil +} + +func (service *StorageAnalyticsService) buildStorageChainAuditWithResponseCache(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, error) { + syfonRevision, err := service.loadStorageAuditSyfonRevision(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + cacheKey := storageChainAuditResponseCacheKey(organization, project, ref, gitSubpath, options.ProbeMode, options.ValidationMode, options.BucketInventoryMode, options.BucketPathPrefix, hash.String(), syfonRevision) + cache := service.chainAuditResponseCache + if !options.ForceAuditRefresh { + start := time.Now() + cached, ok, err := cache.Get(ctx, cacheKey) + options.Timings.Record("audit_response_cache_lookup", time.Since(start)) + if err != nil { + logStorageChainAuditCacheError(options.Timings, cache.Source(), "get", err) + } + if ok { + response := projectStorageChainAuditResponse(cached.Response, options.FindingKind, options.FindingLimit) + applyStorageChainAuditCacheMetadata(response, true, cached.CachedAt, cached.RefreshDurationMillis, cache.Source(), "") + options.Timings.Record("audit_response_cache_hit", 0) + options.Timings.RecordMemory( + "audit_response_cache_hit", + "total_findings", response.Summary.TotalFindings, + "returned_findings", response.Summary.ReturnedFindings, + "finding_limit", options.FindingLimit, + ) + return response, nil + } + options.Timings.Record("audit_response_cache_miss", 0) + } else { + options.Timings.Record("audit_response_cache_force_refresh", 0) + } + + cached, joinedRefresh, err := service.coalesceStorageChainAuditRefresh(ctx, cacheKey, func() (cachedStorageChainAuditResponse, error) { + buildOptions := options + buildOptions.FindingKind = "" + buildOptions.FindingLimit = -1 + refreshStart := time.Now() + response, err := service.buildStorageChainAuditFresh(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, buildOptions) + if err != nil { + return cachedStorageChainAuditResponse{}, err + } + response.Summary.SyfonRevision = syfonRevision + value := cachedStorageChainAuditResponse{ + CachedAt: time.Now(), + RefreshDurationMillis: time.Since(refreshStart).Milliseconds(), + Response: *response, + } + cacheStart := time.Now() + if err := cache.Set(ctx, cacheKey, value, storageChainAuditCacheTTL()); err != nil { + logStorageChainAuditCacheError(options.Timings, cache.Source(), "set", err) + } + options.Timings.Record("audit_response_cache_store", time.Since(cacheStart)) + return value, nil + }) + if err != nil { + return nil, err + } + if joinedRefresh { + options.Timings.Record("audit_response_refresh_join", 0) + } + projected := projectStorageChainAuditResponse(cached.Response, options.FindingKind, options.FindingLimit) + source := cache.Source() + if joinedRefresh { + source += ":refresh_join" + } + applyStorageChainAuditCacheMetadata(projected, false, cached.CachedAt, cached.RefreshDurationMillis, source, "") + return projected, nil +} + +func (service *StorageAnalyticsService) projectStorageChainAuditFromRootResponseCache(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, bool, error) { + cache := service.chainAuditResponseCache + syfonRevision, err := service.loadStorageAuditSyfonRevision(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, false, err + } + cacheKey := storageChainAuditResponseCacheKey(organization, project, ref, "", options.ProbeMode, options.ValidationMode, options.BucketInventoryMode, "", hash.String(), syfonRevision) + start := time.Now() + cached, ok, err := cache.Get(ctx, cacheKey) + options.Timings.Record("audit_response_root_cache_lookup", time.Since(start)) + if err != nil { + logStorageChainAuditCacheError(options.Timings, cache.Source(), "get_root", err) + return nil, false, nil + } + if !ok { + options.Timings.Record("audit_response_root_cache_miss", 0) + return nil, false, nil + } + inventoryStart := time.Now() + inventory, err := service.loadStorageChainInventory(ctx, ref, gitSubpath, mirrorPath, repo, hash) + options.Timings.Record("audit_response_root_cache_project_inventory", time.Since(inventoryStart)) + if err != nil { + return nil, false, err + } + response := projectStorageChainAuditResponseForSubpath(cached.Response, gitSubpath, inventory, options.FindingKind, options.FindingLimit) + applyStorageChainAuditCacheMetadata(response, true, cached.CachedAt, cached.RefreshDurationMillis, cache.Source()+":root", "") + options.Timings.Record("audit_response_root_cache_hit", 0) + options.Timings.RecordMemory( + "audit_response_root_cache_hit", + "git_subpath", normalizeRepoSubpath(gitSubpath), + "git_files", response.Summary.GitTrackedFileCount, + "total_findings", response.Summary.TotalFindings, + "returned_findings", response.Summary.ReturnedFindings, + ) + return response, true, nil +} + +func (service *StorageAnalyticsService) buildStorageChainAuditFresh(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, options StorageChainAuditOptions) (*GitStorageChainAuditResponse, error) { + bucketMode := options.BucketInventoryMode + validationMode := options.ValidationMode + bucketPathPrefix := options.BucketPathPrefix + start := time.Now() + options.Timings.StageStart("chain_setup_total") + inputs, err := service.loadStorageChainInputs(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, bucketMode, validationMode, bucketPathPrefix, options.ForceAuditRefresh, options.Timings) + options.Timings.Record("chain_setup_total", time.Since(start)) + if inputs != nil && inputs.recordSet != nil { + options.Timings.RecordMemory( + "chain_setup_total", + "git_files", len(inputs.inventory), + "syfon_records", countRecordStates(inputs.recordSet.allProjectRecords), + "bucket_objects", len(inputs.bucketObjects), + ) + } + if err != nil { + return nil, err + } + storageViewStart := time.Now() + options.Timings.StageStart("storage_view") + storageView, err := service.buildStorageChainView(ctx, authorizationHeader, organization, project, inputs.recordSet, inputs.inventory, inputs.scopes, inputs.bucketObjects, inputs.bucketObjectsByURL, inputs.bucketInventoryErr, bucketMode, validationMode, options.Timings) + options.Timings.Record("storage_view", time.Since(storageViewStart)) + if storageView != nil { + options.Timings.RecordMemory( + "storage_view", + "syfon_records", countRecordStates(storageView.allProjectRecords), + "bucket_objects", len(storageView.bucketObjectsByURL), + ) + } + if err != nil { + return nil, err + } + modelStart := time.Now() + options.Timings.StageStart("model_build") + includeBucketOrigin := storageView.bucketInventoryAvailable && + (bucketMode == StorageChainBucketModeItems || validationMode == StorageChainValidationModeList) + model := buildStorageChainAuditModel(gitSubpath, inputs.inventory, storageView.recordsByChecksum, storageView.allProjectRecords, storageView.bucketObjectsByURL, inputs.scopes, organization, project, includeBucketOrigin) + options.Timings.Record("model_build", time.Since(modelStart)) + options.Timings.RecordMemory( + "model_build", + "total_findings", len(model.Findings), + "syfon_records", countRecordStates(storageView.allProjectRecords), + "bucket_objects", len(storageView.bucketObjectsByURL), + "git_files", len(inputs.inventory), + ) + model.Summary.BucketInventoryAvailable = storageView.bucketInventoryAvailable + model.Summary.BucketInventoryError = storageView.bucketInventoryError + model.Summary.ValidationMode = validationMode + model.Summary.GitRevision = hash.String() + model.Summary.ObservedAt = time.Now().UTC().Format(time.RFC3339Nano) + if inputs.bucketSummary != nil { + exists := inputs.bucketSummary.Exists + model.Summary.BucketPathExists = &exists + model.Summary.BucketPathObjectURL = strings.TrimSpace(inputs.bucketSummary.ObjectURL) + model.Summary.BucketSummaryMode = strings.TrimSpace(inputs.bucketSummary.Mode) + } + filteredFindings := filterStorageChainFindingsByKind(model.Findings, options.FindingKind) + summary := filterStorageChainSummary(model.Summary, filteredFindings) + findings := limitStorageChainFindings(filteredFindings, options.FindingLimit, options.FindingKind) + summary.ReturnedFindings = len(findings) + summary.FindingLimit = options.FindingLimit + summary.FindingsTruncated = len(findings) < len(filteredFindings) + options.Timings.RecordMemory( + "response_shape", + "total_findings", summary.TotalFindings, + "filtered_findings", len(filteredFindings), + "returned_findings", len(findings), + "finding_limit", options.FindingLimit, + ) + return &GitStorageChainAuditResponse{ + Findings: findings, + Groups: summarizeChainIssueGroups(filteredFindings), + Summary: summary, + PathPrefix: model.PathPrefix, + BucketPathPrefix: bucketPathPrefix, + }, nil +} + +func projectStorageChainAuditResponseForSubpath(base GitStorageChainAuditResponse, gitSubpath string, inventory []RepoInventoryFile, findingKind string, findingLimit int) *GitStorageChainAuditResponse { + pathPrefix := normalizeRepoSubpath(gitSubpath) + filteredFindings := make([]GitStorageChainFinding, 0, len(base.Findings)) + for _, finding := range base.Findings { + if storageChainFindingMatchesSubpath(finding, pathPrefix) { + filteredFindings = append(filteredFindings, finding) + } + } + + summary := base.Summary + countsByKind := make(map[string]int, len(summary.CountsByKind)) + for kind := range summary.CountsByKind { + countsByKind[kind] = 0 + } + issueGitPaths := make(map[string]struct{}) + objectIDs := make(map[string]struct{}) + bucketObjects := make(map[string]struct{}) + for _, finding := range filteredFindings { + countsByKind[finding.Kind]++ + if storageChainFindingHasGitPath(finding.Kind) { + for _, sourcePath := range finding.SourcePaths { + if storagePathMatchesRepoSubpath(sourcePath, pathPrefix) && !strings.Contains(sourcePath, "://") { + issueGitPaths[normalizeRepoSubpath(sourcePath)] = struct{}{} + } + } + if len(finding.SourcePaths) == 0 && storagePathMatchesRepoSubpath(finding.NormalizedPath, pathPrefix) && !strings.Contains(finding.NormalizedPath, "://") { + issueGitPaths[normalizeRepoSubpath(finding.NormalizedPath)] = struct{}{} + } + } + for _, objectID := range finding.ObjectIDs { + if trimmed := strings.TrimSpace(objectID); trimmed != "" { + objectIDs[trimmed] = struct{}{} + } + } + if trimmed := strings.TrimSpace(finding.BucketObjectURL); trimmed != "" { + bucketObjects[trimmed] = struct{}{} + } + for _, accessURL := range finding.AccessURLs { + if trimmed := strings.TrimSpace(accessURL); trimmed != "" { + bucketObjects[trimmed] = struct{}{} + } + } + } + gitTrackedFileCount := len(inventory) + completeCount := gitTrackedFileCount - len(issueGitPaths) + if completeCount < 0 { + completeCount = 0 + } + countsByKind["bucket_syfon_git_complete"] = completeCount + summary.CountsByKind = countsByKind + summary.TotalFindings = len(filteredFindings) + summary.GitTrackedFileCount = gitTrackedFileCount + summary.SyfonRecordCount = completeCount + len(objectIDs) + summary.BucketObjectCount = completeCount + len(bucketObjects) + + projected := base + projected.Findings = filteredFindings + projected.Groups = summarizeChainIssueGroups(filteredFindings) + projected.Summary = summary + projected.PathPrefix = pathPrefix + projected.BucketPathPrefix = "" + return projectStorageChainAuditResponse(projected, findingKind, findingLimit) +} + +func storageChainFindingHasGitPath(kind string) bool { + switch strings.TrimSpace(kind) { + case "git_only_no_syfon", "syfon_git_no_bucket", "git_syfon_metadata_mismatch", "probe_error": + return true + } + return false +} + +func projectStorageChainAuditResponse(base GitStorageChainAuditResponse, findingKind string, findingLimit int) *GitStorageChainAuditResponse { + response := cloneStorageChainAuditResponse(base) + filteredFindings := filterStorageChainFindingsByKind(response.Findings, findingKind) + summary := filterStorageChainSummary(response.Summary, filteredFindings) + findings := limitStorageChainFindings(filteredFindings, findingLimit, findingKind) + summary.ReturnedFindings = len(findings) + summary.FindingLimit = findingLimit + summary.FindingsTruncated = len(findings) < len(filteredFindings) + response.Findings = findings + response.Groups = summarizeChainIssueGroups(filteredFindings) + response.Summary = summary + return &response +} + +func storageChainFindingMatchesSubpath(finding GitStorageChainFinding, pathPrefix string) bool { + if pathPrefix == "" { + return true + } + candidates := []string{finding.NormalizedPath, finding.BucketObjectURL, finding.ResolvedKey} + candidates = append(candidates, finding.SourcePaths...) + candidates = append(candidates, finding.AccessURLs...) + for _, record := range finding.Records { + candidates = append(candidates, record.NormalizedPath) + candidates = append(candidates, record.AccessURLs...) + for _, accessMethod := range record.AccessMethods { + candidates = append(candidates, accessMethod.URL) + } + for _, probe := range record.AccessProbes { + candidates = append(candidates, probe.URL, probe.Path, probe.Key) + } + } + if finding.Evidence != nil { + candidates = append(candidates, finding.Evidence.SourcePaths...) + candidates = append(candidates, finding.Evidence.AccessURLs...) + candidates = append(candidates, finding.Evidence.BucketObjectURLs...) + candidates = append(candidates, finding.Evidence.Keys...) + } + for _, candidate := range candidates { + if storagePathMatchesRepoSubpath(candidate, pathPrefix) { + return true + } + } + return false +} + +func storagePathMatchesRepoSubpath(raw string, pathPrefix string) bool { + pathPrefix = normalizeRepoSubpath(pathPrefix) + if pathPrefix == "" { + return true + } + value := strings.TrimSpace(raw) + if value == "" { + return false + } + if _, key, ok := parseStorageURL(value); ok { + key = normalizeRepoSubpath(key) + return key == pathPrefix || strings.HasPrefix(key, pathPrefix+"/") || strings.Contains("/"+key, "/"+pathPrefix+"/") + } + normalized := normalizeRepoSubpath(value) + return normalized == pathPrefix || strings.HasPrefix(normalized, pathPrefix+"/") || strings.Contains("/"+normalized, "/"+pathPrefix+"/") +} + +func logStorageChainAuditCacheError(timings *StorageChainAuditTimings, source string, operation string, err error) { + if timings == nil || timings.Logf == nil || err == nil { + return + } + timings.Logf("storage_chain_audit_cache_error %s source=%s operation=%s error=%q", strings.TrimSpace(timings.DebugPrefix), strings.TrimSpace(source), strings.TrimSpace(operation), err.Error()) +} + +func applyStorageChainAuditCacheMetadata(response *GitStorageChainAuditResponse, hit bool, cachedAt time.Time, refreshDurationMillis int64, source string, cacheError string) { + if response == nil || cachedAt.IsZero() { + return + } + response.Summary.AuditCacheHit = hit + response.Summary.AuditCachedAt = cachedAt.UTC().Format(time.RFC3339Nano) + response.Summary.AuditCacheAgeSeconds = int64(time.Since(cachedAt).Seconds()) + response.Summary.AuditRefreshDurationMs = refreshDurationMillis + response.Summary.AuditCacheSource = strings.TrimSpace(source) + response.Summary.AuditCacheError = strings.TrimSpace(cacheError) +} + +func filterStorageChainFindingsByKind(findings []GitStorageChainFinding, kind string) []GitStorageChainFinding { + kind = strings.TrimSpace(kind) + if kind == "" { + return append([]GitStorageChainFinding(nil), findings...) + } + filtered := make([]GitStorageChainFinding, 0, len(findings)) + for _, finding := range findings { + if finding.Kind == kind { + filtered = append(filtered, finding) + } + } + return filtered +} + +func filterStorageChainSummary(summary GitStorageChainAuditSummary, findings []GitStorageChainFinding) GitStorageChainAuditSummary { + filteredCounts := make(map[string]int, len(summary.CountsByKind)) + for kind := range summary.CountsByKind { + filteredCounts[kind] = 0 + } + filteredCounts["bucket_syfon_git_complete"] = summary.CountsByKind["bucket_syfon_git_complete"] + for _, finding := range findings { + filteredCounts[finding.Kind]++ + } + summary.CountsByKind = filteredCounts + summary.TotalFindings = len(findings) + return summary +} + +func limitStorageChainFindings(findings []GitStorageChainFinding, limit int, kind string) []GitStorageChainFinding { + if limit <= 0 || limit >= len(findings) { + return append([]GitStorageChainFinding(nil), findings...) + } + if strings.TrimSpace(kind) == "" { + limited := make([]GitStorageChainFinding, 0, len(findings)) + countsByKind := make(map[string]int) + for _, finding := range findings { + if countsByKind[finding.Kind] >= limit { + continue + } + limited = append(limited, finding) + countsByKind[finding.Kind]++ + } + return limited + } + return append([]GitStorageChainFinding(nil), findings[:limit]...) +} + +func (service *StorageAnalyticsService) ApplyStorageCleanup(ctx context.Context, authorizationHeader string, organization string, project string, selectedRepoPaths []string, selectedActions []GitStorageCleanupApplyAction, selectedFindings []GitStorageCleanupApplyFinding, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool, dryRun bool) (*GitStorageCleanupApplyResponse, error) { + if len(selectedFindings) == 0 { + return nil, fmt.Errorf("cleanup apply requires findings from a prior audit; refusing to rebuild audit during apply") + } + canonicalFindings, err := service.canonicalizeStorageCleanupFindings(ctx, authorizationHeader, organization, project, selectedFindings) + if err != nil { + return nil, err + } + actionSelection := indexCleanupActions(selectedActions) + plan := storageCleanupApplyPlan{ + UpdateAccessMethods: make(map[string][]gintegrationsyfon.ProjectAccessMethod), + } + selected := indexCleanupSelection(selectedRepoPaths) + for _, finding := range canonicalFindings { + if len(selected) > 0 && !storageApplyFindingSelected(selected, finding) { + continue + } + action, err := resolveStorageCleanupApplyAction(finding, actionSelection, deleteRepoOrphans, deleteStaleDuplicates, deleteBucketOnlyObjects, repairBrokenBucketMappings) + if err != nil { + return nil, err + } + if err := addStorageCleanupApplyFindingToPlan(&plan, finding, action); err != nil { + return nil, err + } + } + if len(selectedRepoPaths) > 0 && len(plan.DeleteObjectIDs) == 0 && len(plan.DeleteStorageObjectIDs) == 0 && len(plan.DeleteBucketObjects) == 0 && len(plan.UpdateAccessMethods) == 0 && len(plan.ManualPaths) == 0 && len(plan.SkippedPaths) == 0 { + return nil, fmt.Errorf("selected cleanup paths did not match provided cleanup findings") + } + return service.executeStorageCleanupApplyPlan(ctx, authorizationHeader, organization, project, plan, dryRun) +} + +func (service *StorageAnalyticsService) canonicalizeStorageCleanupFindings(ctx context.Context, authorizationHeader string, organization string, project string, findings []GitStorageCleanupApplyFinding) ([]GitStorageCleanupApplyFinding, error) { + if !storageCleanupFindingsContainStorageURL(findings) { + return append([]GitStorageCleanupApplyFinding(nil), findings...), nil + } + scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("load project storage scopes for cleanup apply: %w", err) + } + out := make([]GitStorageCleanupApplyFinding, 0, len(findings)) + for _, finding := range findings { + clone := finding + clone.BucketObjectURL = canonicalizeCleanupStorageURL(finding.BucketObjectURL, scopes, organization, project) + clone.BucketObjectURLs = canonicalizeCleanupStorageURLs(finding.BucketObjectURLs, scopes, organization, project) + clone.AccessURLs = canonicalizeCleanupStorageURLs(finding.AccessURLs, scopes, organization, project) + if finding.Evidence != nil { + evidence := *finding.Evidence + evidence.BucketObjectURLs = canonicalizeCleanupStorageURLs(evidence.BucketObjectURLs, scopes, organization, project) + evidence.AccessURLs = canonicalizeCleanupStorageURLs(evidence.AccessURLs, scopes, organization, project) + clone.Evidence = &evidence + } + out = append(out, clone) + } + return out, nil +} + +func storageCleanupFindingsContainStorageURL(findings []GitStorageCleanupApplyFinding) bool { + for _, finding := range findings { + candidates := []string{finding.NormalizedPath, finding.BucketObjectURL} + candidates = append(candidates, finding.BucketObjectURLs...) + candidates = append(candidates, finding.AccessURLs...) + if finding.Evidence != nil { + candidates = append(candidates, finding.Evidence.BucketObjectURLs...) + candidates = append(candidates, finding.Evidence.AccessURLs...) + } + for _, candidate := range candidates { + if _, _, ok := parseStorageURL(candidate); ok { + return true + } + } + } + return false +} + +func canonicalizeCleanupStorageURL(value string, scopes []domain.StorageBucketScope, organization string, project string) string { + if objectURL := canonicalizeScopedStorageURL(value, scopes, organization, project); objectURL != "" { + return objectURL + } + if objectURL := canonicalStorageURL("", "", value); objectURL != "" { + return objectURL + } + return strings.TrimSpace(value) +} + +func canonicalizeCleanupStorageURLs(values []string, scopes []domain.StorageBucketScope, organization string, project string) []string { + out := make([]string, 0, len(values)) + for _, value := range values { + if canonical := canonicalizeCleanupStorageURL(value, scopes, organization, project); canonical != "" { + out = append(out, canonical) + } + } + return uniqueStrings(out) +} + +func resolveStorageCleanupApplyAction(finding GitStorageCleanupApplyFinding, actionSelection map[string]string, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool) (string, error) { + kind := strings.TrimSpace(finding.Kind) + if !knownStorageRepairKind(kind) { + return "", fmt.Errorf("unsupported cleanup finding kind %q", kind) + } + action := cleanupActionForApplyFinding(actionSelection, finding) + if action == "" && strings.TrimSpace(kind) == "probe_error" { + return storageActionInspectEvidence, nil + } + if action == "" { + action = strings.TrimSpace(finding.SuggestedAction) + } + if action == "" { + action = legacyDefaultStorageCleanupAction(kind, deleteRepoOrphans, deleteStaleDuplicates, deleteBucketOnlyObjects, repairBrokenBucketMappings) + } + if action == "" { + action = strings.TrimSpace(finding.DefaultAction) + } + if action == "" { + action = storageRepairPolicyForKind(kind).defaultAction + } + if action == storageActionInspectEvidence { + return action, nil + } + if !storageRepairActionAllowed(kind, action) { + return "", fmt.Errorf("cleanup action %q is not supported for finding kind %q", action, kind) + } + return action, nil +} + +func knownStorageRepairKind(kind string) bool { + switch strings.TrimSpace(kind) { + case "bucket_only_object", "bucket_syfon_no_git", + "repo_orphan_live_object", "repo_orphan_stale_record", + "stale_duplicate_record", "live_duplicate_conflict", + "broken_access_url_error", "broken_bucket_mapping", "syfon_broken_bucket_mapping", + "storage_validation_mismatch", "git_syfon_metadata_mismatch", + "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object", + "git_only_no_syfon", "storage_probe_error", "probe_error": + return true + default: + return false + } +} + +func legacyDefaultStorageCleanupAction(kind string, deleteRepoOrphans bool, deleteStaleDuplicates bool, deleteBucketOnlyObjects bool, repairBrokenBucketMappings bool) string { + switch strings.TrimSpace(kind) { + case "repo_orphan_live_object", "repo_orphan_stale_record": + if deleteRepoOrphans { + return storageRepairPolicyForKind(kind).defaultAction + } + case "bucket_syfon_no_git": + if deleteRepoOrphans { + return storageActionDeleteBoth + } + case "stale_duplicate_record": + if deleteStaleDuplicates { + return storageActionDeleteSyfonRecord + } + case "bucket_only_object": + if deleteBucketOnlyObjects { + return storageActionDeleteBucketObject + } + case "broken_access_url_error", "broken_bucket_mapping", "syfon_broken_bucket_mapping": + if repairBrokenBucketMappings { + return storageActionRemoveBrokenAccessURLs + } + } + return "" +} + +func storageRepairActionAllowed(kind string, action string) bool { + for _, allowed := range storageRepairPolicyForKind(kind).actions { + if action == allowed { + return true + } + } + return false +} + +func stringSliceContains(values []string, target string) bool { + for _, value := range values { + if strings.TrimSpace(value) == target { + return true + } + } + return false +} + +func addStorageCleanupApplyFindingToPlan(plan *storageCleanupApplyPlan, finding GitStorageCleanupApplyFinding, action string) error { + if err := appendStorageCleanupVerification(plan, finding, action); err != nil { + return err + } + switch action { + case storageActionInspectEvidence: + plan.ManualPaths = append(plan.ManualPaths, finding.NormalizedPath) + case storageActionDeleteSyfonRecord: + objectIDs := storageApplyFindingObjectIDs(finding) + if len(objectIDs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing object_ids for %s", finding.Kind, finding.NormalizedPath, action) + } + plan.DeleteObjectIDs = append(plan.DeleteObjectIDs, objectIDs...) + plan.RepoDeletePaths = append(plan.RepoDeletePaths, finding.NormalizedPath) + case storageActionDeleteBucketObject: + bucketURLs := storageApplyFindingBucketObjectURLs(finding) + if len(bucketURLs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing bucket object URLs for %s", finding.Kind, finding.NormalizedPath, action) + } + plan.DeleteBucketObjects = append(plan.DeleteBucketObjects, bucketURLs...) + case storageActionDeleteBoth: + objectIDs := storageApplyFindingObjectIDs(finding) + bucketURLs := storageApplyFindingBucketObjectURLs(finding) + if len(objectIDs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing object_ids for %s", finding.Kind, finding.NormalizedPath, action) + } + if strings.TrimSpace(finding.Kind) == "repo_orphan_live_object" { + plan.DeleteObjectIDs = append(plan.DeleteObjectIDs, objectIDs...) + plan.DeleteStorageObjectIDs = append(plan.DeleteStorageObjectIDs, objectIDs...) + plan.RepoDeletePaths = append(plan.RepoDeletePaths, finding.NormalizedPath) + return nil + } + if len(bucketURLs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing bucket object URLs for %s", finding.Kind, finding.NormalizedPath, action) + } + plan.DeleteObjectIDs = append(plan.DeleteObjectIDs, objectIDs...) + plan.DeleteBucketObjects = append(plan.DeleteBucketObjects, bucketURLs...) + plan.RepoDeletePaths = append(plan.RepoDeletePaths, finding.NormalizedPath) + case storageActionRemoveBrokenAccessURLs: + return addAccessMethodRepairToPlan(plan, finding) + default: + return fmt.Errorf("unsupported cleanup action %q for finding kind %q", action, finding.Kind) + } + return nil +} + +func storageApplyFindingObjectIDs(finding GitStorageCleanupApplyFinding) []string { + objectIDs := append([]string(nil), finding.ObjectIDs...) + for _, record := range finding.Records { + objectIDs = append(objectIDs, record.ObjectID) + } + return uniqueStrings(objectIDs) +} + +func addAccessMethodRepairToPlan(plan *storageCleanupApplyPlan, finding GitStorageCleanupApplyFinding) error { + records := finding.Records + if len(records) == 0 { + for _, objectID := range finding.ObjectIDs { + records = append(records, GitStorageCleanupRecordAudit{ + ObjectID: objectID, + AccessURLs: append([]string(nil), finding.AccessURLs...), + }) + } + } + if len(records) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing records for %s", finding.Kind, finding.NormalizedPath, storageActionRemoveBrokenAccessURLs) + } + for _, record := range records { + objectID := strings.TrimSpace(record.ObjectID) + if objectID == "" { + return fmt.Errorf("cleanup finding %q at %q has a record without object_id", finding.Kind, finding.NormalizedPath) + } + if len(brokenAccessURLsForRecord(record)) == 0 { + return fmt.Errorf("cleanup finding %q at %q record %q is missing broken access URL evidence", finding.Kind, finding.NormalizedPath, objectID) + } + remaining := remainingAccessMethodsAfterBrokenRemoval(record) + if len(remaining) == 0 { + plan.DeleteObjectIDs = append(plan.DeleteObjectIDs, objectID) + plan.RepoDeletePaths = append(plan.RepoDeletePaths, finding.NormalizedPath) + continue + } + plan.UpdateAccessMethods[objectID] = remaining + plan.UpdatedRecordIDs = append(plan.UpdatedRecordIDs, objectID) + } + return nil +} + +func remainingAccessMethodsAfterBrokenRemoval(record GitStorageCleanupRecordAudit) []gintegrationsyfon.ProjectAccessMethod { + methods := projectAccessMethodsFromCleanupMethods(record.AccessMethods) + if len(methods) == 0 { + for _, accessURL := range record.AccessURLs { + if trimmed := strings.TrimSpace(accessURL); trimmed != "" { + methods = append(methods, gintegrationsyfon.ProjectAccessMethod{URL: trimmed}) + } + } + } + brokenURLs := brokenAccessURLsForRecord(record) + remaining := make([]gintegrationsyfon.ProjectAccessMethod, 0, len(methods)) + for _, method := range methods { + url := strings.TrimSpace(method.URL) + if url == "" { + continue + } + if _, broken := brokenURLs[normalizeCleanupSelectionKey(url)]; broken { + continue + } + remaining = append(remaining, method) + } + return appendReplacementAccessMethods(remaining, record.AccessProbes, brokenURLs) +} + +func brokenAccessURLsForRecord(record GitStorageCleanupRecordAudit) map[string]struct{} { + broken := make(map[string]struct{}) + for _, probe := range record.AccessProbes { + url := normalizeCleanupSelectionKey(probe.URL) + if url == "" { + continue + } + if accessProbeIsBroken(probe) { + broken[url] = struct{}{} + } + } + for _, accessURL := range record.AccessURLs { + if strings.TrimSpace(accessURL) == "" { + continue + } + if len(record.AccessProbes) == 0 && recordStatusMeansBrokenAccess(record.Status) { + broken[normalizeCleanupSelectionKey(accessURL)] = struct{}{} + } + } + return broken +} + +func appendReplacementAccessMethods(existing []gintegrationsyfon.ProjectAccessMethod, probes []GitStorageCleanupAccessProbe, brokenURLs map[string]struct{}) []gintegrationsyfon.ProjectAccessMethod { + seen := make(map[string]struct{}, len(existing)+len(probes)) + out := make([]gintegrationsyfon.ProjectAccessMethod, 0, len(existing)+len(probes)) + for _, method := range existing { + url := strings.TrimSpace(method.URL) + if url == "" { + continue + } + key := normalizeCleanupSelectionKey(url) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, method) + } + for _, probe := range probes { + if strings.TrimSpace(probe.Status) != "present" || !probeValidationMatched(probe.ValidationStatus) { + continue + } + replacementURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.URL) + if replacementURL == "" { + continue + } + key := normalizeCleanupSelectionKey(replacementURL) + if _, broken := brokenURLs[key]; broken { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, gintegrationsyfon.ProjectAccessMethod{ + AccessID: "s3", + Type: "s3", + URL: replacementURL, + }) + } + return out +} + +func probeValidationMatched(status string) bool { + switch strings.TrimSpace(status) { + case "", "not_requested", "matched": + return true + default: + return false + } +} + +func recordStatusMeansBrokenAccess(status string) bool { + switch strings.TrimSpace(status) { + case "missing", "error": + return true + default: + return false + } +} + +func accessProbeIsBroken(probe GitStorageCleanupAccessProbe) bool { + switch strings.TrimSpace(probe.ErrorKind) { + case "missing_access_url", "credential_missing": + return true + } + switch strings.TrimSpace(probe.Status) { + case "missing", "forbidden", "unsupported", "invalid", "error": + return true + } + return false +} + +func (service *StorageAnalyticsService) executeStorageCleanupApplyPlan(ctx context.Context, authorizationHeader string, organization string, project string, plan storageCleanupApplyPlan, dryRun bool) (*GitStorageCleanupApplyResponse, error) { + toDelete := uniqueStrings(plan.DeleteObjectIDs) + toDeleteWithStorage := uniqueStrings(plan.DeleteStorageObjectIDs) + toDeleteMetadataOnly := differenceStrings(toDelete, toDeleteWithStorage) + toDeleteBucketObjects := uniqueStrings(plan.DeleteBucketObjects) + repoDeletePaths := uniqueStrings(plan.RepoDeletePaths) + deletedBucketObjectURLs := make([]string, 0) + updatedRecordIDs := uniqueStrings(plan.UpdatedRecordIDs) + manualPaths := uniqueStrings(plan.ManualPaths) + skippedPaths := uniqueStrings(plan.SkippedPaths) + purgeResults := make([]GitStorageCleanupPurgeResult, 0, len(toDelete)+len(toDeleteBucketObjects)) + if dryRun { + for _, objectID := range toDelete { + purgeResults = append(purgeResults, GitStorageCleanupPurgeResult{ + ObjectID: objectID, + Success: nil, + Status: "dry_run", + }) + } + return &GitStorageCleanupApplyResponse{ + DeletedRecordIDs: toDelete, + DeletedBucketObjectURLs: toDeleteBucketObjects, + UpdatedRecordIDs: updatedRecordIDs, + PurgeResults: purgeResults, + RepoDeletePaths: repoDeletePaths, + ManualPaths: manualPaths, + SkippedPaths: skippedPaths, + DryRun: true, + }, nil + } + if err := service.verifyStorageCleanupApplyPlan(ctx, authorizationHeader, &plan); err != nil { + return nil, err + } + toDeleteBucketObjects = uniqueStrings(plan.DeleteBucketObjects) + skippedPaths = uniqueStrings(plan.SkippedPaths) + if len(plan.UpdateAccessMethods) > 0 { + if err := service.storage.BulkUpdateAccessMethods(ctx, authorizationHeader, plan.UpdateAccessMethods); err != nil { + return nil, fmt.Errorf("update syfon access methods: %w", err) + } + } + if len(toDeleteMetadataOnly) > 0 { + if err := service.storage.BulkDeleteObjects(ctx, authorizationHeader, toDeleteMetadataOnly, false); err != nil { + return nil, fmt.Errorf("delete syfon objects: %w", err) + } + } + if len(toDeleteWithStorage) > 0 { + if err := service.storage.BulkDeleteObjects(ctx, authorizationHeader, toDeleteWithStorage, true); err != nil { + return nil, fmt.Errorf("delete syfon objects: %w", err) + } + } + if len(toDeleteBucketObjects) > 0 { + results, err := service.storage.DeleteProjectBucketObjects(ctx, authorizationHeader, organization, project, toDeleteBucketObjects) + if err != nil { + return nil, fmt.Errorf("delete syfon project bucket objects: %w", err) + } + for _, result := range results { + if strings.EqualFold(strings.TrimSpace(result.Status), "deleted") { + deletedBucketObjectURLs = append(deletedBucketObjectURLs, result.ObjectURL) + continue + } + purgeResults = append(purgeResults, GitStorageCleanupPurgeResult{ + ObjectID: result.ObjectURL, + Success: boolPtr(false), + Status: strings.TrimSpace(result.Status), + Error: strings.TrimSpace(result.Error), + }) + } + deletedBucketObjectURLs = uniqueStrings(deletedBucketObjectURLs) + } + if len(plan.UpdateAccessMethods) > 0 || len(toDelete) > 0 || len(toDeleteBucketObjects) > 0 { + service.evictProjectJoinCache(organization, project) + service.evictProjectAuditRecordCache(organization, project) + service.evictProjectChainInputCache(organization, project) + } + for _, objectID := range toDelete { + success := true + purgeResults = append(purgeResults, GitStorageCleanupPurgeResult{ + ObjectID: objectID, + Success: &success, + Status: "deleted", + }) + } + return &GitStorageCleanupApplyResponse{ + DeletedRecordIDs: toDelete, + DeletedBucketObjectURLs: deletedBucketObjectURLs, + UpdatedRecordIDs: updatedRecordIDs, + PurgeResults: purgeResults, + RepoDeletePaths: repoDeletePaths, + ManualPaths: manualPaths, + SkippedPaths: skippedPaths, + DryRun: false, + }, nil +} + +func indexCleanupSelection(paths []string) map[string]struct{} { + index := make(map[string]struct{}, len(paths)) + for _, path := range paths { + normalized := normalizeCleanupSelectionKey(path) + if normalized == "" { + continue + } + index[normalized] = struct{}{} + } + return index +} + +func storageApplyFindingSelected(selected map[string]struct{}, finding GitStorageCleanupApplyFinding) bool { + for _, candidate := range storageApplyFindingSelectionCandidates(finding) { + if _, ok := selected[candidate]; ok { + return true + } + } + return false +} + +func storageApplyFindingSelectionCandidates(finding GitStorageCleanupApplyFinding) []string { + candidates := []string{ + finding.NormalizedPath, + finding.BucketObjectURL, + } + candidates = append(candidates, finding.ObjectIDs...) + candidates = append(candidates, finding.BucketObjectURLs...) + candidates = append(candidates, finding.AccessURLs...) + if finding.Evidence != nil { + candidates = append(candidates, finding.Evidence.AccessURLs...) + candidates = append(candidates, finding.Evidence.BucketObjectURLs...) + candidates = append(candidates, finding.Evidence.ObjectIDs...) + } + return uniqueCleanupSelectionCandidates(candidates) +} + +func storageApplyFindingBucketObjectURLs(finding GitStorageCleanupApplyFinding) []string { + candidates := []string{ + finding.BucketObjectURL, + } + candidates = append(candidates, finding.BucketObjectURLs...) + candidates = append(candidates, finding.AccessURLs...) + if finding.Evidence != nil { + candidates = append(candidates, finding.Evidence.BucketObjectURLs...) + candidates = append(candidates, finding.Evidence.AccessURLs...) + } + if len(uniqueCleanupSelectionCandidates(candidates)) == 0 && canonicalStorageURL("", "", finding.NormalizedPath) != "" { + candidates = append(candidates, finding.NormalizedPath) + } + return uniqueCleanupSelectionCandidates(candidates) +} + +func cleanupActionForApplyFinding(index map[string]string, finding GitStorageCleanupApplyFinding) string { + for _, path := range storageApplyFindingSelectionCandidates(finding) { + if action := strings.TrimSpace(index[cleanupActionKey(finding.Kind, path)]); action != "" { + return action + } + if action := strings.TrimSpace(index[cleanupActionKey("", path)]); action != "" { + return action + } + } + return "" +} + +func indexCleanupActions(actions []GitStorageCleanupApplyAction) map[string]string { + if len(actions) == 0 { + return nil + } + index := make(map[string]string, len(actions)) + for _, action := range actions { + if strings.TrimSpace(action.Action) == "" { + continue + } + path := normalizeRepoSubpath(action.NormalizedPath) + if path == "" { + continue + } + key := cleanupActionKey(strings.TrimSpace(action.Kind), path) + index[key] = strings.TrimSpace(action.Action) + } + return index +} + +func cleanupActionKey(kind string, path string) string { + return strings.TrimSpace(kind) + "::" + normalizeRepoSubpath(path) +} + +func (service *StorageAnalyticsService) loadJoinState(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, forceRefresh bool) (*repoAnalyticsIndex, []RepoInventoryFile, map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { + index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) + if err != nil { + return nil, nil, nil, nil, err + } + inventory, err := filterRepoInventoryFiles(index, gitSubpath) + if err != nil { + return nil, nil, nil, nil, err + } + recordsByChecksum, usageByObjectID, err := service.loadProjectJoinCache(ctx, authorizationHeader, organization, project, hash, index.sidecar.Files, forceRefresh) + if err != nil { + return nil, nil, nil, nil, err + } + return index, inventory, recordsByChecksum, usageByObjectID, nil +} + +func (service *StorageAnalyticsService) loadProjectJoinCache(ctx context.Context, authorizationHeader string, organization string, project string, hash plumbing.Hash, inventory []RepoInventoryFile, forceRefresh bool) (map[string][]projectRecordState, map[string]gintegrationsyfon.FileUsage, error) { + cacheKey := service.projectJoinCacheKey(organization, project, hash.String()) + redisKey := storageExactProjectJoinCacheKey(organization, project, hash.String()) + service.projectJoinMu.Lock() + if !forceRefresh { + if cached, ok := service.projectJoinCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) { + service.projectJoinMu.Unlock() + log.Printf("storage_folder_exact_join_cache_hit org=%s project=%s source=memory records_by_checksum=%d usage_rows=%d", organization, project, len(cached.recordsByChecksum), len(cached.usageByObjectID)) + return cloneRecordStateMap(cached.recordsByChecksum), cloneFileUsageMap(cached.usageByObjectID), nil + } + } + if inflight, ok := service.projectJoinWork[cacheKey]; ok { + service.projectJoinMu.Unlock() + <-inflight.done + return cloneRecordStateMap(inflight.recordsByChecksum), cloneFileUsageMap(inflight.usageByObjectID), inflight.err + } + inflight := &inflightProjectJoinState{done: make(chan struct{})} + service.projectJoinWork[cacheKey] = inflight + service.projectJoinMu.Unlock() + defer func() { + service.projectJoinMu.Lock() + delete(service.projectJoinWork, cacheKey) + close(inflight.done) + service.projectJoinMu.Unlock() + }() + + if !forceRefresh && service.exactProjectJoinCache != nil { + redisStart := time.Now() + cached, ok, err := service.exactProjectJoinCache.Get(ctx, redisKey) + if err != nil { + log.Printf("storage_folder_exact_join_cache_error org=%s project=%s source=%s operation=get error=%q", organization, project, service.exactProjectJoinCache.Source(), err.Error()) + } + if ok { + state := cachedProjectJoinState{ + expiresAt: time.Now().Add(projectJoinCacheTTL), + recordsByChecksum: cloneRecordStateMap(cached.RecordsByChecksum), + usageByObjectID: cloneFileUsageMap(cached.UsageByObjectID), + } + service.projectJoinMu.Lock() + service.projectJoinCache[cacheKey] = state + service.projectJoinMu.Unlock() + log.Printf("storage_folder_exact_join_cache_hit org=%s project=%s source=%s age_seconds=%d records_by_checksum=%d usage_rows=%d duration_ms=%d", organization, project, service.exactProjectJoinCache.Source(), int64(time.Since(cached.CachedAt).Seconds()), len(cached.RecordsByChecksum), len(cached.UsageByObjectID), time.Since(redisStart).Milliseconds()) + inflight.recordsByChecksum = cloneRecordStateMap(cached.RecordsByChecksum) + inflight.usageByObjectID = cloneFileUsageMap(cached.UsageByObjectID) + return cloneRecordStateMap(cached.RecordsByChecksum), cloneFileUsageMap(cached.UsageByObjectID), nil + } + log.Printf("storage_folder_exact_join_cache_miss org=%s project=%s source=%s duration_ms=%d", organization, project, service.exactProjectJoinCache.Source(), time.Since(redisStart).Milliseconds()) + } + + recordStart := time.Now() + projectRecords, err := service.loadCachedProjectAuditRecords(ctx, authorizationHeader, organization, project) + if err != nil { + inflight.err = err + return nil, nil, inflight.err + } + checksums := inventoryChecksumSet(inventory) + matchedRecords := filterProjectRecordsByChecksum(projectRecords, checksums) + log.Printf("storage_folder_exact_syfon_local_join org=%s project=%s git_checksums=%d project_records=%d matched_records=%d duration_ms=%d", organization, project, len(checksums), len(projectRecords), len(matchedRecords), time.Since(recordStart).Milliseconds()) + + usageStart := time.Now() + objectIDs := projectRecordObjectIDs(matchedRecords) + usageByObjectID, err := service.listProjectFileUsageByObjectIDs(ctx, authorizationHeader, organization, project, objectIDs, cleanupInactiveDays) + if err != nil { + inflight.err = fmt.Errorf("list syfon project file usage: %w", err) + return nil, nil, inflight.err + } + log.Printf("storage_folder_exact_bulk_usage_lookup org=%s project=%s object_ids=%d usage_rows=%d duration_ms=%d", organization, project, len(objectIDs), len(usageByObjectID), time.Since(usageStart).Milliseconds()) + + recordsByChecksum := buildProjectJoinRecordsByChecksum(matchedRecords, usageByObjectID) + service.projectJoinMu.Lock() + service.projectJoinCache[cacheKey] = cachedProjectJoinState{ + expiresAt: time.Now().Add(projectJoinCacheTTL), + recordsByChecksum: cloneRecordStateMap(recordsByChecksum), + usageByObjectID: cloneFileUsageMap(usageByObjectID), + } + service.projectJoinMu.Unlock() + if service.exactProjectJoinCache != nil { + setStart := time.Now() + err := service.exactProjectJoinCache.Set(ctx, redisKey, cachedExactProjectJoinState{ + CachedAt: time.Now(), + RecordsByChecksum: recordsByChecksum, + UsageByObjectID: usageByObjectID, + }, storageChainAuditCacheTTL()) + if err != nil { + log.Printf("storage_folder_exact_join_cache_error org=%s project=%s source=%s operation=set error=%q", organization, project, service.exactProjectJoinCache.Source(), err.Error()) + } else { + log.Printf("storage_folder_exact_join_cache_store org=%s project=%s source=%s records_by_checksum=%d usage_rows=%d duration_ms=%d", organization, project, service.exactProjectJoinCache.Source(), len(recordsByChecksum), len(usageByObjectID), time.Since(setStart).Milliseconds()) + } + } + inflight.recordsByChecksum = cloneRecordStateMap(recordsByChecksum) + inflight.usageByObjectID = cloneFileUsageMap(usageByObjectID) + return recordsByChecksum, usageByObjectID, nil +} + +func inventoryChecksumSet(inventory []RepoInventoryFile) map[string]struct{} { + checksums := make(map[string]struct{}, len(inventory)) + for _, item := range inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" { + continue + } + checksums[checksum] = struct{}{} + } + return checksums +} + +func filterProjectRecordsByChecksum(records []gintegrationsyfon.ProjectRecord, checksums map[string]struct{}) []gintegrationsyfon.ProjectRecord { + out := make([]gintegrationsyfon.ProjectRecord, 0) + for _, record := range records { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + if _, ok := checksums[normalizedChecksum]; !ok { + continue + } + record.Checksum = normalizedChecksum + out = append(out, record) + } + return out +} + +func projectRecordObjectIDs(records []gintegrationsyfon.ProjectRecord) []string { + objectIDs := make([]string, 0, len(records)) + for _, record := range records { + objectIDs = append(objectIDs, record.ObjectID) + } + return uniqueStrings(objectIDs) +} + +func buildProjectJoinRecordsByChecksum(records []gintegrationsyfon.ProjectRecord, usageByObjectID map[string]gintegrationsyfon.FileUsage) map[string][]projectRecordState { + recordsByChecksum := make(map[string][]projectRecordState) + for _, record := range records { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + record.Checksum = normalizedChecksum + recordsByChecksum[normalizedChecksum] = append(recordsByChecksum[normalizedChecksum], projectRecordState{ + ProjectRecord: record, + Usage: usageByObjectID[record.ObjectID], + }) + } + return recordsByChecksum +} + +func (service *StorageAnalyticsService) listProjectFileUsageByObjectIDs(ctx context.Context, authorizationHeader string, organization string, project string, objectIDs []string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) { + usageByObjectID := make(map[string]gintegrationsyfon.FileUsage) + if len(objectIDs) == 0 { + return usageByObjectID, nil + } + for start := 0; start < len(objectIDs); start += projectFileUsageBulkChunkSize { + end := start + projectFileUsageBulkChunkSize + if end > len(objectIDs) { + end = len(objectIDs) + } + chunkUsage, err := service.storage.ListProjectFileUsageByObjectIDs(ctx, authorizationHeader, organization, project, objectIDs[start:end], inactiveDays) + if err != nil { + if gintegrationsyfon.IsHTTPStatus(err, http.StatusNotFound, http.StatusMethodNotAllowed) { + return service.storage.ListProjectFileUsage(ctx, authorizationHeader, organization, project, inactiveDays) + } + return nil, err + } + for objectID, usage := range chunkUsage { + usageByObjectID[objectID] = usage + } + } + return usageByObjectID, nil +} + +func (service *StorageAnalyticsService) projectJoinCacheKey(organization string, project string, commitHash string) string { + return strings.TrimSpace(organization) + "/" + strings.TrimSpace(project) + "::" + strings.TrimSpace(commitHash) +} + +func (service *StorageAnalyticsService) evictProjectJoinCache(organization string, project string) { + prefix := strings.TrimSpace(organization) + "/" + strings.TrimSpace(project) + "::" + service.projectJoinMu.Lock() + defer service.projectJoinMu.Unlock() + for key := range service.projectJoinCache { + if strings.HasPrefix(key, prefix) { + delete(service.projectJoinCache, key) + } + } +} + +func (service *StorageAnalyticsService) evictProjectAuditRecordCache(organization string, project string) { + cacheKey := service.projectChainInputCacheKey(organization, project) + service.chainInputMu.Lock() + delete(service.projectAuditCache, cacheKey) + service.chainInputMu.Unlock() +} + +func (service *StorageAnalyticsService) listProjectRecordStates(ctx context.Context, authorizationHeader string, organization string, project string, usageByObjectID map[string]gintegrationsyfon.FileUsage) (map[string][]projectRecordState, error) { + projectRecords, err := service.storage.ListProjectRecords(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("list syfon project records: %w", err) + } + out := make(map[string][]projectRecordState) + for _, record := range projectRecords { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + record.Checksum = normalizedChecksum + out[normalizedChecksum] = append(out[normalizedChecksum], projectRecordState{ + ProjectRecord: record, + Usage: usageByObjectID[record.ObjectID], + }) + } + return out, nil +} + +func (service *StorageAnalyticsService) loadProjectStorageScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + buckets, err := service.storage.ListBuckets(ctx, authorizationHeader) + if err != nil { + return nil, fmt.Errorf("list syfon buckets: %w", err) + } + bucketNames := make([]string, 0, len(buckets)) + for bucket := range buckets { + bucketNames = append(bucketNames, bucket) + } + sort.Strings(bucketNames) + scopes := make([]domain.StorageBucketScope, 0) + for _, bucket := range bucketNames { + items, err := service.storage.ListBucketScopes(ctx, authorizationHeader, bucket) + if err != nil { + return nil, fmt.Errorf("list syfon bucket scopes for %s: %w", bucket, err) + } + for _, scope := range items { + if !strings.EqualFold(strings.TrimSpace(scope.Organization), organization) { + continue + } + scopeProject := strings.TrimSpace(scope.ProjectID) + if scopeProject != "" && !strings.EqualFold(scopeProject, project) { + continue + } + scopes = append(scopes, scope) + } + } + sort.SliceStable(scopes, func(i, j int) bool { + iProject := strings.TrimSpace(scopes[i].ProjectID) + jProject := strings.TrimSpace(scopes[j].ProjectID) + if iProject == "" && jProject != "" { + return true + } + if iProject != "" && jProject == "" { + return false + } + if scopes[i].Bucket != scopes[j].Bucket { + return scopes[i].Bucket < scopes[j].Bucket + } + return scopes[i].Path < scopes[j].Path + }) + return scopes, nil +} + +func applyScopedStorageMappings(recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, scopes []domain.StorageBucketScope, organization string, project string) (map[string][]projectRecordState, map[string][]projectRecordState) { + attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + clone.CanonicalAccessURLs = canonicalizeRecordAccessURLs(record.AccessURLs, scopes, organization, project) + clone.CanonicalAccessURLByRaw = canonicalizeRecordAccessURLMappings(record.AccessURLs, scopes, organization, project) + states = append(states, clone) + } + out[checksum] = states + } + return out + } + return attach(recordsByChecksum), attach(allProjectRecords) +} + +func (service *StorageAnalyticsService) attachStorageProbes(ctx context.Context, authorizationHeader string, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState) (map[string][]projectRecordState, map[string][]projectRecordState, error) { + return service.attachStorageValidationResults(ctx, authorizationHeader, recordsByChecksum, allProjectRecords, false) +} + +func (service *StorageAnalyticsService) attachStorageListValidations(ctx context.Context, authorizationHeader string, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState) (map[string][]projectRecordState, map[string][]projectRecordState, error) { + return service.attachStorageValidationResults(ctx, authorizationHeader, recordsByChecksum, allProjectRecords, true) +} + +func (service *StorageAnalyticsService) attachStorageValidationResults(ctx context.Context, authorizationHeader string, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, useListValidation bool) (map[string][]projectRecordState, map[string][]projectRecordState, error) { + started := time.Now() + items := make([]gintegrationsyfon.BulkStorageProbeItem, 0) + itemKeys := map[string]string{} + recordProbeKeysByObjectID := map[string][]string{} + accessURLCount := 0 + + for _, group := range allProjectRecords { + for _, record := range group { + for _, accessURL := range probeAccessURLsForRecord(record) { + accessURLCount++ + normalizedURL := strings.TrimSpace(accessURL) + if normalizedURL == "" { + continue + } + key := storageProbeRequestKey(normalizedURL, record.Size, record.Checksum) + expectedName := "" + if useListValidation { + expectedName = expectedStorageObjectNameForListValidation(normalizedURL, record.Name) + key = storageListValidationRequestKey(normalizedURL, record.Size, expectedName) + } + recordProbeKeysByObjectID[record.ObjectID] = append(recordProbeKeysByObjectID[record.ObjectID], key) + if _, ok := itemKeys[key]; ok { + continue + } + itemKeys[key] = key + expectedSize := record.Size + item := gintegrationsyfon.BulkStorageProbeItem{ + ID: key, + ObjectURL: normalizedURL, + ExpectedSizeBytes: &expectedSize, + ExpectedSHA256: strings.TrimSpace(record.Checksum), + } + if useListValidation { + item.ExpectedSHA256 = "" + item.ExpectedName = expectedName + } + items = append(items, item) + if useListValidation && len(items) <= storageChainValidationDebugSampleLimit { + log.Printf( + "INFO: storage_chain_bulk_list_item object_id=%s name=%q checksum=%s submitted_url=%s raw_access_urls=%q canonical_access_urls=%q expected_size=%d expected_name=%q", + strings.TrimSpace(record.ObjectID), + strings.TrimSpace(record.Name), + strings.TrimSpace(record.Checksum), + normalizedURL, + strings.Join(rawAccessURLsForRecord(record), ","), + strings.Join(accessURLsForStorage(record), ","), + record.Size, + item.ExpectedName, + ) + } + } + } + } + mode := "head" + operation := StorageChainValidationModeMetadata + if useListValidation { + mode = "list" + operation = StorageChainValidationModeList + } + log.Printf( + "INFO: storage_chain_validation_request_built mode=%s record_count=%d access_url_count=%d unique_request_count=%d duplicate_request_count=%d build_ms=%d", + mode, + countRecordStates(allProjectRecords), + accessURLCount, + len(items), + accessURLCount-len(items), + time.Since(started).Milliseconds(), + ) + + resultsByKey := map[string]gintegrationsyfon.BulkStorageProbeResult{} + if len(items) > 0 { + var ( + results []gintegrationsyfon.BulkStorageProbeResult + err error + ) + if useListValidation { + results, err = service.storage.BulkListStorageObjects(ctx, authorizationHeader, items) + } else { + results, err = service.storage.BulkProbeStorageObjects(ctx, authorizationHeader, items) + } + if err != nil { + if useListValidation { + log.Printf("INFO: storage_chain_validation_request_done mode=%s request_count=%d duration_ms=%d error=%q", mode, len(items), time.Since(started).Milliseconds(), err.Error()) + return nil, nil, fmt.Errorf("list-validate syfon storage objects: %w", err) + } + log.Printf("INFO: storage_chain_validation_request_done mode=%s request_count=%d duration_ms=%d error=%q", mode, len(items), time.Since(started).Milliseconds(), err.Error()) + return nil, nil, fmt.Errorf("probe syfon storage objects: %w", err) + } + for _, result := range results { + result.Operation = operation + resultsByKey[strings.TrimSpace(result.ID)] = result + } + } + log.Printf("INFO: storage_chain_validation_request_done mode=%s request_count=%d result_count=%d duration_ms=%d", mode, len(items), len(resultsByKey), time.Since(started).Milliseconds()) + + attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + keys := uniqueStrings(recordProbeKeysByObjectID[record.ObjectID]) + probes := make([]gintegrationsyfon.BulkStorageProbeResult, 0, len(keys)) + for _, key := range keys { + if result, ok := resultsByKey[key]; ok { + result.Operation = operation + probes = append(probes, result) + } + } + clone.AccessProbes = append(append([]gintegrationsyfon.BulkStorageProbeResult(nil), record.AccessProbes...), probes...) + states = append(states, clone) + } + out[checksum] = states + } + return out + } + + return attach(recordsByChecksum), attach(allProjectRecords), nil +} + +func countRecordStates(input map[string][]projectRecordState) int { + count := 0 + for _, group := range input { + count += len(group) + } + return count +} + +func summarizeSubtree(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, usageByObjectID map[string]gintegrationsyfon.FileUsage, directChildCount int) storageAggregate { + agg := storageAggregate{ + path: normalizeRepoSubpath(gitSubpath), + rowType: "directory", + } + for _, item := range inventory { + agg.fileCount++ + agg.totalBytes += item.Size + matches := recordsByChecksum[normalizeAnalyticsChecksum(item.Checksum)] + agg.recordCount += len(matches) + if len(matches) > 1 { + agg.duplicateCount++ + } + for _, record := range matches { + applyUsage(&agg, record) + } + } + _ = usageByObjectID + _ = directChildCount + return agg +} + +func aggregateImmediateChildren(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, usageByObjectID map[string]gintegrationsyfon.FileUsage, aggregates []storageAggregate) []storageAggregate { + root := normalizeRepoSubpath(gitSubpath) + if aggregates == nil { + aggregates = make([]storageAggregate, 0) + } + aggregateLookup := make(map[string]*storageAggregate, len(aggregates)) + for index := range aggregates { + aggregateLookup[aggregates[index].path] = &aggregates[index] + } + for _, item := range inventory { + childName, childPath, childType := immediateChild(root, item.RepoPath) + if childPath == "" { + continue + } + wasPrecomputed := true + agg := aggregateLookup[childPath] + if agg == nil { + wasPrecomputed = false + agg = &storageAggregate{ + name: childName, + path: childPath, + rowType: childType, + } + aggregates = append(aggregates, *agg) + agg = &aggregates[len(aggregates)-1] + aggregateLookup[childPath] = agg + } + if !wasPrecomputed { + agg.fileCount++ + agg.totalBytes += item.Size + } + matches := recordsByChecksum[normalizeAnalyticsChecksum(item.Checksum)] + agg.recordCount += len(matches) + if len(matches) > 1 { + agg.duplicateCount++ + } + for _, record := range matches { + applyUsage(agg, record) + } + } + _ = usageByObjectID + return aggregates +} + +func buildProjectDiffAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState) *projectDiffAuditModel { + findings := make([]GitProjectDiffFinding, 0) + countsByKind := map[string]int{ + "duplicate_syfon_paths": 0, + "syfon_missing_in_repo": 0, + "repo_missing_in_syfon": 0, + "unknown": 0, + } + matchedPathCount := 0 + scannedRecordCount := 0 + repoChecksums := make(map[string]struct{}, len(inventory)) + for _, item := range inventory { + normalizedChecksum := normalizeAnalyticsChecksum(item.Checksum) + repoChecksums[normalizedChecksum] = struct{}{} + matches := recordsByChecksum[normalizedChecksum] + scannedRecordCount += len(matches) + if len(matches) > 0 { + matchedPathCount++ + } + if len(matches) == 0 { + evidence := buildFindingEvidence(item.Checksum, []string{item.RepoPath}, nil, "not_checked") + findings = append(findings, GitProjectDiffFinding{ + Kind: "repo_missing_in_syfon", + NormalizedPath: item.RepoPath, + Checksum: item.Checksum, + SourcePaths: []string{item.RepoPath}, + ObjectIDs: []string{}, + RecordCount: 0, + SizeBytes: item.Size, + RecommendedAction: "No Syfon record matched this Git-tracked checksum. Bucket presence is not part of this check; review ingest or metadata mapping for this path.", + Evidence: evidence, + }) + countsByKind["repo_missing_in_syfon"]++ + continue + } + if len(matches) > 1 { + evidence := buildFindingEvidence(item.Checksum, []string{item.RepoPath}, matches, "not_checked") + findings = append(findings, GitProjectDiffFinding{ + Kind: "duplicate_syfon_paths", + NormalizedPath: item.RepoPath, + Checksum: item.Checksum, + SourcePaths: recordSourcePaths(matches), + ObjectIDs: recordObjectIDs(matches), + RecordCount: len(matches), + SizeBytes: aggregateMatchedSize(matches, item.Size), + DownloadCount: aggregateMatchedDownloads(matches), + LastDownload: formatOptionalTime(latestMatchedDownload(matches)), + RecommendedAction: "Review duplicate Syfon records before deleting anything.", + Evidence: evidence, + }) + countsByKind["duplicate_syfon_paths"]++ + } + } + seenOrphanChecksums := map[string]struct{}{} + for checksum, matches := range allProjectRecords { + if len(matches) == 0 { + continue + } + if _, ok := repoChecksums[checksum]; ok { + continue + } + if _, ok := seenOrphanChecksums[checksum]; ok { + continue + } + seenOrphanChecksums[checksum] = struct{}{} + sourcePaths := recordSourcePaths(matches) + evidence := buildFindingEvidence(checksum, nil, matches, "not_checked") + findings = append(findings, GitProjectDiffFinding{ + Kind: "syfon_missing_in_repo", + NormalizedPath: orphanDisplayPath(checksum, sourcePaths), + Checksum: checksum, + SourcePaths: sourcePaths, + ObjectIDs: recordObjectIDs(matches), + RecordCount: len(matches), + SizeBytes: aggregateMatchedSize(matches, 0), + DownloadCount: aggregateMatchedDownloads(matches), + LastDownload: formatOptionalTime(latestMatchedDownload(matches)), + RecommendedAction: "Prepare delete to verify storage before removing Syfon-only records.", + Evidence: evidence, + }) + countsByKind["syfon_missing_in_repo"]++ + } + return &projectDiffAuditModel{ + Findings: findings, + Summary: GitProjectDiffSummary{ + CountsByKind: countsByKind, + TotalFindings: len(findings), + IndexedPathCount: matchedPathCount, + ExpectedPathCount: len(inventory), + MatchedPathCount: matchedPathCount, + IncludesRepoManifest: true, + ScannedRecordCount: scannedRecordCount, + }, + PathPrefix: normalizeRepoSubpath(gitSubpath), + } +} + +func buildCleanupAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, selectedRepoPaths []string, checkStorage bool) *cleanupAuditModel { + allowed := make(map[string]struct{}, len(selectedRepoPaths)) + for _, path := range selectedRepoPaths { + if normalized := normalizeCleanupSelectionKey(path); normalized != "" { + allowed[normalized] = struct{}{} + } + } + includePath := func(paths ...string) bool { + if len(allowed) == 0 { + return true + } + for _, path := range paths { + if _, ok := allowed[normalizeCleanupSelectionKey(path)]; ok { + return true + } + } + return false + } + findings := make([]cleanupFindingModel, 0) + countsByKind := map[string]int{ + "bucket_only_object": 0, + "stale_duplicate_record": 0, + "live_duplicate_conflict": 0, + "broken_access_url_error": 0, + "broken_bucket_mapping": 0, + "repo_orphan_live_object": 0, + "repo_orphan_stale_record": 0, + "storage_object_missing": 0, + "storage_validation_mismatch": 0, + "storage_probe_error": 0, + "unknown": 0, + } + repoChecksums := make(map[string]struct{}, len(inventory)) + referencedBucketURLs := map[string]struct{}{} + for _, item := range inventory { + normalizedChecksum := normalizeAnalyticsChecksum(item.Checksum) + repoChecksums[normalizedChecksum] = struct{}{} + matches := recordsByChecksum[normalizedChecksum] + if !includePath(item.RepoPath) || len(matches) == 0 { + continue + } + if len(matches) > 1 { + sortedMatches := append([]projectRecordState(nil), matches...) + sort.SliceStable(sortedMatches, func(i, j int) bool { + return compareRecordState(sortedMatches[i], sortedMatches[j]) > 0 + }) + if compareRecordState(sortedMatches[0], sortedMatches[1]) == 0 { + public := buildCleanupFinding("live_duplicate_conflict", item.RepoPath, sortedMatches, false, "record", "Manual review required for live duplicate records.") + findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + countsByKind["live_duplicate_conflict"]++ + } else { + candidates := make([]projectRecordState, 0, len(sortedMatches)-1) + deleteIDs := make([]string, 0, len(sortedMatches)-1) + for _, record := range sortedMatches[1:] { + candidates = append(candidates, record) + deleteIDs = append(deleteIDs, record.ObjectID) + } + public := buildCleanupFinding("stale_duplicate_record", item.RepoPath, sortedMatches, true, "record", "Delete stale duplicate records") + findings = append(findings, cleanupFindingModel{Public: public, DeleteObjectIDs: deleteIDs}) + countsByKind["stale_duplicate_record"]++ + } + continue + } + if checkStorage { + for _, bucketURL := range matchedBucketObjectURLs(matches[0], bucketObjectsByURL) { + referencedBucketURLs[bucketURL] = struct{}{} + } + switch storageFindingKind := classifyStorageFinding(matches[0], bucketObjectsByURL); storageFindingKind { + case storageFindingBrokenAccessURL: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Manual review required for broken access URLs") + findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + countsByKind[string(storageFindingKind)]++ + case storageFindingBrokenBucketMap: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Syfon access URL did not resolve through a configured bucket mapping.") + repairDeleteIDs, repairUpdates := brokenBucketMappingRepairPlan(matches) + findings = append(findings, cleanupFindingModel{ + Public: public, + DeleteObjectIDs: repairDeleteIDs, + UpdateAccessMethods: repairUpdates, + Manual: len(repairDeleteIDs) == 0 && len(repairUpdates) == 0, + }) + countsByKind[string(storageFindingKind)]++ + case storageFindingObjectMissing: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Storage object is missing for this Syfon access URL") + findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + countsByKind[string(storageFindingKind)]++ + case storageFindingValidationMismatch: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Mapped bucket object exists, but object evidence disagrees with the Syfon/Git record.") + findings = append(findings, cleanupFindingModel{ + Public: public, + DeleteObjectIDs: recordObjectIDs(matches), + DeleteBucketObjects: cleanupMatchedBucketObjectURLs(matches, bucketObjectsByURL), + Manual: true, + }) + countsByKind[string(storageFindingKind)]++ + case storageFindingProbeError: + public := buildCleanupFinding(string(storageFindingKind), item.RepoPath, matches, false, "access_url", "Manual review required for storage probe errors") + findings = append(findings, cleanupFindingModel{Public: public, Manual: true}) + countsByKind[string(storageFindingKind)]++ + } + } + } + for checksum, matches := range allProjectRecords { + if len(matches) == 0 { + continue + } + if _, ok := repoChecksums[checksum]; ok { + continue + } + displayPath := orphanDisplayPath(checksum, recordSourcePaths(matches)) + if !includePath(cleanupSelectionCandidatesForRecords(checksum, displayPath, matches)...) { + continue + } + kind := "repo_orphan_stale_record" + if checkStorage && recordsReferenceBucketObject(matches, bucketObjectsByURL) { + kind = "repo_orphan_live_object" + } else if recordsContainLiveUsage(matches) { + kind = "repo_orphan_live_object" + } + if checkStorage { + for _, match := range matches { + for _, bucketURL := range matchedBucketObjectURLs(match, bucketObjectsByURL) { + referencedBucketURLs[bucketURL] = struct{}{} + } + } + } + public := buildCleanupFinding(kind, displayPath, matches, true, "record", repoOrphanAction(kind)) + findings = append(findings, cleanupFindingModel{ + Public: public, + DeleteObjectIDs: recordObjectIDs(matches), + DeleteStorageData: kind == "repo_orphan_live_object", + }) + countsByKind[kind]++ + } + if checkStorage { + bucketURLs := make([]string, 0, len(bucketObjectsByURL)) + for objectURL := range bucketObjectsByURL { + bucketURLs = append(bucketURLs, objectURL) + } + sort.Strings(bucketURLs) + for _, objectURL := range bucketURLs { + if _, ok := referencedBucketURLs[objectURL]; ok { + continue + } + if !includePath(cleanupSelectionCandidatesForBucketObject(objectURL, bucketObjectsByURL[objectURL])...) { + continue + } + public := buildBucketOnlyFinding(bucketObjectsByURL[objectURL]) + findings = append(findings, cleanupFindingModel{ + Public: public, + DeleteBucketObjects: []string{objectURL}, + }) + countsByKind["bucket_only_object"]++ + } + } + sort.Slice(findings, func(i, j int) bool { + return findings[i].Public.NormalizedPath < findings[j].Public.NormalizedPath + }) + publicFindings := make([]GitStorageCleanupFinding, 0, len(findings)) + repoDeleteCandidateCount := 0 + manualFindingCount := 0 + repoOrphanCount := 0 + staleDuplicateCount := 0 + for _, finding := range findings { + publicFindings = append(publicFindings, finding.Public) + if finding.Public.RepoDeleteCandidate { + repoDeleteCandidateCount++ + } + if finding.Manual { + manualFindingCount++ + } + if finding.Public.Kind == "repo_orphan_live_object" || finding.Public.Kind == "repo_orphan_stale_record" { + repoOrphanCount++ + } + if finding.Public.Kind == "stale_duplicate_record" { + staleDuplicateCount++ + } + } + return &cleanupAuditModel{ + Findings: findings, + PublicFindings: publicFindings, + Summary: GitStorageCleanupAuditSummary{ + CountsByKind: countsByKind, + TotalFindings: len(publicFindings), + ManualFindingCount: manualFindingCount, + RepoDeleteCandidateCount: repoDeleteCandidateCount, + StaleDuplicateCount: staleDuplicateCount, + RepoOrphanCount: repoOrphanCount, + }, + ExpectedPathCount: len(inventory), + IncludesRepoManifest: true, + PathPrefix: normalizeRepoSubpath(gitSubpath), + } +} + +func normalizeCleanupSelectionKey(raw string) string { + trimmed := strings.Trim(strings.TrimSpace(raw), "/") + if trimmed == "" { + return "" + } + if objectURL := canonicalStorageURL("", "", trimmed); objectURL != "" { + return objectURL + } + return trimmed +} + +func cleanupSelectionCandidatesForRecords(checksum string, displayPath string, matches []projectRecordState) []string { + candidates := []string{displayPath} + normalizedChecksum := normalizeAnalyticsChecksum(checksum) + if normalizedChecksum != "" { + candidates = append(candidates, normalizedChecksum, "sha256/"+normalizedChecksum) + } + for _, match := range matches { + candidates = append(candidates, match.ObjectID) + candidates = append(candidates, match.AccessURLs...) + candidates = append(candidates, match.CanonicalAccessURLs...) + candidates = append(candidates, recordStorageCandidateURLs(match)...) + for _, probe := range match.AccessProbes { + candidates = append(candidates, probe.ObjectURL) + candidates = append(candidates, canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL)) + } + } + return uniqueCleanupSelectionCandidates(candidates) +} + +func cleanupSelectionCandidatesForBucketObject(objectURL string, item gintegrationsyfon.ProjectBucketObject) []string { + return uniqueCleanupSelectionCandidates([]string{ + objectURL, + item.ObjectURL, + canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL), + item.Key, + }) +} + +func uniqueCleanupSelectionCandidates(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + normalized := normalizeCleanupSelectionKey(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + +func bucketObjectHasCompleteChain(matches []projectRecordState, repoPathsByChecksum map[string][]string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + for _, match := range matches { + if classifyStorageFinding(match, bucketObjectsByURL) != storageFindingNone { + continue + } + if len(uniqueStrings(repoPathsByChecksum[normalizeAnalyticsChecksum(match.Checksum)])) > 0 { + return true + } + } + return false +} + +func buildCleanupFinding(kind string, normalizedPath string, matches []projectRecordState, repoDeleteCandidate bool, cleanupScope string, action string) GitStorageCleanupFinding { + records := make([]GitStorageCleanupRecordAudit, 0, len(matches)) + var latestUpdate *time.Time + var latestDownload *time.Time + var totalBytes int64 + var totalDownloads int64 + checksum := "" + for _, match := range matches { + if checksum == "" { + checksum = strings.TrimSpace(match.Checksum) + } + records = append(records, cleanupRecordAuditForRecord(normalizedPath, cleanupScope, match)) + totalBytes += match.Size + totalDownloads += match.Usage.DownloadCount + latestUpdate = laterTime(latestUpdate, match.UpdatedAt) + latestDownload = laterTime(latestDownload, match.Usage.LastDownloadTime) + } + bucketEvaluation := "not_checked" + if cleanupScope == "access_url" { + bucketEvaluation = "probed" + } + actionability, availableActions, defaultAction, supportsDryRun := storageCleanupActionSupport(kind) + return GitStorageCleanupFinding{ + Kind: kind, + NormalizedPath: normalizedPath, + Checksum: checksum, + ObjectIDs: recordObjectIDs(matches), + Records: records, + RecommendedAction: action, + RepoDeleteCandidate: repoDeleteCandidate, + CleanupScope: cleanupScope, + SizeBytes: totalBytes, + LastUpdated: formatOptionalTime(latestUpdate), + DownloadCount: totalDownloads, + LastDownload: formatOptionalTime(latestDownload), + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, + Evidence: buildFindingEvidence(checksum, nil, matches, bucketEvaluation), + } +} + +func cleanupRecordAuditForRecord(normalizedPath string, cleanupScope string, record projectRecordState) GitStorageCleanupRecordAudit { + return GitStorageCleanupRecordAudit{ + ObjectID: record.ObjectID, + Checksum: strings.TrimSpace(record.Checksum), + NormalizedPath: normalizedPath, + CleanupScope: cleanupScope, + AccessURLs: uniqueStrings(record.AccessURLs), + AccessMethods: cleanupAccessMethodsFromProjectMethods(record.AccessMethods), + AccessProbes: accessProbesForRecord(record), + Status: accessStatusForRecord(record), + Error: accessErrorForRecord(record), + SizeBytes: record.Size, + LastUpdated: formatOptionalTime(record.UpdatedAt), + DownloadCount: record.Usage.DownloadCount, + LastDownload: formatOptionalTime(record.Usage.LastDownloadTime), + } +} + +func cleanupAccessMethodsFromProjectMethods(methods []gintegrationsyfon.ProjectAccessMethod) []GitStorageCleanupAccessMethod { + out := make([]GitStorageCleanupAccessMethod, 0, len(methods)) + for _, method := range methods { + out = append(out, GitStorageCleanupAccessMethod{ + AccessID: strings.TrimSpace(method.AccessID), + Type: strings.TrimSpace(method.Type), + URL: strings.TrimSpace(method.URL), + Headers: append([]string(nil), method.Headers...), + }) + } + return out +} + +func projectAccessMethodsFromCleanupMethods(methods []GitStorageCleanupAccessMethod) []gintegrationsyfon.ProjectAccessMethod { + out := make([]gintegrationsyfon.ProjectAccessMethod, 0, len(methods)) + for _, method := range methods { + url := strings.TrimSpace(method.URL) + if url == "" { + continue + } + out = append(out, gintegrationsyfon.ProjectAccessMethod{ + AccessID: strings.TrimSpace(method.AccessID), + Type: strings.TrimSpace(method.Type), + URL: url, + Headers: append([]string(nil), method.Headers...), + }) + } + return out +} + +func repoOrphanAction(kind string) string { + if kind == "repo_orphan_live_object" { + return "Delete Syfon record and purge storage object" + } + return "Delete stale Syfon record" +} + +func storageCleanupActionSupport(kind string) (string, []string, string, bool) { + policy := storageRepairPolicyForKind(kind) + return policy.actionability, append([]string(nil), policy.actions...), policy.defaultAction, policy.supportsDryRun +} + +func compareRecordState(left projectRecordState, right projectRecordState) int { + if left.Usage.DownloadCount != right.Usage.DownloadCount { + if left.Usage.DownloadCount > right.Usage.DownloadCount { + return 1 + } + return -1 + } + if compareOptionalTime(left.Usage.LastDownloadTime, right.Usage.LastDownloadTime) != 0 { + return compareOptionalTime(left.Usage.LastDownloadTime, right.Usage.LastDownloadTime) + } + return compareOptionalTime(left.UpdatedAt, right.UpdatedAt) +} + +func compareOptionalTime(left *time.Time, right *time.Time) int { + switch { + case left == nil && right == nil: + return 0 + case left == nil: + return -1 + case right == nil: + return 1 + case left.After(*right): + return 1 + case left.Before(*right): + return -1 + default: + return 0 + } +} + +func recordHasBrokenAccess(record projectRecordState) bool { + if len(record.AccessURLs) == 0 { + return true + } + for _, accessURL := range record.AccessURLs { + if strings.TrimSpace(accessURL) != "" { + return false + } + } + return true +} + +func recordsContainLiveUsage(matches []projectRecordState) bool { + for _, match := range matches { + if match.Usage.DownloadCount > 0 || match.Usage.UploadCount > 0 || match.Usage.LastAccessTime != nil || match.Usage.LastDownloadTime != nil || match.Usage.LastUploadTime != nil { + return true + } + } + return false +} + +func classifyStorageFinding(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) storageFindingKind { + if recordHasBrokenAccess(record) { + return storageFindingBrokenAccessURL + } + resolution := resolveRecordStorage(record, bucketObjectsByURL) + if hasExactPathBucketMismatch(record, resolution, bucketObjectsByURL) { + return storageFindingBrokenBucketMap + } + assessment := assessStorageRecordEvidence(record, len(resolution.matchedBucketObjectURLs) > 0) + if resolution.hasAcceptedCanonicalProbe { + return storageFindingNone + } + if inventoryHasValidationMismatch(record, resolution.matchedBucketObjectURLs, bucketObjectsByURL) { + return storageFindingValidationMismatch + } + if resolution.hasValidationMismatch || assessment.MetadataMismatch { + return storageFindingValidationMismatch + } + if resolution.hasAcceptedCanonicalProbe || assessment.Present { + return storageFindingNone + } + if len(repairableBrokenAccessProbes(record)) > 0 || assessment.MappingBroken { + return storageFindingBrokenBucketMap + } + if len(record.AccessProbes) == 0 { + if len(bucketObjectsByURL) > 0 && len(resolution.candidateURLs) > 0 { + return storageFindingProbeError + } + return storageFindingNone + } + if assessment.Missing { + return storageFindingObjectMissing + } + // A metadata miss is definitive for cleanup and chain classification when + // the backend did not attach a more specific error kind. + if hasNotFoundMetadataProbe(record) { + return storageFindingObjectMissing + } + if hasNotFoundListProbe(record) && len(bucketObjectsByURL) > 0 { + return storageFindingObjectMissing + } + if resolution.hasProbeError || assessment.Status == "unknown" { + return storageFindingProbeError + } + return storageFindingNone +} + +func hasNotFoundMetadataProbe(record projectRecordState) bool { + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Operation) == StorageChainValidationModeMetadata && strings.TrimSpace(probe.Status) == "not_found" { + return true + } + } + return false +} + +func hasNotFoundListProbe(record projectRecordState) bool { + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Operation) == StorageChainValidationModeList && strings.TrimSpace(probe.Status) == "not_found" { + return true + } + } + return false +} + +func hasExactPathBucketMismatch(record projectRecordState, resolution recordStorageResolution, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + // A basename or size match elsewhere in a project bucket is not evidence + // that this record's mapping is wrong: BForePC contains many repeated names + // such as features.tsv.gz. Mapping failures must come from an explicit + // resolver/probe error, not an inventory-side similarity heuristic. + _ = record + _ = resolution + _ = bucketObjectsByURL + return false +} + +func storageProbeValidationMismatchIsSignificant(record projectRecordState, probe gintegrationsyfon.BulkStorageProbeResult) bool { + if strings.TrimSpace(probe.ValidationStatus) != "mismatched" { + return false + } + significantMismatches := 0 + for _, mismatch := range probe.ValidationMismatches { + switch strings.TrimSpace(mismatch) { + case "", "name_mismatch": + continue + default: + significantMismatches++ + } + } + if significantMismatches == 0 { + return false + } + if !syfonProbeHasMismatch(probe, "size_mismatch") || significantMismatches != 1 { + return true + } + if probe.SizeBytes == nil { + return true + } + return !storageSizesMatchForAudit(record.Size, *probe.SizeBytes) +} + +func inventoryHasValidationMismatch(record projectRecordState, bucketObjectURLs []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + if len(bucketObjectURLs) == 0 { + return false + } + for _, objectURL := range bucketObjectURLs { + item, ok := bucketObjectsByURL[objectURL] + if !ok { + continue + } + if !storageSizesMatchForAudit(record.Size, item.SizeBytes) { + return true + } + } + return false +} + +func storageSizesMatchForAudit(expectedSize int64, observedSize int64) bool { + if expectedSize <= 0 || observedSize <= 0 { + return true + } + diff := expectedSize - observedSize + if diff < 0 { + diff = -diff + } + return diff <= 1 +} + +func accessProbesForRecord(record projectRecordState) []GitStorageCleanupAccessProbe { + if len(record.AccessProbes) > 0 { + probes := make([]GitStorageCleanupAccessProbe, 0, len(record.AccessProbes)) + for _, probe := range record.AccessProbes { + exists := probe.Exists + probes = append(probes, GitStorageCleanupAccessProbe{ + URL: probe.ObjectURL, + Operation: probe.Operation, + Provider: probe.Provider, + Bucket: probe.Bucket, + Key: probe.Key, + Path: probe.Path, + Exists: &exists, + Status: probe.Status, + Error: probe.Error, + ErrorKind: probe.ErrorKind, + SizeBytes: probe.SizeBytes, + MetaSHA256: probe.MetaSHA256, + ETag: probe.ETag, + LastModified: probe.LastModified, + ValidationStatus: probe.ValidationStatus, + SizeMatch: probe.SizeMatch, + NameMatch: probe.NameMatch, + SHA256Match: probe.SHA256Match, + ValidationMismatches: append([]string(nil), probe.ValidationMismatches...), + }) + } + return probes + } + if len(record.AccessURLs) == 0 { + return []GitStorageCleanupAccessProbe{{ + URL: "", + Status: "missing", + Error: "no access URLs present", + ErrorKind: "missing_access_url", + }} + } + probes := make([]GitStorageCleanupAccessProbe, 0, len(record.AccessURLs)) + for _, accessURL := range record.AccessURLs { + if strings.TrimSpace(accessURL) == "" { + probes = append(probes, GitStorageCleanupAccessProbe{ + URL: accessURL, + Status: "missing", + Error: "blank access URL", + ErrorKind: "missing_access_url", + }) + continue + } + probes = append(probes, GitStorageCleanupAccessProbe{ + URL: accessURL, + Status: "present", + }) + } + return probes +} + +func accessStatusForRecord(record projectRecordState) string { + switch classifyStorageFinding(record, nil) { + case storageFindingBrokenAccessURL, storageFindingObjectMissing: + return "missing" + case storageFindingBrokenBucketMap, storageFindingProbeError: + return "error" + case storageFindingValidationMismatch: + return "mismatched" + } + return "present" +} + +func accessErrorForRecord(record projectRecordState) string { + switch classifyStorageFinding(record, nil) { + case storageFindingBrokenAccessURL: + return "no usable access URL present" + case storageFindingBrokenBucketMap: + return "Syfon access URL did not resolve through a configured bucket mapping" + case storageFindingObjectMissing: + return "storage object not found" + case storageFindingValidationMismatch: + return "mapped bucket object exists, but object evidence disagrees with the Syfon/Git record" + case storageFindingProbeError: + return "storage probe failed" + } + return "" +} + +func buildBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStorageCleanupFinding { + objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL) + actionability, availableActions, defaultAction, supportsDryRun := storageCleanupActionSupport("bucket_only_object") + return GitStorageCleanupFinding{ + Kind: "bucket_only_object", + NormalizedPath: objectURL, + ObjectIDs: []string{}, + Records: []GitStorageCleanupRecordAudit{}, + RecommendedAction: "Review and delete bucket object that has no Syfon record", + RepoDeleteCandidate: false, + CleanupScope: "bucket_object", + SizeBytes: item.SizeBytes, + LastUpdated: strings.TrimSpace(item.LastModified), + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, + Evidence: &GitAuditEvidence{ + AccessURLs: []string{objectURL}, + Buckets: uniqueStrings([]string{item.Bucket}), + Keys: uniqueStrings([]string{item.Key}), + StorageOperations: []string{StorageChainValidationModeInventory}, + ProbeStatuses: []string{"enumerated"}, + BucketEvaluation: "enumerated", + }, + } +} + +func buildChainBucketOnlyFinding(item gintegrationsyfon.ProjectBucketObject) GitStorageChainFinding { + objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL) + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence("bucket_only_object", "verified") + return GitStorageChainFinding{ + Kind: "bucket_only_object", + EvidenceStatus: "verified", + NormalizedPath: objectURL, + ObjectIDs: []string{}, + AccessURLs: []string{objectURL}, + BucketObjectURL: objectURL, + ResolvedBucket: strings.TrimSpace(item.Bucket), + ResolvedKey: strings.TrimSpace(item.Key), + ProbeStatus: "enumerated", + RecordCount: 0, + SizeBytes: item.SizeBytes, + RecommendedAction: "Bucket object exists, but no Syfon record matched it.", + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, + Evidence: &GitAuditEvidence{ + AccessURLs: []string{objectURL}, + BucketObjectURLs: []string{objectURL}, + Buckets: uniqueStrings([]string{item.Bucket}), + Keys: uniqueStrings([]string{item.Key}), + StorageOperations: []string{StorageChainValidationModeInventory}, + ProbeStatuses: []string{"enumerated"}, + BucketEvaluation: "enumerated", + }, + } +} + +func buildChainRecordFindings(kind string, record projectRecordState, gitPaths []string, bucketObjectURLs []string, action string) []GitStorageChainFinding { + return buildChainRecordFindingsWithOptions(kind, record, gitPaths, bucketObjectURLs, action, false) +} + +func buildChainRecordFindingsWithOptions(kind string, record projectRecordState, gitPaths []string, bucketObjectURLs []string, action string, preferSyfonRecordPath bool) []GitStorageChainFinding { + paths := uniqueStrings(gitPaths) + if len(paths) == 0 { + displayPath := orphanDisplayPath(strings.TrimSpace(record.Checksum), append(bucketObjectURLs, record.AccessURLs...)) + if preferSyfonRecordPath { + displayPath = syfonRecordDisplayPath(record) + } + if displayPath == "" { + displayPath = strings.TrimSpace(record.Checksum) + } + paths = []string{displayPath} + } + objectIDs := uniqueStrings([]string{record.ObjectID}) + accessURLs := uniqueStrings(record.AccessURLs) + evidence := buildFindingEvidence(strings.TrimSpace(record.Checksum), gitPaths, []projectRecordState{record}, "enumerated_and_probed") + if evidence != nil { + evidence.BucketObjectURLs = uniqueStrings(append(evidence.BucketObjectURLs, bucketObjectURLs...)) + } + evidenceStatus := storageEvidenceStatus(record, len(bucketObjectURLs) > 0) + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence(kind, evidenceStatus) + primaryProbe := selectChainProbe(record, bucketObjectURLs) + suggestedFix := suggestedFixForChainFinding(kind, record) + suggestedAction := suggestedActionForChainFinding(kind, record) + if suggestedAction != "" && !stringSliceContains(availableActions, suggestedAction) { + availableActions = append([]string(nil), availableActions...) + availableActions = append(availableActions, suggestedAction) + } + findings := make([]GitStorageChainFinding, 0, len(paths)) + for _, path := range paths { + findings = append(findings, GitStorageChainFinding{ + Kind: kind, + EvidenceStatus: evidenceStatus, + NormalizedPath: path, + Checksum: strings.TrimSpace(record.Checksum), + SourcePaths: uniqueStrings(gitPaths), + ObjectIDs: objectIDs, + Records: []GitStorageCleanupRecordAudit{cleanupRecordAuditForRecord(path, "access_url", record)}, + AccessURLs: accessURLs, + BucketObjectURL: primaryProbe.bucketObjectURL, + ResolvedBucket: primaryProbe.probe.Bucket, + ResolvedKey: primaryProbe.probe.Key, + ProbeStatus: primaryProbe.probe.Status, + ErrorKind: primaryProbe.probe.ErrorKind, + Error: chainFindingError(kind, record, primaryProbe.probe), + RecordCount: 1, + SizeBytes: record.Size, + RecommendedAction: action, + SuggestedFix: suggestedFix, + SuggestedAction: suggestedAction, + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, + Evidence: evidence, + }) + } + return findings +} + +func suggestedActionForChainFinding(kind string, record projectRecordState) string { + return "" +} + +func suggestedFixForChainFinding(kind string, record projectRecordState) string { + normalizedKind := strings.TrimSpace(kind) + switch normalizedKind { + case "git_syfon_metadata_mismatch": + case "syfon_broken_bucket_mapping": + return "" + default: + return "" + } + mismatches := make([]string, 0, 2) + if probe, ok := firstAccessProbeMismatch(record, "size_mismatch"); ok { + bucketSize := int64(0) + if probe.SizeBytes != nil { + bucketSize = *probe.SizeBytes + } + mismatches = append(mismatches, fmt.Sprintf("Syfon record size is %s, bucket inventory reports %s", formatAuditSize(record.Size), formatAuditSize(bucketSize))) + } + if len(mismatches) == 0 { + return "Bucket object exists, but its storage evidence does not match the Syfon/Git record. Review the record and bucket object before applying a destructive fix." + } + return strings.Join(mismatches, ". ") + ". Because the byte lengths differ, the bucket object cannot have the current Git/Syfon SHA-256. Recompute the bucket object's SHA-256 before manually reconciling Git and Syfon." +} + +func firstAccessProbeMismatch(record projectRecordState, mismatch string) (GitStorageCleanupAccessProbe, bool) { + for _, probe := range accessProbesForRecord(record) { + if accessProbeHasMismatch(probe, mismatch) { + return probe, true + } + } + return GitStorageCleanupAccessProbe{}, false +} + +func accessProbeHasMismatch(probe GitStorageCleanupAccessProbe, mismatch string) bool { + for _, item := range probe.ValidationMismatches { + if strings.TrimSpace(item) == mismatch { + return true + } + } + return false +} + +func formatAuditSize(size int64) string { + if size <= 0 { + return "unknown" + } + return fmt.Sprintf("%d B", size) +} + +type chainProbeSelection struct { + bucketObjectURL string + probe GitStorageCleanupAccessProbe +} + +func selectChainProbe(record projectRecordState, bucketObjectURLs []string) chainProbeSelection { + probes := accessProbesForRecord(record) + if len(bucketObjectURLs) > 0 { + targets := map[string]struct{}{} + for _, bucketURL := range bucketObjectURLs { + targets[strings.TrimSpace(bucketURL)] = struct{}{} + } + for _, probe := range probes { + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.URL) + if _, ok := targets[objectURL]; ok { + return chainProbeSelection{bucketObjectURL: objectURL, probe: probe} + } + } + } + for _, probe := range probes { + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.URL) + if probe.Status != "" || probe.ErrorKind != "" || objectURL != "" || probe.URL != "" { + return chainProbeSelection{probe: probe} + } + } + return chainProbeSelection{} +} + +func chainFindingError(kind string, record projectRecordState, probe GitStorageCleanupAccessProbe) string { + if trimmed := strings.TrimSpace(probe.Error); trimmed != "" { + return trimmed + } + switch kind { + case "syfon_broken_bucket_mapping": + return "Syfon access URL did not resolve through a configured bucket mapping" + case "syfon_missing_bucket_object", "syfon_git_no_bucket": + return "mapped bucket object was not found" + case "git_syfon_metadata_mismatch": + return "mapped bucket object exists, but object evidence disagrees with the Syfon/Git record" + case "probe_error": + return accessErrorForRecord(record) + default: + return "" + } +} + +func recordsReferenceBucketObject(matches []projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + for _, match := range matches { + if len(resolveRecordStorage(match, bucketObjectsByURL).matchedBucketObjectURLs) > 0 { + return true + } + } + return false +} + +func matchedBucketObjectURLs(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []string { + return resolveRecordStorage(record, bucketObjectsByURL).matchedBucketObjectURLs +} + +func cleanupMatchedBucketObjectURLs(matches []projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []string { + objectURLs := make([]string, 0) + for _, match := range matches { + objectURLs = append(objectURLs, matchedBucketObjectURLs(match, bucketObjectsByURL)...) + for _, probe := range accessProbesForRecord(match) { + if strings.TrimSpace(probe.Status) != "present" { + continue + } + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.URL) + if strings.TrimSpace(objectURL) == "" { + continue + } + objectURLs = append(objectURLs, objectURL) + } + } + return uniqueStrings(objectURLs) +} + +func recordBucketURLs(record projectRecordState) []string { + return recordStorageCandidateURLs(record) +} + +func recordStorageCandidateURLs(record projectRecordState) []string { + out := make([]string, 0) + for _, accessURL := range record.CanonicalAccessURLs { + if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { + out = append(out, objectURL) + } + } + for _, probe := range record.AccessProbes { + if objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL); objectURL != "" { + out = append(out, objectURL) + } + } + if len(record.CanonicalAccessURLs) == 0 { + for _, accessURL := range rawAccessURLsForRecord(record) { + if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { + out = append(out, objectURL) + } + } + } + return uniqueStrings(out) +} + +type recordStorageResolution struct { + candidateURLs []string + matchedBucketObjectURLs []string + hasPresentProbe bool + hasMissingProbe bool + hasBrokenMappingProbe bool + hasProbeError bool + hasValidationMismatch bool + hasAcceptedCanonicalProbe bool + hasPresentRawAccessProbe bool + hasMissingRawAccessProbe bool + hasBrokenRawAccessProbe bool + hasRawAccessProbeError bool +} + +func resolveRecordStorage(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) recordStorageResolution { + resolution := recordStorageResolution{ + candidateURLs: recordStorageCandidateURLs(record), + } + rawURLs := make(map[string]struct{}, len(record.AccessURLs)) + rawCandidateURLs := make(map[string]struct{}, len(record.AccessURLs)) + for _, accessURL := range record.AccessURLs { + if trimmed := strings.TrimSpace(accessURL); trimmed != "" { + rawURLs[trimmed] = struct{}{} + if objectURL := canonicalStorageURL("", "", trimmed); objectURL != "" { + rawCandidateURLs[objectURL] = struct{}{} + } + } + } + canonicalURLs := make(map[string]struct{}, len(record.CanonicalAccessURLs)) + for _, accessURL := range record.CanonicalAccessURLs { + trimmed := strings.TrimSpace(accessURL) + if trimmed == "" { + continue + } + if _, raw := rawURLs[trimmed]; !raw { + canonicalURLs[trimmed] = struct{}{} + } + } + for _, probe := range record.AccessProbes { + objectURL := syfonProbeObjectURL(probe) + status := strings.TrimSpace(probe.Status) + _, rawAccessProbe := rawCandidateURLs[objectURL] + if status == "present" { + resolution.hasPresentProbe = true + if rawAccessProbe { + resolution.hasPresentRawAccessProbe = true + } + if _, ok := canonicalURLs[objectURL]; ok { + switch strings.TrimSpace(probe.ValidationStatus) { + case "", "not_requested", "matched": + resolution.hasAcceptedCanonicalProbe = true + } + } + } + if storageProbeValidationMismatchIsSignificant(record, probe) { + resolution.hasValidationMismatch = true + } + switch strings.TrimSpace(probe.ErrorKind) { + case "credential_missing": + resolution.hasBrokenMappingProbe = true + if rawAccessProbe { + resolution.hasBrokenRawAccessProbe = true + } + } + switch status { + case "not_found": + if strings.TrimSpace(probe.Operation) == StorageChainValidationModeInventory { + resolution.hasProbeError = true + if rawAccessProbe { + resolution.hasRawAccessProbeError = true + } + continue + } + resolution.hasMissingProbe = true + if rawAccessProbe { + resolution.hasMissingRawAccessProbe = true + } + case "unknown", "forbidden", "unsupported", "invalid", "error": + resolution.hasProbeError = true + if rawAccessProbe { + resolution.hasRawAccessProbeError = true + } + } + } + if len(bucketObjectsByURL) > 0 { + for _, objectURL := range resolution.candidateURLs { + if _, ok := bucketObjectsByURL[objectURL]; ok { + resolution.matchedBucketObjectURLs = append(resolution.matchedBucketObjectURLs, objectURL) + } + } + resolution.matchedBucketObjectURLs = uniqueStrings(resolution.matchedBucketObjectURLs) + } + return resolution +} + +func accessURLsForStorage(record projectRecordState) []string { + if len(record.CanonicalAccessURLs) > 0 { + return record.CanonicalAccessURLs + } + return record.AccessURLs +} + +func probeAccessURLsForRecord(record projectRecordState) []string { + urls := make([]string, 0, len(record.CanonicalAccessURLs)+len(record.AccessURLs)+len(record.AccessProbes)) + if len(record.CanonicalAccessURLs) > 0 { + urls = append(urls, record.CanonicalAccessURLs...) + } else { + urls = append(urls, rawAccessURLsForRecord(record)...) + } + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Operation) != StorageChainValidationModeInventory || strings.TrimSpace(probe.Status) != "unknown" { + continue + } + if objectURL := syfonProbeObjectURL(probe); objectURL != "" { + urls = append(urls, objectURL) + } + } + return uniqueStrings(urls) +} + +func rawAccessURLsForRecord(record projectRecordState) []string { + out := make([]string, 0, len(record.AccessURLs)) + for _, accessURL := range record.AccessURLs { + if trimmed := strings.TrimSpace(accessURL); trimmed != "" { + out = append(out, trimmed) + } + } + return uniqueStrings(out) +} + +func brokenBucketMappingRepairPlan(records []projectRecordState) ([]string, map[string][]gintegrationsyfon.ProjectAccessMethod) { + deleteIDs := make([]string, 0) + updateAccessMethods := make(map[string][]gintegrationsyfon.ProjectAccessMethod) + for _, record := range records { + remainingMethods, shouldDelete, ok := repairBrokenBucketMappingRecord(record) + if !ok { + continue + } + if shouldDelete { + deleteIDs = append(deleteIDs, record.ObjectID) + continue + } + updateAccessMethods[record.ObjectID] = remainingMethods + } + return uniqueStrings(deleteIDs), updateAccessMethods +} + +func repairBrokenBucketMappingRecord(record projectRecordState) ([]gintegrationsyfon.ProjectAccessMethod, bool, bool) { + if len(record.AccessMethods) == 0 || len(record.AccessProbes) == 0 { + return nil, false, false + } + probesByURL := make(map[string][]gintegrationsyfon.BulkStorageProbeResult, len(record.AccessProbes)) + for _, probe := range record.AccessProbes { + probesByURL[strings.TrimSpace(probe.ObjectURL)] = append(probesByURL[strings.TrimSpace(probe.ObjectURL)], probe) + } + remaining := make([]gintegrationsyfon.ProjectAccessMethod, 0, len(record.AccessMethods)) + removedAny := false + for _, method := range record.AccessMethods { + if !accessURLHasBrokenBucketMapping(method.URL, probesByURL) { + remaining = append(remaining, method) + continue + } + removedAny = true + } + if !removedAny { + return nil, false, false + } + auditRecord := cleanupRecordAuditForRecord("", "access_url", record) + remaining = appendReplacementAccessMethods(remaining, auditRecord.AccessProbes, brokenAccessURLsForRecord(auditRecord)) + if len(remaining) == 0 { + return nil, true, true + } + return remaining, false, true +} + +func accessURLHasBrokenBucketMapping(accessURL string, probesByURL map[string][]gintegrationsyfon.BulkStorageProbeResult) bool { + probes := probesByURL[strings.TrimSpace(accessURL)] + if len(probes) == 0 { + return false + } + hasBrokenBucketMapping := false + for _, probe := range probes { + if strings.TrimSpace(probe.Status) == "present" { + return false + } + if syfonProbeIsBrokenAccess(probe) { + hasBrokenBucketMapping = true + } + } + return hasBrokenBucketMapping +} + +func repairableBrokenAccessRecord(record projectRecordState) projectRecordState { + clone := record + clone.AccessProbes = repairableBrokenAccessProbes(record) + return clone +} + +func repairableBrokenAccessProbes(record projectRecordState) []gintegrationsyfon.BulkStorageProbeResult { + if len(record.AccessProbes) == 0 { + return nil + } + probesByURL := make(map[string][]gintegrationsyfon.BulkStorageProbeResult, len(record.AccessProbes)) + for _, probe := range record.AccessProbes { + objectURL := syfonProbeObjectURL(probe) + if objectURL == "" { + continue + } + probesByURL[objectURL] = append(probesByURL[objectURL], probe) + } + out := make([]gintegrationsyfon.BulkStorageProbeResult, 0) + for _, accessURL := range rawAccessURLsForRecord(record) { + rawURL := strings.TrimSpace(accessURL) + if rawURL == "" { + continue + } + if accessURLHasPresentProbe(rawURL, probesByURL) { + continue + } + if mappedURL := strings.TrimSpace(record.CanonicalAccessURLByRaw[rawURL]); mappedURL != "" && mappedURL != rawURL && accessURLHasPresentProbe(mappedURL, probesByURL) { + continue + } + for _, probe := range probesByURL[rawURL] { + if syfonProbeIsBrokenAccess(probe) { + out = append(out, probe) + } + } + } + return out +} + +func accessURLHasPresentProbe(accessURL string, probesByURL map[string][]gintegrationsyfon.BulkStorageProbeResult) bool { + for _, probe := range probesByURL[strings.TrimSpace(accessURL)] { + if strings.TrimSpace(probe.Status) != "present" { + continue + } + switch strings.TrimSpace(probe.ValidationStatus) { + case "", "not_requested", "matched": + return true + } + } + return false +} + +func syfonProbeObjectURL(probe gintegrationsyfon.BulkStorageProbeResult) string { + if objectURL := strings.TrimSpace(probe.ObjectURL); objectURL != "" { + return objectURL + } + return canonicalStorageURL(probe.Bucket, probe.Key, "") +} + +func syfonProbeIsBrokenAccess(probe gintegrationsyfon.BulkStorageProbeResult) bool { + switch strings.TrimSpace(probe.ErrorKind) { + case "missing_access_url", "scope_not_found", "credential_missing": + return true + } + return false +} + +func syfonProbeHasMismatch(probe gintegrationsyfon.BulkStorageProbeResult, mismatch string) bool { + for _, item := range probe.ValidationMismatches { + if strings.TrimSpace(item) == mismatch { + return true + } + } + return false +} + +func canonicalizeRecordAccessURLs(accessURLs []string, scopes []domain.StorageBucketScope, organization string, project string) []string { + out := make([]string, 0, len(accessURLs)) + for _, accessURL := range accessURLs { + if objectURL := canonicalizeScopedStorageURL(accessURL, scopes, organization, project); objectURL != "" { + out = append(out, objectURL) + continue + } + if objectURL := canonicalStorageURL("", "", accessURL); objectURL != "" { + out = append(out, objectURL) + } + } + return uniqueStrings(out) +} + +func canonicalizeRecordAccessURLMappings(accessURLs []string, scopes []domain.StorageBucketScope, organization string, project string) map[string]string { + out := make(map[string]string, len(accessURLs)) + for _, accessURL := range accessURLs { + rawURL := strings.TrimSpace(accessURL) + if rawURL == "" { + continue + } + if objectURL := canonicalizeScopedStorageURL(rawURL, scopes, organization, project); objectURL != "" { + out[rawURL] = objectURL + continue + } + if objectURL := canonicalStorageURL("", "", rawURL); objectURL != "" { + out[rawURL] = objectURL + } + } + return out +} + +func canonicalizeScopedStorageURL(accessURL string, scopes []domain.StorageBucketScope, organization string, project string) string { + if len(scopes) == 0 { + return "" + } + _, key, ok := parseStorageURL(accessURL) + if !ok { + return "" + } + targetBucket := "" + prefixes := make([]string, 0, len(scopes)) + for _, scope := range scopes { + if !storageScopeApplies(scope, organization, project) { + continue + } + scopeBucket, scopePrefix, ok := parseStorageScopePath(scope) + if !ok { + continue + } + if strings.TrimSpace(scopeBucket) != "" { + targetBucket = strings.TrimSpace(scopeBucket) + } + if strings.TrimSpace(scopePrefix) != "" { + prefixes = append(prefixes, strings.Trim(strings.TrimSpace(scopePrefix), "/")) + } + } + if targetBucket == "" { + return "" + } + normalizedKey := normalizeScopedStorageKeyForGecko(key, prefixes) + if normalizedKey == "" { + return "" + } + return canonicalStorageURL(targetBucket, normalizedKey, "") +} + +func storageScopeApplies(scope domain.StorageBucketScope, organization string, project string) bool { + scopeOrg := strings.TrimSpace(scope.Organization) + if scopeOrg != "" && !strings.EqualFold(scopeOrg, strings.TrimSpace(organization)) { + return false + } + scopeProject := strings.TrimSpace(scope.ProjectID) + return scopeProject == "" || strings.EqualFold(scopeProject, strings.TrimSpace(project)) +} + +func parseStorageScopePath(scope domain.StorageBucketScope) (string, string, bool) { + bucket := strings.TrimSpace(scope.Bucket) + pathValue := strings.TrimSpace(scope.Path) + if pathValue == "" { + return bucket, "", bucket != "" + } + if strings.HasPrefix(strings.ToLower(pathValue), "s3://") { + parsedBucket, parsedPrefix, ok := parseStorageURLAllowRoot(pathValue) + if !ok { + return bucket, "", bucket != "" + } + if bucket == "" { + bucket = parsedBucket + } + return bucket, parsedPrefix, bucket != "" + } + return bucket, strings.Trim(pathValue, "/"), bucket != "" +} + +func parseStorageURL(raw string) (string, string, bool) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "", "", false + } + if !strings.HasPrefix(strings.ToLower(trimmed), "s3://") { + return "", "", false + } + rest := strings.TrimPrefix(trimmed, "s3://") + rest = strings.TrimLeft(rest, "/") + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 { + return "", "", false + } + bucket := strings.TrimSpace(parts[0]) + key := strings.Trim(strings.TrimSpace(parts[1]), "/") + if bucket == "" || key == "" { + return "", "", false + } + return bucket, key, true +} + +func parseStorageURLAllowRoot(raw string) (string, string, bool) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "", "", false + } + if !strings.HasPrefix(strings.ToLower(trimmed), "s3://") { + return "", "", false + } + rest := strings.TrimPrefix(trimmed, "s3://") + rest = strings.TrimLeft(rest, "/") + parts := strings.SplitN(rest, "/", 2) + bucket := strings.TrimSpace(parts[0]) + if bucket == "" { + return "", "", false + } + if len(parts) == 1 { + return bucket, "", true + } + return bucket, strings.Trim(strings.TrimSpace(parts[1]), "/"), true +} + +func normalizeScopedStorageKeyForGecko(key string, prefixes []string) string { + key = strings.Trim(strings.TrimSpace(key), "/") + normalizedPrefixes := normalizedScopePrefixesForGecko(prefixes) + remainder := key + for _, prefix := range normalizedPrefixes { + remainder = trimLeadingStoragePrefixForGecko(remainder, prefix) + } + composedPrefix := strings.Join(normalizedPrefixes, "/") + switch { + case composedPrefix == "": + return remainder + case remainder == "": + return composedPrefix + default: + return path.Join(composedPrefix, remainder) + } +} + +func normalizedScopePrefixesForGecko(prefixes []string) []string { + out := make([]string, 0, len(prefixes)) + for _, prefix := range prefixes { + prefix = strings.Trim(strings.TrimSpace(prefix), "/") + if prefix == "" { + continue + } + if len(out) == 0 { + out = append(out, prefix) + continue + } + last := out[len(out)-1] + switch { + case prefix == last: + continue + case strings.HasPrefix(prefix, last+"/"): + out[len(out)-1] = prefix + case strings.HasPrefix(last, prefix+"/"): + continue + default: + out = append(out, prefix) + } + } + return out +} + +func trimLeadingStoragePrefixForGecko(key, prefix string) string { + key = strings.Trim(strings.TrimSpace(key), "/") + prefix = strings.Trim(strings.TrimSpace(prefix), "/") + if key == "" || prefix == "" { + return key + } + if key == prefix { + return "" + } + if strings.HasPrefix(key, prefix+"/") { + return strings.TrimPrefix(key, prefix+"/") + } + return key +} + +func canonicalStorageURL(bucket string, key string, objectURL string) string { + cleanBucket := strings.TrimSpace(bucket) + cleanKey := strings.Trim(strings.TrimSpace(key), "/") + if cleanBucket != "" && cleanKey != "" { + return "s3://" + cleanBucket + "/" + cleanKey + } + trimmed := strings.TrimSpace(objectURL) + if trimmed == "" { + return "" + } + if !strings.HasPrefix(strings.ToLower(trimmed), "s3://") { + return "" + } + rest := strings.TrimPrefix(trimmed, "s3://") + if strings.HasPrefix(rest, "/") { + rest = strings.TrimLeft(rest, "/") + } + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || strings.TrimSpace(parts[1]) == "" { + return "" + } + return "s3://" + strings.TrimSpace(parts[0]) + "/" + strings.Trim(strings.TrimSpace(parts[1]), "/") +} + +func normalizeAnalyticsChecksum(value string) string { + trimmed := strings.TrimSpace(value) + trimmed = strings.TrimPrefix(trimmed, "sha256:") + trimmed = strings.TrimPrefix(trimmed, "SHA256:") + if trimmed == "" { + return "" + } + return strings.ToLower(trimmed) +} + +func applyUsage(agg *storageAggregate, record projectRecordState) { + agg.downloadCount += record.Usage.DownloadCount + agg.lastDownload = laterTime(agg.lastDownload, record.Usage.LastDownloadTime) + agg.latestUpdate = laterTime(agg.latestUpdate, record.UpdatedAt) +} + +func aggregateMatchedSize(matches []projectRecordState, fallback int64) int64 { + var total int64 + for _, match := range matches { + if match.Size > 0 { + total += match.Size + } + } + if total == 0 { + return fallback + } + return total +} + +func aggregateMatchedDownloads(matches []projectRecordState) int64 { + var total int64 + for _, match := range matches { + total += match.Usage.DownloadCount + } + return total +} + +func latestMatchedDownload(matches []projectRecordState) *time.Time { + var latest *time.Time + for _, match := range matches { + latest = laterTime(latest, match.Usage.LastDownloadTime) + } + return latest +} + +func laterTime(current *time.Time, candidate *time.Time) *time.Time { + if candidate == nil { + return current + } + if current == nil || candidate.After(*current) { + copyTime := candidate.UTC() + return ©Time + } + return current +} + +func immediateChild(root string, repoPath string) (string, string, string) { + normalizedPath := normalizeRepoSubpath(repoPath) + if root != "" { + prefix := root + "/" + if !strings.HasPrefix(normalizedPath, prefix) { + return "", "", "" + } + normalizedPath = strings.TrimPrefix(normalizedPath, prefix) + } + if normalizedPath == "" { + return "", "", "" + } + parts := strings.Split(normalizedPath, "/") + childName := parts[0] + if len(parts) == 1 { + if root == "" { + return childName, childName, "file" + } + return childName, root + "/" + childName, "file" + } + if root == "" { + return childName, childName, "directory" + } + return childName, root + "/" + childName, "directory" +} + +func normalizeRepoSubpath(raw string) string { + return strings.Trim(strings.TrimSpace(raw), "/") +} + +func recordObjectIDs(matches []projectRecordState) []string { + out := make([]string, 0, len(matches)) + for _, match := range matches { + if trimmed := strings.TrimSpace(match.ObjectID); trimmed != "" { + out = append(out, trimmed) + } + } + return uniqueStrings(out) +} + +func buildFindingEvidence(checksum string, sourcePaths []string, matches []projectRecordState, bucketEvaluation string) *GitAuditEvidence { + evidence := &GitAuditEvidence{ + Checksum: strings.TrimSpace(checksum), + SourcePaths: uniqueStrings(sourcePaths), + ObjectIDs: []string{}, + AccessURLs: []string{}, + BucketObjectURLs: []string{}, + Buckets: []string{}, + Keys: []string{}, + StorageOperations: []string{}, + ProbeStatuses: []string{}, + ValidationStates: []string{}, + ErrorKinds: []string{}, + Errors: []string{}, + BucketEvaluation: strings.TrimSpace(bucketEvaluation), + } + for _, match := range matches { + if objectID := strings.TrimSpace(match.ObjectID); objectID != "" { + evidence.ObjectIDs = append(evidence.ObjectIDs, objectID) + } + evidence.AccessURLs = append(evidence.AccessURLs, match.AccessURLs...) + if len(match.AccessProbes) == 0 { + continue + } + for _, probe := range match.AccessProbes { + if objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL); objectURL != "" { + evidence.BucketObjectURLs = append(evidence.BucketObjectURLs, objectURL) + } + if bucket := strings.TrimSpace(probe.Bucket); bucket != "" { + evidence.Buckets = append(evidence.Buckets, bucket) + } + if key := strings.TrimSpace(probe.Key); key != "" { + evidence.Keys = append(evidence.Keys, key) + } + if operation := strings.TrimSpace(probe.Operation); operation != "" { + evidence.StorageOperations = append(evidence.StorageOperations, operation) + } + if status := strings.TrimSpace(probe.Status); status != "" { + evidence.ProbeStatuses = append(evidence.ProbeStatuses, status) + } + if validation := strings.TrimSpace(probe.ValidationStatus); validation != "" { + evidence.ValidationStates = append(evidence.ValidationStates, validation) + } + if kind := strings.TrimSpace(probe.ErrorKind); kind != "" { + evidence.ErrorKinds = append(evidence.ErrorKinds, kind) + } + if err := strings.TrimSpace(probe.Error); err != "" { + evidence.Errors = append(evidence.Errors, err) + } + } + } + evidence.ObjectIDs = uniqueStrings(evidence.ObjectIDs) + evidence.AccessURLs = uniqueStrings(evidence.AccessURLs) + evidence.BucketObjectURLs = uniqueStrings(evidence.BucketObjectURLs) + evidence.Buckets = uniqueStrings(evidence.Buckets) + evidence.Keys = uniqueStrings(evidence.Keys) + evidence.StorageOperations = uniqueStrings(evidence.StorageOperations) + evidence.ProbeStatuses = uniqueStrings(evidence.ProbeStatuses) + evidence.ValidationStates = uniqueStrings(evidence.ValidationStates) + evidence.ErrorKinds = uniqueStrings(evidence.ErrorKinds) + evidence.Errors = uniqueStrings(evidence.Errors) + if evidence.Checksum == "" && + len(evidence.SourcePaths) == 0 && + len(evidence.ObjectIDs) == 0 && + len(evidence.AccessURLs) == 0 && + len(evidence.BucketObjectURLs) == 0 && + len(evidence.Buckets) == 0 && + len(evidence.Keys) == 0 && + len(evidence.StorageOperations) == 0 && + len(evidence.ProbeStatuses) == 0 && + len(evidence.ValidationStates) == 0 && + len(evidence.ErrorKinds) == 0 && + len(evidence.Errors) == 0 && + evidence.BucketEvaluation == "" { + return nil + } + return evidence +} + +func flattenRecordStates(recordsByChecksum map[string][]projectRecordState) []projectRecordState { + out := make([]projectRecordState, 0) + seen := map[string]struct{}{} + for _, group := range recordsByChecksum { + for _, record := range group { + objectID := strings.TrimSpace(record.ObjectID) + if objectID == "" { + continue + } + if _, ok := seen[objectID]; ok { + continue + } + seen[objectID] = struct{}{} + out = append(out, record) + } + } + sort.Slice(out, func(i, j int) bool { + return out[i].ObjectID < out[j].ObjectID + }) + return out +} + +func chainPathCount(gitPaths []string) int { + if len(gitPaths) == 0 { + return 1 + } + return len(gitPaths) +} + +func uniqueStrings(values []string) []string { + if len(values) == 0 { + return []string{} + } + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func storageProbeRequestKey(objectURL string, size int64, checksum string) string { + return strings.TrimSpace(objectURL) + "|" + fmt.Sprintf("%d", size) + "|" + strings.TrimSpace(checksum) +} + +func storageListValidationRequestKey(objectURL string, size int64, expectedName string) string { + return strings.TrimSpace(objectURL) + "|" + fmt.Sprintf("%d", size) + "|" + strings.TrimSpace(expectedName) +} + +func expectedStorageObjectNameForListValidation(objectURL string, recordName string) string { + expectedName := path.Base(strings.Trim(strings.TrimSpace(recordName), "/")) + if expectedName == "." || expectedName == "/" || expectedName == "" { + return "" + } + return expectedName +} + +func sortStorageAggregates(items []storageAggregate, sortBy string, sortOrder string) { + desc := !strings.EqualFold(strings.TrimSpace(sortOrder), "asc") + switch strings.ToLower(strings.TrimSpace(sortBy)) { + case "name": + sort.Slice(items, func(i, j int) bool { + left := strings.ToLower(items[i].name) + right := strings.ToLower(items[j].name) + if desc { + return left > right + } + return left < right + }) + default: + sort.Slice(items, func(i, j int) bool { + if items[i].totalBytes != items[j].totalBytes { + if desc { + return items[i].totalBytes > items[j].totalBytes + } + return items[i].totalBytes < items[j].totalBytes + } + if items[i].rowType != items[j].rowType { + return items[i].rowType == "directory" + } + return strings.ToLower(items[i].name) < strings.ToLower(items[j].name) + }) + } +} + +func orphanDisplayPath(checksum string, sourcePaths []string) string { + if len(sourcePaths) > 0 && strings.TrimSpace(sourcePaths[0]) != "" { + return strings.TrimSpace(sourcePaths[0]) + } + return "sha256/" + strings.TrimSpace(checksum) +} + +func syfonRecordDisplayPath(record projectRecordState) string { + if name := strings.TrimSpace(record.Name); name != "" { + return name + } + if objectID := strings.TrimSpace(record.ObjectID); objectID != "" { + return "syfon/" + objectID + } + if checksum := strings.TrimSpace(record.Checksum); checksum != "" { + return "sha256/" + checksum + } + return "" +} + +func recordSourcePaths(matches []projectRecordState) []string { + seen := make(map[string]struct{}) + out := make([]string, 0) + for _, match := range matches { + for _, accessURL := range match.AccessURLs { + normalized := strings.TrimSpace(accessURL) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + } + sort.Strings(out) + return out +} + +func formatOptionalTime(value *time.Time) string { + if value == nil { + return "" + } + return value.UTC().Format(time.RFC3339) +} + +func boolPtr(value bool) *bool { + return &value +} + +func differenceStrings(values []string, toRemove []string) []string { + if len(values) == 0 { + return []string{} + } + removeSet := make(map[string]struct{}, len(toRemove)) + for _, value := range toRemove { + removeSet[value] = struct{}{} + } + out := make([]string, 0, len(values)) + for _, value := range values { + if _, ok := removeSet[value]; ok { + continue + } + out = append(out, value) + } + return out +} diff --git a/internal/git/storage_analytics_pipeline.go b/internal/git/storage_analytics_pipeline.go new file mode 100644 index 0000000..84f89e2 --- /dev/null +++ b/internal/git/storage_analytics_pipeline.go @@ -0,0 +1,1591 @@ +package git + +import ( + "context" + "fmt" + "log" + "path" + "sort" + "strings" + "time" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +type storageAuditBaseInputs struct { + index *repoAnalyticsIndex + inventory []RepoInventoryFile + recordsByChecksum map[string][]projectRecordState + usageByObjectID map[string]gintegrationsyfon.FileUsage +} + +type storageAuditRecordSet struct { + recordsByChecksum map[string][]projectRecordState + allProjectRecords map[string][]projectRecordState +} + +type storageAuditStorageView struct { + scopes []domain.StorageBucketScope + bucketObjects []gintegrationsyfon.ProjectBucketObject + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + recordsByChecksum map[string][]projectRecordState + allProjectRecords map[string][]projectRecordState + bucketInventoryAvailable bool + bucketInventoryError string +} + +type storageChainIndex struct { + inventory []RepoInventoryFile + allRecords []projectRecordState + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + repoPathsByChecksum map[string][]string + repoPathsByBucketURL map[string][]string + repoChecksumsByPath map[string]string + recordsByBucketURL map[string][]projectRecordState + equivalentRecordKeys equivalentRecordKeyIndex +} + +type equivalentRecordKeyIndex struct { + byName map[string]struct{} + byNameSize map[string]map[int64]struct{} + unknownSizeByName map[string]struct{} +} + +func (service *StorageAnalyticsService) loadStorageChainInventory(ctx context.Context, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) ([]RepoInventoryFile, error) { + index, err := loadOrBuildRepoAnalyticsIndex(ctx, mirrorPath, ref, repo, hash) + if err != nil { + return nil, err + } + return filterRepoInventoryFiles(index, gitSubpath) +} + +type storageChainInputs struct { + inventory []RepoInventoryFile + recordSet *storageAuditRecordSet + scopes []domain.StorageBucketScope + bucketSummary *gintegrationsyfon.ProjectBucketSummary + bucketObjects []gintegrationsyfon.ProjectBucketObject + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + bucketInventoryErr error +} + +func (service *StorageAnalyticsService) loadStorageChainInputs(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, bucketMode string, validationMode string, bucketPathPrefix string, forceRefresh bool, timings *StorageChainAuditTimings) (*storageChainInputs, error) { + type inventoryResult struct { + inventory []RepoInventoryFile + err error + } + type recordResult struct { + recordSet *storageAuditRecordSet + err error + } + type scopeResult struct { + scopes []domain.StorageBucketScope + err error + } + type bucketResult struct { + bucketSummary *gintegrationsyfon.ProjectBucketSummary + bucketObjects []gintegrationsyfon.ProjectBucketObject + bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + err error + } + + inventoryCh := make(chan inventoryResult, 1) + recordCh := make(chan recordResult, 1) + scopeCh := make(chan scopeResult, 1) + bucketCh := make(chan bucketResult, 1) + + go func() { + start := time.Now() + timings.StageStart("repo_index") + inventory, err := service.loadStorageChainInventory(ctx, ref, gitSubpath, mirrorPath, repo, hash) + timings.Record("repo_index", time.Since(start)) + timings.RecordMemory("repo_index", "git_files", len(inventory)) + logStorageChainInputResult("repo_index", len(inventory), err) + inventoryCh <- inventoryResult{inventory: inventory, err: err} + }() + go func() { + start := time.Now() + timings.StageStart("syfon_project_records") + recordSet, err := service.loadCachedProjectAuditRecordSet(ctx, authorizationHeader, organization, project) + timings.Record("syfon_project_records", time.Since(start)) + recordCount := 0 + if recordSet != nil { + recordCount = countRecordStates(recordSet.allProjectRecords) + } + timings.RecordMemory("syfon_project_records", "syfon_records", recordCount) + logStorageChainInputResult("syfon_project_records", recordCount, err) + recordCh <- recordResult{recordSet: recordSet, err: err} + }() + go func() { + start := time.Now() + timings.StageStart("syfon_project_scopes") + scopes, err := service.loadCachedProjectChainScopeMappings(ctx, authorizationHeader, organization, project) + timings.Record("syfon_project_scopes", time.Since(start)) + logStorageChainInputResult("syfon_project_scopes", len(scopes), err) + scopeCh <- scopeResult{scopes: scopes, err: err} + }() + go func() { + if bucketMode == StorageChainBucketModeValidate && validationMode != StorageChainValidationModeList { + timings.Record("syfon_bucket_inventory_skipped", 0) + bucketCh <- bucketResult{ + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, + } + return + } + start := time.Now() + timings.StageStart("syfon_bucket_inventory") + var bucketObjects []gintegrationsyfon.ProjectBucketObject + var bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject + var err error + if bucketMode == StorageChainBucketModeValidate { + if forceRefresh && service.projectBucketCache == nil { + bucketObjects, bucketObjectsByURL, err = service.loadProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + } else { + // Full bucket LIST calls are expensive on some providers. Reuse the + // validated project inventory during the cooldown instead of issuing a + // second Syfon scan for every audit or Git ref. + bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + } + } else { + bucketObjects, bucketObjectsByURL, err = service.loadCachedProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) + } + timings.Record("syfon_bucket_inventory", time.Since(start)) + timings.RecordMemory("syfon_bucket_inventory", "bucket_objects", len(bucketObjects), "bucket_lookup", len(bucketObjectsByURL)) + logStorageChainInputResult("syfon_bucket_items", len(bucketObjects), err) + bucketCh <- bucketResult{bucketObjects: bucketObjects, bucketObjectsByURL: bucketObjectsByURL, err: err} + }() + + inventory := <-inventoryCh + recordSet := <-recordCh + scopes := <-scopeCh + bucketObjects := <-bucketCh + if inventory.err != nil { + return nil, inventory.err + } + if recordSet.err != nil { + return nil, recordSet.err + } + if scopes.err != nil { + return nil, scopes.err + } + return &storageChainInputs{ + inventory: inventory.inventory, + recordSet: recordSet.recordSet, + scopes: scopes.scopes, + bucketSummary: bucketObjects.bucketSummary, + bucketObjects: bucketObjects.bucketObjects, + bucketObjectsByURL: bucketObjects.bucketObjectsByURL, + bucketInventoryErr: bucketObjects.err, + }, nil +} + +func logStorageChainInputResult(stage string, count int, err error) { + if err != nil { + log.Printf("INFO: storage_chain_input_done stage=%s count=%d error=%q", strings.TrimSpace(stage), count, err.Error()) + return + } + log.Printf("INFO: storage_chain_input_done stage=%s count=%d", strings.TrimSpace(stage), count) +} + +type storageFindingKind string + +const ( + storageFindingNone storageFindingKind = "" + storageFindingBrokenAccessURL storageFindingKind = "broken_access_url_error" + storageFindingBrokenBucketMap storageFindingKind = "broken_bucket_mapping" + storageFindingObjectMissing storageFindingKind = "storage_object_missing" + storageFindingValidationMismatch storageFindingKind = "storage_validation_mismatch" + storageFindingProbeError storageFindingKind = "storage_probe_error" +) + +type chainAuditAccumulator struct { + findings []GitStorageChainFinding + summary GitStorageChainAuditSummary + missingBucketDebugLogCount int +} + +func newChainSummary(bucketObjectCount, syfonRecordCount, gitTrackedFileCount int) GitStorageChainAuditSummary { + return GitStorageChainAuditSummary{ + CountsByKind: map[string]int{ + "bucket_only_object": 0, + "bucket_syfon_no_git": 0, + "bucket_syfon_git_complete": 0, + "syfon_broken_bucket_mapping": 0, + "syfon_missing_bucket_object": 0, + "syfon_git_no_bucket": 0, + "git_only_no_syfon": 0, + "git_syfon_metadata_mismatch": 0, + "probe_error": 0, + }, + BucketObjectCount: bucketObjectCount, + SyfonRecordCount: syfonRecordCount, + GitTrackedFileCount: gitTrackedFileCount, + BucketInventoryAvailable: true, + } +} + +func (acc *chainAuditAccumulator) add(kind string, findings ...GitStorageChainFinding) { + acc.findings = append(acc.findings, findings...) + acc.summary.CountsByKind[kind] += len(findings) +} + +func (acc *chainAuditAccumulator) addCount(kind string, count int) { + acc.summary.CountsByKind[kind] += count +} + +func finalizeChainFindings(gitSubpath string, acc chainAuditAccumulator) *chainAuditModel { + sort.Slice(acc.findings, func(i, j int) bool { + if acc.findings[i].Kind == acc.findings[j].Kind { + return acc.findings[i].NormalizedPath < acc.findings[j].NormalizedPath + } + return acc.findings[i].Kind < acc.findings[j].Kind + }) + acc.summary.TotalFindings = len(acc.findings) + return &chainAuditModel{ + Findings: acc.findings, + Summary: acc.summary, + PathPrefix: normalizeRepoSubpath(gitSubpath), + } +} + +func summarizeChainIssueGroups(findings []GitStorageChainFinding) []GitStorageChainIssueGroup { + groups := make(map[string]*GitStorageChainIssueGroup) + groupPaths := make(map[string]map[string]struct{}) + groupObjects := make(map[string]map[string]struct{}) + for _, finding := range findings { + group := groups[finding.Kind] + if group == nil { + group = &GitStorageChainIssueGroup{Kind: finding.Kind} + groups[finding.Kind] = group + groupPaths[finding.Kind] = map[string]struct{}{} + groupObjects[finding.Kind] = map[string]struct{}{} + } + group.FindingCount++ + group.RecordCount += finding.RecordCount + group.TotalBytes += finding.SizeBytes + groupPaths[finding.Kind][finding.NormalizedPath] = struct{}{} + for _, objectID := range finding.ObjectIDs { + groupObjects[finding.Kind][objectID] = struct{}{} + } + } + kinds := make([]string, 0, len(groups)) + for kind := range groups { + kinds = append(kinds, kind) + } + sort.Strings(kinds) + out := make([]GitStorageChainIssueGroup, 0, len(kinds)) + for _, kind := range kinds { + group := *groups[kind] + group.PathCount = len(groupPaths[kind]) + group.ObjectCount = len(groupObjects[kind]) + out = append(out, group) + } + return out +} + +func (service *StorageAnalyticsService) loadStorageAuditBaseInputs(ctx context.Context, authorizationHeader string, organization string, project string, ref string, gitSubpath string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash) (*storageAuditBaseInputs, error) { + index, inventory, recordsByChecksum, usageByObjectID, err := service.loadJoinState(ctx, authorizationHeader, organization, project, ref, gitSubpath, mirrorPath, repo, hash, false) + if err != nil { + return nil, err + } + return &storageAuditBaseInputs{ + index: index, + inventory: inventory, + recordsByChecksum: recordsByChecksum, + usageByObjectID: usageByObjectID, + }, nil +} + +func (service *StorageAnalyticsService) loadScopedProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string, base *storageAuditBaseInputs) (*storageAuditRecordSet, error) { + allProjectRecords, err := service.listProjectRecordStates(ctx, authorizationHeader, organization, project, base.usageByObjectID) + if err != nil { + return nil, err + } + return &storageAuditRecordSet{ + recordsByChecksum: base.recordsByChecksum, + allProjectRecords: allProjectRecords, + }, nil +} + +func (service *StorageAnalyticsService) loadProjectScopeMappings(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + return service.loadProjectStorageScopes(ctx, authorizationHeader, organization, project) +} + +func (service *StorageAnalyticsService) loadProjectChainScopeMappings(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + scopes, err := service.storage.ListProjectScopes(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("list syfon project scopes: %w", err) + } + if len(scopes) == 0 { + scopes, err = service.loadProjectStorageScopes(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + } + sort.SliceStable(scopes, func(i, j int) bool { + iProject := strings.TrimSpace(scopes[i].ProjectID) + jProject := strings.TrimSpace(scopes[j].ProjectID) + if iProject == "" && jProject != "" { + return true + } + if iProject != "" && jProject == "" { + return false + } + if scopes[i].Bucket != scopes[j].Bucket { + return scopes[i].Bucket < scopes[j].Bucket + } + return scopes[i].Path < scopes[j].Path + }) + return scopes, nil +} + +func applyScopeCanonicalization(recordSet *storageAuditRecordSet, scopes []domain.StorageBucketScope, organization string, project string) *storageAuditRecordSet { + if recordSet == nil { + return nil + } + recordsByChecksum, allProjectRecords := applyScopedStorageMappings(recordSet.recordsByChecksum, recordSet.allProjectRecords, scopes, organization, project) + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: allProjectRecords, + } +} + +func (service *StorageAnalyticsService) loadCachedProjectAuditRecordSet(ctx context.Context, authorizationHeader string, organization string, project string) (*storageAuditRecordSet, error) { + projectRecords, err := service.loadCachedProjectAuditRecords(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + return buildProjectAuditRecordSet(projectRecords), nil +} + +func (service *StorageAnalyticsService) loadCachedProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) { + start := time.Now() + cacheKey := service.projectChainInputCacheKey(organization, project) + summary, err := service.storage.GetProjectMetricsSummary(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("get syfon project metrics summary: %w", err) + } + if summary != nil { + validator := projectAuditRecordValidatorFromSummary(*summary) + service.chainInputMu.RLock() + cached, ok := service.projectAuditCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && cached.validator == validator && time.Since(cached.cachedAt) < chainProjectRecordCacheMaxAge { + log.Printf("syfon_project_record_cache_hit org=%s project=%s record_count=%d age_ms=%d duration_ms=%d", organization, project, len(cached.records), time.Since(cached.cachedAt).Milliseconds(), time.Since(start).Milliseconds()) + return cached.records, nil + } + + workKey := projectAuditRecordWorkKey(cacheKey, validator) + service.chainInputMu.Lock() + if cached, ok := service.projectAuditCache[cacheKey]; ok && cached.validator == validator && time.Since(cached.cachedAt) < chainProjectRecordCacheMaxAge { + service.chainInputMu.Unlock() + log.Printf("syfon_project_record_cache_hit org=%s project=%s record_count=%d age_ms=%d duration_ms=%d", organization, project, len(cached.records), time.Since(cached.cachedAt).Milliseconds(), time.Since(start).Milliseconds()) + return cached.records, nil + } + if inflight, ok := service.projectAuditWork[workKey]; ok { + service.chainInputMu.Unlock() + <-inflight.done + if inflight.err != nil { + return nil, inflight.err + } + log.Printf("syfon_project_record_cache_wait org=%s project=%s record_count=%d duration_ms=%d", organization, project, len(inflight.records), time.Since(start).Milliseconds()) + return inflight.records, nil + } + inflight := &inflightProjectAuditRecordState{ + done: make(chan struct{}), + validator: validator, + } + service.projectAuditWork[workKey] = inflight + service.chainInputMu.Unlock() + log.Printf("syfon_project_record_cache_refresh org=%s project=%s validator_count=%d validator_latest=%q validator_revision=%q", organization, project, validator.RecordCount, validator.RecordLatestUpdatedTime, validator.RecordRevision) + defer func() { + service.chainInputMu.Lock() + delete(service.projectAuditWork, workKey) + close(inflight.done) + service.chainInputMu.Unlock() + }() + + projectRecords, err := service.storage.ListProjectAuditRecords(context.WithoutCancel(ctx), authorizationHeader, organization, project, "") + if err != nil { + inflight.err = fmt.Errorf("list syfon project audit records: %w", err) + return nil, inflight.err + } + copiedRecords := append([]gintegrationsyfon.ProjectRecord(nil), projectRecords...) + service.chainInputMu.Lock() + service.projectAuditCache[cacheKey] = cachedProjectAuditRecordState{ + records: copiedRecords, + validator: validator, + cachedAt: time.Now(), + } + service.chainInputMu.Unlock() + inflight.records = copiedRecords + log.Printf("syfon_project_record_cache_refreshed org=%s project=%s record_count=%d duration_ms=%d", organization, project, len(copiedRecords), time.Since(start).Milliseconds()) + return copiedRecords, nil + } + projectRecords, err := service.storage.ListProjectAuditRecords(ctx, authorizationHeader, organization, project, "") + if err != nil { + return nil, fmt.Errorf("list syfon project audit records: %w", err) + } + log.Printf("syfon_project_record_cache_uncached org=%s project=%s record_count=%d duration_ms=%d", organization, project, len(projectRecords), time.Since(start).Milliseconds()) + return projectRecords, nil +} + +func projectAuditRecordWorkKey(cacheKey string, validator projectAuditRecordValidator) string { + return fmt.Sprintf("%s::records::%d::%s::%s", cacheKey, validator.RecordCount, validator.RecordLatestUpdatedTime, validator.RecordRevision) +} + +func buildProjectAuditRecordSet(projectRecords []gintegrationsyfon.ProjectRecord) *storageAuditRecordSet { + recordsByChecksum := make(map[string][]projectRecordState) + for _, record := range projectRecords { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + record.Checksum = normalizedChecksum + recordsByChecksum[normalizedChecksum] = append(recordsByChecksum[normalizedChecksum], projectRecordState{ + ProjectRecord: record, + }) + } + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: recordsByChecksum, + } +} + +func projectAuditRecordValidatorFromSummary(summary gintegrationsyfon.ProjectMetricsSummary) projectAuditRecordValidator { + return projectAuditRecordValidator{ + RecordCount: summary.RecordCount, + RecordLatestUpdatedTime: strings.TrimSpace(summary.RecordLatestUpdatedTime), + RecordRevision: strings.TrimSpace(summary.RecordRevision), + } +} + +func (service *StorageAnalyticsService) loadProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { + bucketObjects, err := service.storage.ListProjectBucketObjects(ctx, authorizationHeader, organization, project, bucketPathPrefix) + if err != nil { + return nil, nil, fmt.Errorf("list syfon project bucket objects: %w", err) + } + objects, lookup := buildBucketObjectLookup(bucketObjects) + return objects, lookup, nil +} + +func (service *StorageAnalyticsService) loadProjectBucketValidationInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { + bucketObjects, err := service.storage.ListProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) + if err != nil { + return nil, nil, fmt.Errorf("list syfon project bucket inventory: %w", err) + } + objects, lookup := buildBucketObjectLookup(bucketObjects) + return objects, lookup, nil +} + +func (service *StorageAnalyticsService) loadCachedProjectChainScopeMappings(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + service.chainInputMu.RLock() + cached, ok := service.chainInputCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && time.Now().Before(cached.expiresAt) && cached.projectScopes != nil { + return append([]domain.StorageBucketScope(nil), cached.projectScopes...), nil + } + scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { + state.projectScopes = append([]domain.StorageBucketScope(nil), scopes...) + }) + return scopes, nil +} + +func (service *StorageAnalyticsService) loadCachedProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + "::bucket-items::" + normalizeRepoSubpath(bucketPathPrefix) + service.chainInputMu.RLock() + cached, ok := service.chainInputCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && time.Now().Before(cached.expiresAt) && cached.bucketObjects != nil && cached.bucketObjectsByURL != nil { + objects, lookup := cloneBucketInventory(cached.bucketObjects, cached.bucketObjectsByURL) + return objects, lookup, nil + } + bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) + if err != nil { + return nil, nil, err + } + service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { + state.bucketObjects, state.bucketObjectsByURL = cloneBucketInventory(bucketObjects, bucketObjectsByURL) + }) + objects, lookup := cloneBucketInventory(bucketObjects, bucketObjectsByURL) + return objects, lookup, nil +} + +func (service *StorageAnalyticsService) loadCachedProjectBucketValidationInventory(ctx context.Context, authorizationHeader string, organization string, project string, bucketPathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject, error) { + cacheKey := service.projectChainInputCacheKey(organization, project) + "::bucket-validation-inventory::" + normalizeRepoSubpath(bucketPathPrefix) + var stale cachedProjectBucketInventory + var hasStale bool + if cache := service.projectBucketCache; cache != nil { + redisKey := projectBucketInventoryCacheKey(organization, project, bucketPathPrefix) + cached, ok, err := cache.Get(ctx, redisKey) + if err != nil { + log.Printf("syfon_project_bucket_inventory_cache_error org=%s project=%s path_prefix=%q source=%s operation=get error=%q", organization, project, bucketPathPrefix, cache.Source(), err.Error()) + } else if ok && time.Since(cached.CachedAt) < projectBucketInventoryRefreshInterval { + objects, lookup := buildBucketObjectLookup(cached.Objects) + log.Printf("syfon_project_bucket_inventory_cache_hit org=%s project=%s path_prefix=%q source=%s object_count=%d age_seconds=%d", organization, project, bucketPathPrefix, cache.Source(), len(objects), int64(time.Since(cached.CachedAt).Seconds())) + return objects, lookup, nil + } else if ok { + stale = cached + hasStale = true + } + } + service.chainInputMu.RLock() + cached, ok := service.chainInputCache[cacheKey] + service.chainInputMu.RUnlock() + if ok && time.Now().Before(cached.expiresAt) && cached.bucketObjects != nil && cached.bucketObjectsByURL != nil { + objects, lookup := cloneBucketInventory(cached.bucketObjects, cached.bucketObjectsByURL) + return objects, lookup, nil + } + bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, bucketPathPrefix) + if err != nil { + if hasStale { + objects, lookup := buildBucketObjectLookup(stale.Objects) + log.Printf("syfon_project_bucket_inventory_cache_stale_fallback org=%s project=%s path_prefix=%q source=%s object_count=%d age_seconds=%d refresh_error=%q", organization, project, bucketPathPrefix, service.projectBucketCache.Source(), len(objects), int64(time.Since(stale.CachedAt).Seconds()), err.Error()) + return objects, lookup, nil + } + return nil, nil, err + } + if cache := service.projectBucketCache; cache != nil && len(bucketObjects) > 0 { + redisKey := projectBucketInventoryCacheKey(organization, project, bucketPathPrefix) + value := cachedProjectBucketInventory{CachedAt: time.Now(), Objects: append([]gintegrationsyfon.ProjectBucketObject(nil), bucketObjects...)} + if err := cache.Set(ctx, redisKey, value, projectBucketInventoryStaleTTL); err != nil { + log.Printf("syfon_project_bucket_inventory_cache_error org=%s project=%s path_prefix=%q source=%s operation=set error=%q", organization, project, bucketPathPrefix, cache.Source(), err.Error()) + } else { + log.Printf("syfon_project_bucket_inventory_cache_store org=%s project=%s path_prefix=%q source=%s object_count=%d refresh_interval_seconds=%d stale_ttl_seconds=%d", organization, project, bucketPathPrefix, cache.Source(), len(bucketObjects), int64(projectBucketInventoryRefreshInterval.Seconds()), int64(projectBucketInventoryStaleTTL.Seconds())) + } + } + service.updateChainInputCache(cacheKey, func(state *cachedChainInputState) { + state.bucketObjects, state.bucketObjectsByURL = cloneBucketInventory(bucketObjects, bucketObjectsByURL) + }) + objects, lookup := cloneBucketInventory(bucketObjects, bucketObjectsByURL) + return objects, lookup, nil +} + +func (service *StorageAnalyticsService) projectChainInputCacheKey(organization string, project string) string { + return strings.TrimSpace(organization) + "/" + strings.TrimSpace(project) +} + +func (service *StorageAnalyticsService) updateChainInputCache(cacheKey string, update func(*cachedChainInputState)) { + service.chainInputMu.Lock() + defer service.chainInputMu.Unlock() + state := service.chainInputCache[cacheKey] + if time.Now().After(state.expiresAt) { + state = cachedChainInputState{} + } + state.expiresAt = time.Now().Add(chainInputCacheTTL) + update(&state) + service.chainInputCache[cacheKey] = state +} + +func (service *StorageAnalyticsService) evictProjectChainInputCache(organization string, project string) { + baseKey := service.projectChainInputCacheKey(organization, project) + prefix := baseKey + "::" + service.chainInputMu.Lock() + defer service.chainInputMu.Unlock() + for key := range service.chainInputCache { + if key == baseKey || strings.HasPrefix(key, prefix) { + delete(service.chainInputCache, key) + } + } +} + +func cloneBucketInventory(bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { + objects := append([]gintegrationsyfon.ProjectBucketObject(nil), bucketObjects...) + lookup := make(map[string]gintegrationsyfon.ProjectBucketObject, len(bucketObjectsByURL)) + for objectURL, item := range bucketObjectsByURL { + lookup[objectURL] = item + } + return objects, lookup +} + +func buildBucketObjectLookup(bucketObjects []gintegrationsyfon.ProjectBucketObject) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { + bucketObjectsByURL := make(map[string]gintegrationsyfon.ProjectBucketObject, len(bucketObjects)) + for _, item := range bucketObjects { + if objectURL := canonicalStorageURL(item.Bucket, item.Key, item.ObjectURL); objectURL != "" { + bucketObjectsByURL[objectURL] = item + } + } + return append([]gintegrationsyfon.ProjectBucketObject(nil), bucketObjects...), bucketObjectsByURL +} + +func synthesizeBucketInventoryFromProbes(allProjectRecords map[string][]projectRecordState) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { + if len(allProjectRecords) == 0 { + return []gintegrationsyfon.ProjectBucketObject{}, map[string]gintegrationsyfon.ProjectBucketObject{} + } + bucketObjectsByURL := make(map[string]gintegrationsyfon.ProjectBucketObject) + for _, group := range allProjectRecords { + for _, record := range group { + for _, probe := range record.AccessProbes { + if !strings.EqualFold(strings.TrimSpace(probe.Status), "present") { + continue + } + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL) + if objectURL == "" { + continue + } + if _, ok := bucketObjectsByURL[objectURL]; ok { + continue + } + bucketObjectsByURL[objectURL] = gintegrationsyfon.ProjectBucketObject{ + ObjectURL: objectURL, + Provider: strings.TrimSpace(probe.Provider), + Bucket: strings.TrimSpace(probe.Bucket), + Key: strings.TrimSpace(probe.Key), + Path: strings.TrimSpace(probe.Path), + SizeBytes: derefInt64(probe.SizeBytes), + MetaSHA256: strings.TrimSpace(probe.MetaSHA256), + ETag: strings.TrimSpace(probe.ETag), + LastModified: strings.TrimSpace(probe.LastModified), + } + } + } + } + bucketObjects := make([]gintegrationsyfon.ProjectBucketObject, 0, len(bucketObjectsByURL)) + for _, item := range bucketObjectsByURL { + bucketObjects = append(bucketObjects, item) + } + sort.Slice(bucketObjects, func(i, j int) bool { + return bucketObjects[i].ObjectURL < bucketObjects[j].ObjectURL + }) + return bucketObjects, bucketObjectsByURL +} + +func shouldDegradeBucketInventory(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(strings.TrimSpace(err.Error())) + return strings.Contains(message, "project-bucket") && + (strings.Contains(message, "status 403") || + strings.Contains(message, "status 409") || + strings.Contains(message, "permission denied") || + strings.Contains(message, "provider denied list access") || + strings.Contains(message, "bucket inventory request") || + strings.Contains(message, "bucket target may be missing or inaccessible")) +} + +func (service *StorageAnalyticsService) attachProjectStorageProbes(ctx context.Context, authorizationHeader string, recordSet *storageAuditRecordSet) (*storageAuditRecordSet, error) { + recordsByChecksum, allProjectRecords, err := service.attachStorageProbes(ctx, authorizationHeader, recordSet.recordsByChecksum, recordSet.allProjectRecords) + if err != nil { + return nil, err + } + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: allProjectRecords, + }, nil +} + +func (service *StorageAnalyticsService) attachProjectStorageListValidations(ctx context.Context, authorizationHeader string, recordSet *storageAuditRecordSet) (*storageAuditRecordSet, error) { + recordsByChecksum, allProjectRecords, err := service.attachStorageListValidations(ctx, authorizationHeader, recordSet.recordsByChecksum, recordSet.allProjectRecords) + if err != nil { + return nil, err + } + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: allProjectRecords, + }, nil +} + +func attachProjectStorageInventoryValidations(recordSet *storageAuditRecordSet, inventory []RepoInventoryFile, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, scopes []domain.StorageBucketScope, organization string, project string) (*storageAuditRecordSet, error) { + if recordSet == nil { + return nil, nil + } + inventoryBuckets := projectInventoryBuckets(bucketObjectsByURL) + repoPathsByChecksum := repoPathsByChecksumForInventory(inventory) + attach := func(input map[string][]projectRecordState) (map[string][]projectRecordState, error) { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + probes, err := inventoryValidationProbesForRecord(record, repoPathsByChecksum[normalizeAnalyticsChecksum(record.Checksum)], bucketObjectsByURL, inventoryBuckets, scopes, organization, project) + if err != nil { + return nil, err + } + clone.AccessProbes = probes + states = append(states, clone) + } + out[checksum] = states + } + return out, nil + } + recordsByChecksum, err := attach(recordSet.recordsByChecksum) + if err != nil { + return nil, err + } + allProjectRecords, err := attach(recordSet.allProjectRecords) + if err != nil { + return nil, err + } + return &storageAuditRecordSet{ + recordsByChecksum: recordsByChecksum, + allProjectRecords: allProjectRecords, + }, nil +} + +func repoPathsByChecksumForInventory(inventory []RepoInventoryFile) map[string][]string { + out := make(map[string][]string) + for _, item := range inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" { + continue + } + out[checksum] = append(out[checksum], item.RepoPath) + } + for checksum, paths := range out { + out[checksum] = uniqueStrings(paths) + } + return out +} + +func inventoryValidationProbesForRecord(record projectRecordState, repoPaths []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, inventoryBuckets map[string]struct{}, scopes []domain.StorageBucketScope, organization string, project string) ([]gintegrationsyfon.BulkStorageProbeResult, error) { + probes := make([]gintegrationsyfon.BulkStorageProbeResult, 0) + seen := make(map[string]struct{}) + accessURLs, err := recordAccessURLsForProjectInventory(record.AccessURLs, inventoryBuckets, scopes, organization, project) + if err != nil { + return nil, err + } + accessURLs = append(accessURLs, projectScopeRepoPathObjectURLs(repoPaths, scopes, organization, project)...) + accessURLs = uniqueStrings(accessURLs) + for _, accessURL := range accessURLs { + objectURL := canonicalStorageURL("", "", accessURL) + if objectURL == "" { + continue + } + bucket, _, _ := parseStorageURL(objectURL) + if len(inventoryBuckets) > 0 { + if _, ok := inventoryBuckets[bucket]; !ok { + return nil, fmt.Errorf("storage access URL mapped to %q, but bucket %q is not present in project bucket inventory for project %s/%s", objectURL, bucket, strings.TrimSpace(organization), strings.TrimSpace(project)) + } + } + if _, ok := seen[objectURL]; ok { + continue + } + seen[objectURL] = struct{}{} + expectedName := expectedStorageObjectNameForListValidation(objectURL, record.Name) + if item, ok := bucketObjectsByURL[objectURL]; ok { + probes = append(probes, inventoryPresentProbe(record, objectURL, expectedName, item)) + continue + } + probes = append(probes, inventoryMissingProbe(record, objectURL, expectedName)) + } + return probes, nil +} + +func recordAccessURLsForProjectInventory(accessURLs []string, inventoryBuckets map[string]struct{}, scopes []domain.StorageBucketScope, organization string, project string) ([]string, error) { + out := make([]string, 0, len(accessURLs)) + for _, accessURL := range accessURLs { + mapped := false + if objectURL := canonicalizeScopedStorageURL(accessURL, scopes, organization, project); objectURL != "" { + out = append(out, objectURL) + mapped = true + } + objectURL := canonicalStorageURL("", "", accessURL) + if objectURL == "" { + continue + } + bucket, _, ok := parseStorageURL(objectURL) + if !ok { + continue + } + if len(inventoryBuckets) > 0 { + if _, ok := inventoryBuckets[bucket]; ok { + out = append(out, objectURL) + mapped = true + } + } + if !mapped { + return nil, fmt.Errorf("storage access URL %q could not be mapped into bucket scopes for project %s/%s", strings.TrimSpace(accessURL), strings.TrimSpace(organization), strings.TrimSpace(project)) + } + } + return uniqueStrings(out), nil +} + +func projectScopeRepoPathObjectURLs(repoPaths []string, scopes []domain.StorageBucketScope, organization string, project string) []string { + out := make([]string, 0, len(repoPaths)) + for _, repoPath := range repoPaths { + normalizedPath := normalizeRepoSubpath(repoPath) + if normalizedPath == "" { + continue + } + for _, scope := range scopes { + if !storageScopeApplies(scope, organization, project) { + continue + } + bucket, prefix, ok := parseStorageScopePath(scope) + if !ok || strings.TrimSpace(bucket) == "" { + continue + } + key := normalizedPath + if strings.TrimSpace(prefix) != "" { + key = path.Join(strings.Trim(strings.TrimSpace(prefix), "/"), normalizedPath) + } + if objectURL := canonicalStorageURL(bucket, key, ""); objectURL != "" { + out = append(out, objectURL) + } + } + } + return uniqueStrings(out) +} + +func projectInventoryBuckets(bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) map[string]struct{} { + out := make(map[string]struct{}) + for objectURL, item := range bucketObjectsByURL { + if bucket := strings.TrimSpace(item.Bucket); bucket != "" { + out[bucket] = struct{}{} + continue + } + bucket, _, ok := parseStorageURL(objectURL) + if ok { + out[bucket] = struct{}{} + } + } + return out +} + +func inventoryPresentProbe(record projectRecordState, objectURL string, expectedName string, item gintegrationsyfon.ProjectBucketObject) gintegrationsyfon.BulkStorageProbeResult { + size := item.SizeBytes + sizeMatch := storageSizesMatchForAudit(record.Size, item.SizeBytes) + nameMatch := true + mismatches := make([]string, 0, 1) + if !sizeMatch { + mismatches = append(mismatches, "size_mismatch") + } + validationStatus := "matched" + if len(mismatches) > 0 { + validationStatus = "mismatched" + } + return gintegrationsyfon.BulkStorageProbeResult{ + ID: storageListValidationRequestKey(objectURL, record.Size, expectedName), + Operation: StorageChainValidationModeInventory, + ObjectURL: objectURL, + Provider: strings.TrimSpace(item.Provider), + Bucket: strings.TrimSpace(item.Bucket), + Key: strings.Trim(strings.TrimSpace(item.Key), "/"), + Path: strings.TrimSpace(item.Path), + Exists: true, + Status: "present", + SizeBytes: &size, + MetaSHA256: strings.TrimSpace(item.MetaSHA256), + ETag: strings.TrimSpace(item.ETag), + LastModified: strings.TrimSpace(item.LastModified), + ValidationStatus: validationStatus, + SizeMatch: &sizeMatch, + NameMatch: &nameMatch, + ValidationMismatches: mismatches, + } +} + +func inventoryMissingProbe(record projectRecordState, objectURL string, expectedName string) gintegrationsyfon.BulkStorageProbeResult { + bucket, key, _ := parseStorageURL(objectURL) + sizeMatch := false + nameMatch := false + return gintegrationsyfon.BulkStorageProbeResult{ + ID: storageListValidationRequestKey(objectURL, record.Size, expectedName), + Operation: StorageChainValidationModeInventory, + ObjectURL: objectURL, + Provider: "s3", + Bucket: bucket, + Key: key, + Path: path.Base(key), + Exists: false, + Status: "unknown", + Error: fmt.Sprintf("object %q was absent from project bucket inventory and requires exact verification", objectURL), + ErrorKind: "inventory_miss", + ValidationStatus: "unverifiable", + SizeMatch: &sizeMatch, + NameMatch: &nameMatch, + ValidationMismatches: []string{}, + } +} + +func (service *StorageAnalyticsService) buildStorageChainView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, inventory []RepoInventoryFile, scopes []domain.StorageBucketScope, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, bucketInventoryErr error, bucketMode string, validationMode string, timings *StorageChainAuditTimings) (*storageAuditStorageView, error) { + recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) + view := &storageAuditStorageView{ + scopes: append([]domain.StorageBucketScope(nil), scopes...), + recordsByChecksum: recordSet.recordsByChecksum, + allProjectRecords: recordSet.allProjectRecords, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, + bucketInventoryAvailable: true, + } + if bucketMode == StorageChainBucketModeValidate { + if bucketInventoryErr != nil { + view.bucketInventoryAvailable = false + view.bucketInventoryError = strings.TrimSpace(bucketInventoryErr.Error()) + } + if validationMode == StorageChainValidationModeInventory { + return view, nil + } + if validationMode == StorageChainValidationModeList { + if bucketInventoryErr != nil { + return nil, bucketInventoryErr + } + view.bucketObjects = bucketObjects + view.bucketObjectsByURL = bucketObjectsByURL + validateStart := time.Now() + probedRecordSet, err := attachProjectStorageInventoryValidations(recordSet, inventory, view.bucketObjectsByURL, scopes, organization, project) + if err != nil { + return nil, err + } + timings.Record("inventory_list_validation", time.Since(validateStart)) + + candidateStart := time.Now() + var candidates *storageAuditRecordSet + if len(view.bucketObjectsByURL) == 0 { + candidates = selectInventoryMissRecordSet(probedRecordSet) + } + timings.Record("exact_list_candidate_selection", time.Since(candidateStart)) + if candidates != nil { + probeStart := time.Now() + exactRecordSet, probeErr := service.attachProjectStorageListValidations(ctx, authorizationHeader, candidates) + timings.Record("targeted_exact_list_validation", time.Since(probeStart)) + if probeErr != nil { + return nil, probeErr + } + probedRecordSet = mergeRecordSetProbes(probedRecordSet, exactRecordSet) + view.bucketObjects, view.bucketObjectsByURL = mergeBucketInventoryWithPresentProbes(view.bucketObjects, view.bucketObjectsByURL, exactRecordSet) + } + timings.RecordMemory("inventory_list_validation", "syfon_records", countRecordStates(probedRecordSet.allProjectRecords), "bucket_objects", len(view.bucketObjectsByURL), "exact_probe_records", countRecordSet(candidates)) + view.recordsByChecksum = probedRecordSet.recordsByChecksum + view.allProjectRecords = probedRecordSet.allProjectRecords + return view, nil + } + probeStart := time.Now() + var ( + probedRecordSet *storageAuditRecordSet + probeErr error + stage string + ) + switch validationMode { + case StorageChainValidationModeMetadata: + probedRecordSet, probeErr = service.attachProjectStorageProbes(ctx, authorizationHeader, recordSet) + stage = "bulk_metadata_validation" + default: + probedRecordSet, probeErr = service.attachProjectStorageListValidations(ctx, authorizationHeader, recordSet) + stage = "bulk_list_validation" + } + timings.Record(stage, time.Since(probeStart)) + if probedRecordSet != nil { + timings.RecordMemory(stage, "syfon_records", countRecordStates(probedRecordSet.allProjectRecords)) + } + if probeErr != nil { + return nil, probeErr + } + view.recordsByChecksum = probedRecordSet.recordsByChecksum + view.allProjectRecords = probedRecordSet.allProjectRecords + view.bucketObjects, view.bucketObjectsByURL = synthesizeBucketInventoryFromProbes(probedRecordSet.allProjectRecords) + return view, nil + } + if bucketInventoryErr != nil { + return nil, bucketInventoryErr + } + view.bucketObjects, view.bucketObjectsByURL = cloneBucketInventory(bucketObjects, bucketObjectsByURL) + if validationMode == StorageChainValidationModeInventory || validationMode == StorageChainValidationModeList { + return view, nil + } + + probeSelectStart := time.Now() + probeCandidates := selectTargetedProbeRecordSet(recordSet, view.bucketObjectsByURL) + timings.Record("targeted_probe_selection", time.Since(probeSelectStart)) + if probeCandidates == nil { + return view, nil + } + probeStart := time.Now() + probedSubset, probeErr := service.attachProjectStorageProbes(ctx, authorizationHeader, probeCandidates) + timings.Record("bulk_probe", time.Since(probeStart)) + if probedSubset != nil { + timings.RecordMemory("bulk_probe", "probe_records", countRecordStates(probedSubset.allProjectRecords)) + } + if probeErr != nil { + return nil, probeErr + } + merged := mergeRecordSetProbes(recordSet, probedSubset) + view.recordsByChecksum = merged.recordsByChecksum + view.allProjectRecords = merged.allProjectRecords + return view, nil +} + +func (service *StorageAnalyticsService) loadStorageAuditStorageView(ctx context.Context, authorizationHeader string, organization string, project string, recordSet *storageAuditRecordSet, includeBucketInventory bool, includeProbes bool) (*storageAuditStorageView, error) { + scopes, err := service.loadProjectScopeMappings(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, err + } + recordSet = applyScopeCanonicalization(recordSet, scopes, organization, project) + if includeProbes { + recordSet, err = service.attachProjectStorageProbes(ctx, authorizationHeader, recordSet) + if err != nil { + return nil, err + } + } + view := &storageAuditStorageView{ + scopes: scopes, + recordsByChecksum: recordSet.recordsByChecksum, + allProjectRecords: recordSet.allProjectRecords, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{}, + bucketInventoryAvailable: includeBucketInventory, + } + if includeBucketInventory { + bucketObjects, bucketObjectsByURL, err := service.loadProjectBucketInventory(ctx, authorizationHeader, organization, project, "") + if err != nil { + if !includeProbes || !shouldDegradeBucketInventory(err) { + return nil, err + } + view.bucketInventoryAvailable = false + view.bucketInventoryError = strings.TrimSpace(err.Error()) + bucketObjects, bucketObjectsByURL = synthesizeBucketInventoryFromProbes(recordSet.allProjectRecords) + } + view.bucketObjects = bucketObjects + view.bucketObjectsByURL = bucketObjectsByURL + } + return view, nil +} + +func cloneRecordStateMap(input map[string][]projectRecordState) map[string][]projectRecordState { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + clone.CanonicalAccessURLs = append([]string(nil), record.CanonicalAccessURLs...) + clone.AccessProbes = append([]gintegrationsyfon.BulkStorageProbeResult(nil), record.AccessProbes...) + states = append(states, clone) + } + out[checksum] = states + } + return out +} + +func selectTargetedProbeRecordSet(recordSet *storageAuditRecordSet, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) *storageAuditRecordSet { + if recordSet == nil { + return nil + } + selected := make(map[string][]projectRecordState) + for checksum, group := range recordSet.allProjectRecords { + for _, record := range group { + if !recordNeedsTargetedProbe(record, bucketObjectsByURL) { + continue + } + selected[checksum] = append(selected[checksum], record) + } + } + if len(selected) == 0 { + return nil + } + return &storageAuditRecordSet{ + recordsByChecksum: cloneRecordStateMap(selected), + allProjectRecords: cloneRecordStateMap(selected), + } +} + +func selectInventoryMissRecordSet(recordSet *storageAuditRecordSet) *storageAuditRecordSet { + if recordSet == nil { + return nil + } + selected := make(map[string][]projectRecordState) + for checksum, group := range recordSet.allProjectRecords { + for _, record := range group { + assessment := assessStorageRecordEvidence(record, false) + if assessment.Present || !assessment.HasInventoryMiss { + continue + } + selected[checksum] = append(selected[checksum], record) + } + } + if len(selected) == 0 { + return nil + } + return &storageAuditRecordSet{ + recordsByChecksum: cloneRecordStateMap(selected), + allProjectRecords: cloneRecordStateMap(selected), + } +} + +func countRecordSet(recordSet *storageAuditRecordSet) int { + if recordSet == nil { + return 0 + } + return countRecordStates(recordSet.allProjectRecords) +} + +func mergeBucketInventoryWithPresentProbes(bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, recordSet *storageAuditRecordSet) ([]gintegrationsyfon.ProjectBucketObject, map[string]gintegrationsyfon.ProjectBucketObject) { + if recordSet == nil { + return bucketObjects, bucketObjectsByURL + } + if bucketObjectsByURL == nil { + bucketObjectsByURL = make(map[string]gintegrationsyfon.ProjectBucketObject) + } + for _, group := range recordSet.allProjectRecords { + for _, record := range group { + for _, probe := range record.AccessProbes { + if strings.TrimSpace(probe.Status) != "present" { + continue + } + objectURL := canonicalStorageURL(probe.Bucket, probe.Key, probe.ObjectURL) + if objectURL == "" { + continue + } + if _, ok := bucketObjectsByURL[objectURL]; ok { + continue + } + item := bucketObjectFromPresentProbe(objectURL, probe) + bucketObjectsByURL[objectURL] = item + bucketObjects = append(bucketObjects, item) + } + } + } + return bucketObjects, bucketObjectsByURL +} + +func bucketObjectFromPresentProbe(objectURL string, probe gintegrationsyfon.BulkStorageProbeResult) gintegrationsyfon.ProjectBucketObject { + return gintegrationsyfon.ProjectBucketObject{ + ObjectURL: objectURL, + Provider: strings.TrimSpace(probe.Provider), + Bucket: strings.TrimSpace(probe.Bucket), + Key: strings.Trim(strings.TrimSpace(probe.Key), "/"), + Path: strings.TrimSpace(probe.Path), + SizeBytes: derefInt64(probe.SizeBytes), + MetaSHA256: strings.TrimSpace(probe.MetaSHA256), + ETag: strings.TrimSpace(probe.ETag), + LastModified: strings.TrimSpace(probe.LastModified), + } +} + +func recordNeedsTargetedProbe(record projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + resolution := resolveRecordStorage(record, bucketObjectsByURL) + if len(resolution.matchedBucketObjectURLs) == 0 { + return true + } + if !sameStringSet(rawAccessURLsForRecord(record), accessURLsForStorage(record)) { + return true + } + checksum := normalizeAnalyticsChecksum(record.Checksum) + if checksum == "" { + return false + } + for _, objectURL := range resolution.matchedBucketObjectURLs { + item, ok := bucketObjectsByURL[objectURL] + if !ok { + continue + } + if normalizeAnalyticsChecksum(item.MetaSHA256) != "" { + return false + } + } + return true +} + +func sameStringSet(left []string, right []string) bool { + a := uniqueStrings(left) + b := uniqueStrings(right) + if len(a) != len(b) { + return false + } + set := make(map[string]struct{}, len(a)) + for _, value := range a { + set[strings.TrimSpace(value)] = struct{}{} + } + for _, value := range b { + if _, ok := set[strings.TrimSpace(value)]; !ok { + return false + } + } + return true +} + +func mergeRecordSetProbes(base *storageAuditRecordSet, probed *storageAuditRecordSet) *storageAuditRecordSet { + if base == nil { + return probed + } + if probed == nil { + return base + } + probesByObjectID := make(map[string][]gintegrationsyfon.BulkStorageProbeResult) + for _, group := range probed.allProjectRecords { + for _, record := range group { + probesByObjectID[strings.TrimSpace(record.ObjectID)] = append([]gintegrationsyfon.BulkStorageProbeResult(nil), record.AccessProbes...) + } + } + attach := func(input map[string][]projectRecordState) map[string][]projectRecordState { + out := make(map[string][]projectRecordState, len(input)) + for checksum, group := range input { + states := make([]projectRecordState, 0, len(group)) + for _, record := range group { + clone := record + if probes, ok := probesByObjectID[strings.TrimSpace(record.ObjectID)]; ok { + clone.AccessProbes = append([]gintegrationsyfon.BulkStorageProbeResult(nil), probes...) + } + states = append(states, clone) + } + out[checksum] = states + } + return out + } + return &storageAuditRecordSet{ + recordsByChecksum: attach(base.recordsByChecksum), + allProjectRecords: attach(base.allProjectRecords), + } +} + +func derefInt64(value *int64) int64 { + if value == nil { + return 0 + } + return *value +} + +func buildStorageChainIndex(inventory []RepoInventoryFile, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, scopes []domain.StorageBucketScope, organization string, project string) storageChainIndex { + repoPathsByChecksum := make(map[string][]string, len(inventory)) + repoPathsByBucketURL := make(map[string][]string, len(inventory)) + repoChecksumsByPath := make(map[string]string, len(inventory)) + for _, item := range inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" { + continue + } + repoPath := normalizeRepoSubpath(item.RepoPath) + if repoPath == "" { + continue + } + repoPathsByChecksum[checksum] = append(repoPathsByChecksum[checksum], repoPath) + repoChecksumsByPath[repoPath] = checksum + for _, bucketURL := range projectScopeRepoPathObjectURLs([]string{repoPath}, scopes, organization, project) { + repoPathsByBucketURL[bucketURL] = append(repoPathsByBucketURL[bucketURL], repoPath) + } + } + allRecords := flattenRecordStates(allProjectRecords) + recordsByBucketURL := make(map[string][]projectRecordState) + for _, record := range allRecords { + for _, bucketURL := range recordBucketURLs(record) { + recordsByBucketURL[bucketURL] = append(recordsByBucketURL[bucketURL], record) + } + } + return storageChainIndex{ + inventory: inventory, + allRecords: allRecords, + bucketObjectsByURL: bucketObjectsByURL, + repoPathsByChecksum: repoPathsByChecksum, + repoPathsByBucketURL: repoPathsByBucketURL, + repoChecksumsByPath: repoChecksumsByPath, + recordsByBucketURL: recordsByBucketURL, + equivalentRecordKeys: buildEquivalentRecordKeyIndex(allRecords), + } +} + +func buildStorageChainAuditModel(gitSubpath string, inventory []RepoInventoryFile, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, scopes []domain.StorageBucketScope, organization string, project string, includeBucketOrigin bool) *chainAuditModel { + index := buildStorageChainIndex(inventory, allProjectRecords, bucketObjectsByURL, scopes, organization, project) + acc := chainAuditAccumulator{ + findings: make([]GitStorageChainFinding, 0), + summary: newChainSummary(len(bucketObjectsByURL), len(index.allRecords), len(inventory)), + } + if includeBucketOrigin { + buildBucketOriginChainFindings(index, &acc) + } + buildSyfonOriginChainFindings(index, &acc, !includeBucketOrigin) + buildGitOriginChainFindings(index, recordsByChecksum, allProjectRecords, scopes, organization, project, &acc) + return finalizeChainFindings(gitSubpath, acc) +} + +func buildBucketOriginChainFindings(index storageChainIndex, acc *chainAuditAccumulator) { + bucketURLs := make([]string, 0, len(index.bucketObjectsByURL)) + for bucketURL := range index.bucketObjectsByURL { + bucketURLs = append(bucketURLs, bucketURL) + } + sort.Strings(bucketURLs) + for _, bucketURL := range bucketURLs { + item := index.bucketObjectsByURL[bucketURL] + if bucketObjectHasCompleteChain(index.recordsByBucketURL[bucketURL], index.repoPathsByChecksum, index.bucketObjectsByURL) { + acc.addCount("bucket_syfon_git_complete", 1) + continue + } + if len(index.recordsByBucketURL[bucketURL]) > 0 { + // An exact Syfon-to-bucket URL match is not a bucket orphan, even + // when the Git and Syfon checksums disagree. The Syfon-origin pass + // reports that conflict without offering the bucket for deletion. + continue + } + if bucketObjectHasEquivalentSyfonRecord(item, index.equivalentRecordKeys) { + continue + } + acc.add("bucket_only_object", buildChainBucketOnlyFinding(item)) + } +} + +func buildEquivalentRecordKeyIndex(records []projectRecordState) equivalentRecordKeyIndex { + index := equivalentRecordKeyIndex{ + byName: make(map[string]struct{}), + byNameSize: make(map[string]map[int64]struct{}), + unknownSizeByName: make(map[string]struct{}), + } + for _, record := range records { + for _, accessURL := range recordStorageCandidateURLs(record) { + _, key, ok := parseStorageURL(accessURL) + if !ok { + continue + } + name := strings.TrimSpace(path.Base(key)) + if name == "" { + continue + } + index.byName[name] = struct{}{} + if record.Size <= 0 { + index.unknownSizeByName[name] = struct{}{} + continue + } + if index.byNameSize[name] == nil { + index.byNameSize[name] = make(map[int64]struct{}) + } + index.byNameSize[name][record.Size] = struct{}{} + } + } + return index +} + +func bucketObjectHasEquivalentSyfonRecord(item gintegrationsyfon.ProjectBucketObject, index equivalentRecordKeyIndex) bool { + itemName := strings.TrimSpace(path.Base(strings.TrimSpace(item.Key))) + if itemName == "" { + return false + } + if _, ok := index.byName[itemName]; !ok { + return false + } + if item.SizeBytes <= 0 { + return true + } + if _, ok := index.unknownSizeByName[itemName]; ok { + return true + } + if sizes := index.byNameSize[itemName]; sizes != nil { + _, ok := sizes[item.SizeBytes] + return ok + } + return false +} + +func recordHasResolvedStorage(record projectRecordState, bucketMatches []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) bool { + if len(bucketMatches) > 0 { + return true + } + return resolveRecordStorage(record, bucketObjectsByURL).hasPresentProbe +} + +func logMissingBucketCandidateDebug(record projectRecordState, gitPaths []string, bucketMatches []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject, acc *chainAuditAccumulator) { + if acc == nil || acc.missingBucketDebugLogCount >= storageChainValidationDebugSampleLimit { + return + } + resolution := resolveRecordStorage(record, bucketObjectsByURL) + log.Printf( + "INFO: storage_chain_missing_bucket_candidates object_id=%s checksum=%s name=%q git_paths=%q raw_access_urls=%q canonical_access_urls=%q candidate_status=%q bucket_matches=%q access_probe_status=%q", + strings.TrimSpace(record.ObjectID), + strings.TrimSpace(record.Checksum), + strings.TrimSpace(record.Name), + strings.Join(uniqueStrings(gitPaths), ","), + strings.Join(rawAccessURLsForRecord(record), ","), + strings.Join(uniqueStrings(record.CanonicalAccessURLs), ","), + strings.Join(storageCandidateStatuses(resolution.candidateURLs, bucketObjectsByURL), ","), + strings.Join(uniqueStrings(bucketMatches), ","), + strings.Join(storageProbeDebugStatuses(record.AccessProbes), ","), + ) + acc.missingBucketDebugLogCount++ +} + +func storageCandidateStatuses(candidateURLs []string, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []string { + out := make([]string, 0, len(candidateURLs)) + for _, candidate := range uniqueStrings(candidateURLs) { + status := "missing" + if _, ok := bucketObjectsByURL[candidate]; ok { + status = "found" + } + out = append(out, candidate+"="+status) + } + return out +} + +func storageProbeDebugStatuses(probes []gintegrationsyfon.BulkStorageProbeResult) []string { + out := make([]string, 0, len(probes)) + for _, probe := range probes { + objectURL := syfonProbeObjectURL(probe) + if objectURL == "" { + objectURL = strings.TrimSpace(probe.ObjectURL) + } + status := strings.TrimSpace(probe.Status) + if status == "" { + status = "unknown" + } + validation := strings.TrimSpace(probe.ValidationStatus) + if validation != "" { + status += "/" + validation + } + if errKind := strings.TrimSpace(probe.ErrorKind); errKind != "" { + status += "/" + errKind + } + out = append(out, objectURL+"="+status) + } + return out +} + +func buildSyfonOriginChainFindings(index storageChainIndex, acc *chainAuditAccumulator, countCompleteFromSyfon bool) { + for _, record := range index.allRecords { + gitPaths := uniqueStrings(index.repoPathsByChecksum[normalizeAnalyticsChecksum(record.Checksum)]) + bucketMatches := matchedBucketObjectURLs(record, index.bucketObjectsByURL) + classification := classifyStorageFinding(record, index.bucketObjectsByURL) + if len(gitPaths) == 0 && len(bucketMatches) > 0 && (classification == storageFindingNone || classification == storageFindingValidationMismatch) { + if samePathGitPaths := repoPathsForBucketURLs(index.repoPathsByBucketURL, bucketMatches); len(samePathGitPaths) > 0 { + findings := buildSamePathChecksumConflictFindings(index, record, samePathGitPaths, bucketMatches) + acc.add("git_syfon_metadata_mismatch", findings...) + continue + } + } + switch classification { + case storageFindingBrokenBucketMap: + findingRecord := repairableBrokenAccessRecord(record) + if len(findingRecord.AccessProbes) == 0 { + findingRecord = record + } + findings := buildChainRecordFindingsWithOptions("syfon_broken_bucket_mapping", findingRecord, gitPaths, bucketMatches, "Syfon access URL did not resolve through a configured bucket mapping.", true) + acc.findings = append(acc.findings, findings...) + acc.addCount("syfon_broken_bucket_mapping", len(findings)) + case storageFindingObjectMissing: + if len(gitPaths) > 0 { + logMissingBucketCandidateDebug(record, gitPaths, bucketMatches, index.bucketObjectsByURL, acc) + findings := buildChainRecordFindings("syfon_git_no_bucket", record, gitPaths, bucketMatches, "Git and Syfon matched, but the mapped bucket object does not exist.") + acc.findings = append(acc.findings, findings...) + acc.addCount("syfon_git_no_bucket", len(gitPaths)) + continue + } + acc.add("syfon_missing_bucket_object", buildChainRecordFindings("syfon_missing_bucket_object", record, nil, bucketMatches, "Syfon record points to a mapped bucket location, but the object does not exist.")...) + case storageFindingValidationMismatch: + if len(gitPaths) > 0 { + findings := buildChainRecordFindings("git_syfon_metadata_mismatch", record, gitPaths, bucketMatches, "Mapped bucket object exists, but object evidence disagrees with the Syfon/Git record.") + acc.findings = append(acc.findings, findings...) + acc.addCount("git_syfon_metadata_mismatch", len(gitPaths)) + continue + } + acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) + case storageFindingBrokenAccessURL, storageFindingProbeError: + findings := buildChainRecordFindings("probe_error", record, gitPaths, bucketMatches, "Storage probes could not confirm this mapped object. Gecko does not treat it as missing; after review, you may delete the Syfon record.") + acc.findings = append(acc.findings, findings...) + acc.addCount("probe_error", chainPathCount(gitPaths)) + case storageFindingNone: + if countCompleteFromSyfon && len(gitPaths) > 0 && recordHasResolvedStorage(record, bucketMatches, index.bucketObjectsByURL) { + acc.addCount("bucket_syfon_git_complete", len(gitPaths)) + continue + } + if len(gitPaths) == 0 && len(bucketMatches) > 0 { + acc.add("bucket_syfon_no_git", buildChainRecordFindings("bucket_syfon_no_git", record, nil, bucketMatches, "Bucket object and Syfon record matched, but no Git-tracked file matched this checksum.")...) + } + } + } +} + +func repoPathsForBucketURLs(repoPathsByBucketURL map[string][]string, bucketURLs []string) []string { + paths := make([]string, 0, len(bucketURLs)) + for _, bucketURL := range uniqueStrings(bucketURLs) { + paths = append(paths, repoPathsByBucketURL[bucketURL]...) + } + return uniqueStrings(paths) +} + +func hasSyfonRecordForBucketURLs(recordsByBucketURL map[string][]projectRecordState, bucketURLs []string) bool { + for _, bucketURL := range uniqueStrings(bucketURLs) { + if len(recordsByBucketURL[bucketURL]) > 0 { + return true + } + } + return false +} + +func buildSamePathChecksumConflictFindings(index storageChainIndex, record projectRecordState, gitPaths []string, bucketMatches []string) []GitStorageChainFinding { + findings := buildChainRecordFindings( + "git_syfon_metadata_mismatch", + record, + gitPaths, + bucketMatches, + "Git and Syfon map to the same bucket object path, but their checksums disagree. Do not delete this object automatically.", + ) + for findingIndex := range findings { + findings[findingIndex].Actionability = storageActionabilityInspectOnly + findings[findingIndex].AvailableActions = []string{storageActionInspectEvidence} + findings[findingIndex].DefaultAction = storageActionInspectEvidence + findings[findingIndex].SupportsDryRun = false + gitChecksum := index.repoChecksumsByPath[normalizeRepoSubpath(findings[findingIndex].NormalizedPath)] + findings[findingIndex].Error = "Git LFS and Syfon map to the same bucket object path but have different SHA-256 values." + findings[findingIndex].SuggestedFix = fmt.Sprintf( + "Git LFS pointer SHA-256 is %s; Syfon record SHA-256 is %s. Recompute the bucket object's SHA-256 and reconcile the stale side manually before changing Git, Syfon, or the bucket object.", + gitChecksum, + strings.TrimSpace(record.Checksum), + ) + } + return findings +} + +func buildGitOriginChainFindings(index storageChainIndex, recordsByChecksum map[string][]projectRecordState, allProjectRecords map[string][]projectRecordState, scopes []domain.StorageBucketScope, organization string, project string, acc *chainAuditAccumulator) { + for _, item := range index.inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if len(allProjectRecords[checksum]) > 0 || len(recordsByChecksum[checksum]) > 0 { + continue + } + bucketURLs := projectScopeRepoPathObjectURLs([]string{item.RepoPath}, scopes, organization, project) + if hasSyfonRecordForBucketURLs(index.recordsByBucketURL, bucketURLs) { + // The Syfon-origin pass reports an exact-path checksum conflict as a + // metadata mismatch. Do not also offer this Git pointer for Syfon + // creation just because its checksum differs. + continue + } + bucketObjectURL := "" + bucketSizeBytes := int64(0) + bucketEvaluation := "not_evaluated" + if len(bucketURLs) == 1 { + bucketObjectURL = bucketURLs[0] + if bucketObject, ok := index.bucketObjectsByURL[bucketObjectURL]; ok { + bucketSizeBytes = bucketObject.SizeBytes + bucketEvaluation = "present" + } else { + bucketEvaluation = "not_found" + } + } else if len(bucketURLs) > 1 { + bucketEvaluation = "ambiguous_mapping" + } + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence("git_only_no_syfon", "verified") + recommendedAction := "Git checksum has no matching Syfon record. Bucket presence is not claimed by this finding." + if bucketEvaluation == "present" && bucketSizeBytes == item.Size { + recommendedAction = "Git checksum has no matching Syfon record. The mapped bucket object exists and its byte size matches the Git LFS pointer; create a Syfon record after reviewing that RGW did not provide a SHA-256 for verification." + } else if bucketEvaluation == "present" { + recommendedAction = "Git checksum has no matching Syfon record. The mapped bucket object exists but its byte size differs from the Git LFS pointer, so it is not safe to create a Syfon record." + } + acc.add("git_only_no_syfon", GitStorageChainFinding{ + Kind: "git_only_no_syfon", + EvidenceStatus: "verified", + NormalizedPath: item.RepoPath, + Checksum: checksum, + SourcePaths: []string{item.RepoPath}, + ObjectIDs: []string{}, + RecordCount: 0, + SizeBytes: item.Size, + BucketObjectURL: bucketObjectURL, + BucketSizeBytes: bucketSizeBytes, + RecommendedAction: recommendedAction, + Actionability: actionability, + AvailableActions: availableActions, + DefaultAction: defaultAction, + SupportsDryRun: supportsDryRun, + Evidence: &GitAuditEvidence{ + Checksum: checksum, + SourcePaths: []string{item.RepoPath}, + ObjectIDs: []string{}, + BucketObjectURLs: bucketURLs, + BucketEvaluation: bucketEvaluation, + }, + }) + } +} diff --git a/internal/git/storage_analytics_test.go b/internal/git/storage_analytics_test.go new file mode 100644 index 0000000..e06b9ad --- /dev/null +++ b/internal/git/storage_analytics_test.go @@ -0,0 +1,4518 @@ +package git + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +type fakeStorageAnalyticsBackend struct { + projectRecords []gintegrationsyfon.ProjectRecord + projectMetricsSummary *gintegrationsyfon.ProjectMetricsSummary + bulkRecords map[string][]gintegrationsyfon.ProjectRecord + buckets map[string]domain.StorageBucket + bucketScopes map[string][]domain.StorageBucketScope + projectScopes []domain.StorageBucketScope + usageByObject map[string]gintegrationsyfon.FileUsage + bulkChecksums []string + probeResults map[string]gintegrationsyfon.BulkStorageProbeResult + listProbeResults map[string]gintegrationsyfon.BulkStorageProbeResult + bucketObjects []gintegrationsyfon.ProjectBucketObject + listProjectBucketObjectsErr error + listBucketsCalls int + listBucketScopesCalls int + listProjectAuditRecordsCalls int + listProjectAuditRecordsPathPrefix string + listProjectAuditRecordsDelay time.Duration + getProjectMetricsSummaryCalls int + listProjectScopesCalls int + listProjectFileUsageCalls int + listProjectFileUsageByObjectIDsCalls int + listProjectFileUsageObjectIDs []string + listProjectFileUsageByObjectIDsErr error + listProjectBucketObjectsCalls int + listProjectBucketObjectsPathPrefix string + listProjectBucketInventoryCalls int + listProjectBucketInventoryPathPrefix string + listProjectBucketInventoryDelay time.Duration + listProjectBucketSummaryCalls int + listProjectBucketSummaryMode string + bulkGetProjectRecordsCalls int + bulkGetDelay time.Duration + probeCalls int + probeItems []gintegrationsyfon.BulkStorageProbeItem + listProbeCalls int + listProbeItems []gintegrationsyfon.BulkStorageProbeItem + deletedIDs []string + deletedStorageIDs []string + updatedAccessMethods map[string][]gintegrationsyfon.ProjectAccessMethod + deletedBucketObjects []string + registeredObjects []gintegrationsyfon.ProjectObjectRegistration +} + +func (fake *fakeStorageAnalyticsBackend) ListBuckets(ctx context.Context, authorizationHeader string) (map[string]domain.StorageBucket, error) { + fake.listBucketsCalls++ + out := make(map[string]domain.StorageBucket, len(fake.buckets)) + for bucket, metadata := range fake.buckets { + out[bucket] = metadata + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) { + fake.listBucketScopesCalls++ + return append([]domain.StorageBucketScope(nil), fake.bucketScopes[bucket]...), nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]gintegrationsyfon.ProjectRecord, error) { + return append([]gintegrationsyfon.ProjectRecord(nil), fake.projectRecords...), nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectRecord, error) { + fake.listProjectAuditRecordsCalls++ + fake.listProjectAuditRecordsPathPrefix = pathPrefix + if fake.listProjectAuditRecordsDelay > 0 { + time.Sleep(fake.listProjectAuditRecordsDelay) + } + return append([]gintegrationsyfon.ProjectRecord(nil), fake.projectRecords...), nil +} + +func (fake *fakeStorageAnalyticsBackend) GetProjectMetricsSummary(ctx context.Context, authorizationHeader string, organization string, project string) (*gintegrationsyfon.ProjectMetricsSummary, error) { + fake.getProjectMetricsSummaryCalls++ + if fake.projectMetricsSummary != nil { + summary := *fake.projectMetricsSummary + return &summary, nil + } + latest := "" + for _, record := range fake.projectRecords { + if record.UpdatedAt == nil { + continue + } + formatted := record.UpdatedAt.UTC().Format(time.RFC3339Nano) + if formatted > latest { + latest = formatted + } + } + return &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: len(fake.projectRecords), + RecordLatestUpdatedTime: latest, + }, nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + fake.listProjectScopesCalls++ + if fake.projectScopes != nil { + return append([]domain.StorageBucketScope(nil), fake.projectScopes...), nil + } + out := make([]domain.StorageBucketScope, 0) + for _, scopes := range fake.bucketScopes { + out = append(out, scopes...) + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]gintegrationsyfon.ProjectRecord, error) { + fake.bulkGetProjectRecordsCalls++ + fake.bulkChecksums = append([]string(nil), checksums...) + if fake.bulkGetDelay > 0 { + time.Sleep(fake.bulkGetDelay) + } + if fake.bulkRecords != nil { + out := make(map[string][]gintegrationsyfon.ProjectRecord, len(fake.bulkRecords)) + for checksum, records := range fake.bulkRecords { + out[checksum] = append([]gintegrationsyfon.ProjectRecord(nil), records...) + } + return out, nil + } + allowed := make(map[string]struct{}, len(checksums)) + for _, checksum := range checksums { + allowed[strings.TrimSpace(checksum)] = struct{}{} + } + out := make(map[string][]gintegrationsyfon.ProjectRecord) + for _, record := range fake.projectRecords { + if _, ok := allowed[record.Checksum]; !ok { + continue + } + if record.Organization != organization || record.Project != project { + continue + } + out[record.Checksum] = append(out[record.Checksum], record) + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectFileUsage(ctx context.Context, authorizationHeader string, organization string, project string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) { + fake.listProjectFileUsageCalls++ + out := make(map[string]gintegrationsyfon.FileUsage, len(fake.usageByObject)) + for objectID, usage := range fake.usageByObject { + out[objectID] = usage + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectFileUsageByObjectIDs(ctx context.Context, authorizationHeader string, organization string, project string, objectIDs []string, inactiveDays int) (map[string]gintegrationsyfon.FileUsage, error) { + fake.listProjectFileUsageByObjectIDsCalls++ + fake.listProjectFileUsageObjectIDs = append(fake.listProjectFileUsageObjectIDs, objectIDs...) + if fake.listProjectFileUsageByObjectIDsErr != nil { + return nil, fake.listProjectFileUsageByObjectIDsErr + } + out := make(map[string]gintegrationsyfon.FileUsage, len(objectIDs)) + for _, objectID := range objectIDs { + if usage, ok := fake.usageByObject[objectID]; ok { + out[objectID] = usage + } + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) { + fake.listProjectBucketObjectsCalls++ + fake.listProjectBucketObjectsPathPrefix = pathPrefix + if fake.listProjectBucketObjectsErr != nil { + return nil, fake.listProjectBucketObjectsErr + } + return append([]gintegrationsyfon.ProjectBucketObject(nil), fake.bucketObjects...), nil +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]gintegrationsyfon.ProjectBucketObject, error) { + fake.listProjectBucketInventoryCalls++ + fake.listProjectBucketInventoryPathPrefix = pathPrefix + if fake.listProjectBucketInventoryDelay > 0 { + time.Sleep(fake.listProjectBucketInventoryDelay) + } + if fake.listProjectBucketObjectsErr != nil { + return nil, fake.listProjectBucketObjectsErr + } + return append([]gintegrationsyfon.ProjectBucketObject(nil), fake.bucketObjects...), nil +} + +func TestBuildStorageChainAuditCoalescesConcurrentForcedRefreshes(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + listProjectBucketInventoryDelay: 100 * time.Millisecond, + projectMetricsSummary: &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 1, + RecordLatestUpdatedTime: "2026-07-01T00:00:00Z", + }, + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/root/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket/root"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/root/data/a.txt", Bucket: "bucket", Key: "root/data/a.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + service.chainAuditResponseCache = newMemoryStorageChainAuditResponseCache() + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate, ForceAuditRefresh: true} + + var wg sync.WaitGroup + errs := make(chan error, 2) + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options) + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("build concurrent forced audit: %v", err) + } + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected concurrent forced refreshes to share one bucket inventory, got %d", backend.listProjectBucketInventoryCalls) + } +} + +func (fake *fakeStorageAnalyticsBackend) ListProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*gintegrationsyfon.ProjectBucketSummary, error) { + fake.listProjectBucketSummaryCalls++ + fake.listProjectBucketSummaryMode = mode + var totalBytes int64 + for _, item := range fake.bucketObjects { + totalBytes += item.SizeBytes + } + return &gintegrationsyfon.ProjectBucketSummary{ + Provider: "s3", + Bucket: "bucket", + Prefix: organization + "/" + project, + ObjectURL: "s3://bucket/" + organization + "/" + project, + Exists: len(fake.bucketObjects) > 0, + ObjectCount: len(fake.bucketObjects), + TotalBytes: totalBytes, + Mode: mode, + }, nil +} + +func (fake *fakeStorageAnalyticsBackend) BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) { + fake.probeCalls++ + fake.probeItems = append(fake.probeItems, items...) + out := make([]gintegrationsyfon.BulkStorageProbeResult, 0, len(items)) + for _, item := range items { + if result, ok := fake.probeResults[item.ID]; ok { + out = append(out, result) + continue + } + exists := true + out = append(out, gintegrationsyfon.BulkStorageProbeResult{ + ID: item.ID, + ObjectURL: item.ObjectURL, + Exists: exists, + Status: "present", + ValidationStatus: "matched", + SizeBytes: item.ExpectedSizeBytes, + MetaSHA256: item.ExpectedSHA256, + }) + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) BulkListStorageObjects(ctx context.Context, authorizationHeader string, items []gintegrationsyfon.BulkStorageProbeItem) ([]gintegrationsyfon.BulkStorageProbeResult, error) { + fake.listProbeCalls++ + fake.listProbeItems = append(fake.listProbeItems, items...) + out := make([]gintegrationsyfon.BulkStorageProbeResult, 0, len(items)) + for _, item := range items { + if result, ok := fake.listProbeResults[item.ID]; ok { + out = append(out, result) + continue + } + exists := true + out = append(out, gintegrationsyfon.BulkStorageProbeResult{ + ID: item.ID, + ObjectURL: item.ObjectURL, + Exists: exists, + Status: "present", + ValidationStatus: "matched", + SizeBytes: item.ExpectedSizeBytes, + SizeMatch: ptrBool(true), + NameMatch: ptrBool(true), + }) + } + return out, nil +} + +func (fake *fakeStorageAnalyticsBackend) BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error { + fake.deletedIDs = append(fake.deletedIDs, objectIDs...) + if deleteStorageData { + fake.deletedStorageIDs = append(fake.deletedStorageIDs, objectIDs...) + } + return nil +} + +func (fake *fakeStorageAnalyticsBackend) DeleteProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, objectURLs []string) ([]gintegrationsyfon.ProjectBucketDeleteResult, error) { + fake.deletedBucketObjects = append(fake.deletedBucketObjects, objectURLs...) + results := make([]gintegrationsyfon.ProjectBucketDeleteResult, 0, len(objectURLs)) + for _, objectURL := range objectURLs { + results = append(results, gintegrationsyfon.ProjectBucketDeleteResult{ + ObjectURL: objectURL, + Status: "deleted", + }) + } + return results, nil +} + +func (fake *fakeStorageAnalyticsBackend) BulkUpdateAccessMethods(ctx context.Context, authorizationHeader string, updates map[string][]gintegrationsyfon.ProjectAccessMethod) error { + if fake.updatedAccessMethods == nil { + fake.updatedAccessMethods = map[string][]gintegrationsyfon.ProjectAccessMethod{} + } + for objectID, methods := range updates { + fake.updatedAccessMethods[objectID] = append([]gintegrationsyfon.ProjectAccessMethod(nil), methods...) + } + return nil +} + +func (fake *fakeStorageAnalyticsBackend) RegisterProjectObjects(ctx context.Context, authorizationHeader string, candidates []gintegrationsyfon.ProjectObjectRegistration) ([]gintegrationsyfon.ProjectObjectRegistrationResult, error) { + fake.registeredObjects = append(fake.registeredObjects, candidates...) + results := make([]gintegrationsyfon.ProjectObjectRegistrationResult, len(candidates)) + for index, candidate := range candidates { + results[index] = gintegrationsyfon.ProjectObjectRegistrationResult{ObjectID: "created-" + candidate.Checksum[:8]} + fake.projectRecords = append(fake.projectRecords, gintegrationsyfon.ProjectRecord{ + ObjectID: results[index].ObjectID, + Name: candidate.Name, + Checksum: candidate.Checksum, + Size: candidate.Size, + Organization: "HTAN_INT", + Project: "BForePC", + AccessURLs: candidate.AccessURLs, + }) + } + return results, nil +} + +func TestBuildGitRepoInventoryAndStorageAnalytics(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/c.txt": lfsPointer("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", 300), + "data/e.txt": lfsPointer("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", 50), + "data/nested/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "plain.txt": "not lfs\n", + }) + index, err := buildRepoAnalyticsIndex(refName, repo, hash) + if err != nil { + t.Fatalf("build repo analytics index: %v", err) + } + inventory, err := filterRepoInventoryFiles(index, "data") + if err != nil { + t.Fatalf("filter repo inventory: %v", err) + } + if len(inventory) != 4 { + t.Fatalf("expected 4 lfs files under data, got %+v", inventory) + } + if inventory[0].RepoPath != "data/a.txt" || inventory[3].RepoPath != "data/nested/b.txt" { + t.Fatalf("unexpected inventory ordering: %+v", inventory) + } + + now := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC) + older := now.Add(-48 * time.Hour) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: inventory[0].Checksum, Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-d", Checksum: inventory[1].Checksum, Organization: "org", Project: "proj", Size: 300, UpdatedAt: &older}, + {ObjectID: "obj-b-old", Checksum: inventory[3].Checksum, Organization: "org", Project: "proj", Size: 200, UpdatedAt: &older, AccessURLs: []string{"s3://bucket/b-old"}}, + {ObjectID: "obj-b-new", Checksum: inventory[3].Checksum, Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/b-new"}}, + {ObjectID: "obj-orphan", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj", Size: 500, UpdatedAt: &older, AccessURLs: []string{"s3://bucket/orphan"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 5, LastDownloadTime: ptrTime(now)}, + "obj-b-old": {ObjectID: "obj-b-old", DownloadCount: 0}, + "obj-b-new": {ObjectID: "obj-b-new", DownloadCount: 10, LastDownloadTime: ptrTime(now)}, + "obj-orphan": {ObjectID: "obj-orphan", DownloadCount: 0}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{}, + } + service := NewStorageAnalyticsService(backend) + + summary, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build storage summary: %v", err) + } + if summary.FileCount != 4 || summary.RecordCount != 4 || summary.DirectChildCount != 4 || summary.DuplicatePathCount != 1 { + t.Fatalf("unexpected summary: %+v", summary) + } + if summary.TotalBytes != 650 || summary.DownloadCount != 15 { + t.Fatalf("unexpected summary bytes/downloads: %+v", summary) + } + if backend.listProjectFileUsageByObjectIDsCalls != 1 { + t.Fatalf("expected exact summary to use bulk file usage lookup, got %d", backend.listProjectFileUsageByObjectIDsCalls) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected exact summary to load shared project audit records once, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to join cached project records locally, got %d checksum lookup calls", backend.bulkGetProjectRecordsCalls) + } + if contains(backend.listProjectFileUsageObjectIDs, "obj-orphan") { + t.Fatalf("expected exact summary usage lookup to include only Git-matched records, got %+v", backend.listProjectFileUsageObjectIDs) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected exact summary to skip paginated project usage lookup, got %d", backend.listProjectFileUsageCalls) + } + + usageCallsBeforeChildren := backend.listProjectFileUsageCalls + children, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 10, "bytes", "desc", "") + if err != nil { + t.Fatalf("build storage children: %v", err) + } + if backend.listProjectFileUsageCalls != usageCallsBeforeChildren { + t.Fatalf("expected storage children to skip project-wide file usage, got calls before=%d after=%d", usageCallsBeforeChildren, backend.listProjectFileUsageCalls) + } + if len(children.Items) != 4 { + t.Fatalf("expected 4 child rows, got %+v", children.Items) + } + if children.HasMore || children.NextCursor != "" { + t.Fatalf("expected unpaginated children response, got %+v", children) + } + if children.Items[0].Path != "data/c.txt" || children.Items[1].Path != "data/nested" { + t.Fatalf("unexpected child ordering: %+v", children.Items) + } + + diff, err := service.BuildProjectDiffAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build project diff audit: %v", err) + } + if diff.Summary.TotalFindings != 3 { + t.Fatalf("expected 3 diff findings, got %+v", diff) + } + assertHasDiffFinding(t, diff.Findings, "duplicate_syfon_paths", "data/nested/b.txt") + missingFinding := assertHasDiffFinding(t, diff.Findings, "repo_missing_in_syfon", "data/e.txt") + if missingFinding.Checksum != "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" { + t.Fatalf("expected checksum on repo_missing_in_syfon, got %+v", missingFinding) + } + if missingFinding.Evidence == nil || missingFinding.Evidence.Checksum != missingFinding.Checksum || missingFinding.Evidence.BucketEvaluation != "not_checked" { + t.Fatalf("expected evidence on repo_missing_in_syfon, got %+v", missingFinding) + } + orphanFinding := assertHasDiffFinding(t, diff.Findings, "syfon_missing_in_repo", "s3://bucket/orphan") + if len(orphanFinding.SourcePaths) != 1 || orphanFinding.SourcePaths[0] != "s3://bucket/orphan" { + t.Fatalf("expected orphan source path, got %+v", orphanFinding) + } + if orphanFinding.Checksum != "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" { + t.Fatalf("expected checksum on syfon_missing_in_repo, got %+v", orphanFinding) + } + if orphanFinding.Evidence == nil || len(orphanFinding.Evidence.AccessURLs) != 1 || orphanFinding.Evidence.AccessURLs[0] != "s3://bucket/orphan" { + t.Fatalf("expected orphan access URL evidence, got %+v", orphanFinding) + } + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + if backend.probeCalls != 1 { + t.Fatalf("expected one bulk storage probe call, got %d", backend.probeCalls) + } + if cleanup.Summary.TotalFindings != 3 { + t.Fatalf("expected 3 cleanup findings, got %+v", cleanup) + } + assertHasCleanupFinding(t, cleanup.Findings, "stale_duplicate_record", "data/nested/b.txt") + assertHasCleanupFinding(t, cleanup.Findings, "broken_access_url_error", "data/c.txt") + assertHasCleanupFinding(t, cleanup.Findings, "repo_orphan_stale_record", "s3://bucket/orphan") + + applyResult, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + nil, + nil, + []GitStorageCleanupApplyFinding{ + {Kind: "stale_duplicate_record", NormalizedPath: "data/nested/b.txt", ObjectIDs: []string{"obj-b-old"}}, + {Kind: "repo_orphan_stale_record", NormalizedPath: "s3://bucket/orphan", ObjectIDs: []string{"obj-orphan"}, AccessURLs: []string{"s3://bucket/orphan"}}, + }, + true, + true, + false, + false, + true, + ) + if err != nil { + t.Fatalf("apply cleanup dry run: %v", err) + } + if !applyResult.DryRun { + t.Fatalf("expected dry run apply result, got %+v", applyResult) + } + if len(applyResult.DeletedRecordIDs) != 2 || !contains(applyResult.DeletedRecordIDs, "obj-b-old") || !contains(applyResult.DeletedRecordIDs, "obj-orphan") { + t.Fatalf("unexpected dry run delete ids: %+v", applyResult) + } + if len(backend.deletedIDs) != 0 { + t.Fatalf("dry run should not delete objects, got %+v", backend.deletedIDs) + } +} + +func TestBuildStorageChildrenEnrichesOnlySelectedPage(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/big/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 600), + "data/big/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 300), + "data/other.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 50), + "data/small.txt": lfsPointer("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", 100), + "outside/file.txt": lfsPointer("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", 1000), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj"}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj"}, + {ObjectID: "obj-c", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj"}, + {ObjectID: "obj-d", Checksum: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Organization: "org", Project: "proj"}, + {ObjectID: "obj-e", Checksum: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", Organization: "org", Project: "proj"}, + }, + } + service := NewStorageAnalyticsService(backend) + + children, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "") + if err != nil { + t.Fatalf("build storage children: %v", err) + } + if len(children.Items) != 1 || children.Items[0].Path != "data/big" { + t.Fatalf("expected only biggest child page, got %+v", children.Items) + } + if !children.HasMore || children.NextCursor == "" { + t.Fatalf("expected next cursor for truncated children page, got %+v", children) + } + expectedChecksums := []string{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + } + if strings.Join(backend.bulkChecksums, ",") != strings.Join(expectedChecksums, ",") { + t.Fatalf("expected page-scoped checksums %v, got %v", expectedChecksums, backend.bulkChecksums) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected storage children to skip project file usage, got %d calls", backend.listProjectFileUsageCalls) + } +} + +func TestBuildStorageChildrenCursorPagination(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "data/c.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 100), + }) + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + + first, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "") + if err != nil { + t.Fatalf("build first storage children page: %v", err) + } + if len(first.Items) != 2 || first.Items[0].Path != "data/a.txt" || first.Items[1].Path != "data/b.txt" { + t.Fatalf("unexpected first page: %+v", first.Items) + } + if !first.HasMore || first.NextCursor == "" { + t.Fatalf("expected next cursor on first page, got %+v", first) + } + + second, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", first.NextCursor) + if err != nil { + t.Fatalf("build second storage children page: %v", err) + } + if len(second.Items) != 1 || second.Items[0].Path != "data/c.txt" { + t.Fatalf("unexpected second page: %+v", second.Items) + } + if second.HasMore || second.NextCursor != "" { + t.Fatalf("expected final page without cursor, got %+v", second) + } + if second.Items[0].Path == first.Items[0].Path || second.Items[0].Path == first.Items[1].Path { + t.Fatalf("expected cursor pages not to duplicate rows: first=%+v second=%+v", first.Items, second.Items) + } +} + +func TestBuildStorageFolderCombinesSummaryAndChildren(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "data/c.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj"}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj"}, + {ObjectID: "obj-c", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj"}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 3}, + "obj-b": {ObjectID: "obj-b", DownloadCount: 2}, + "obj-c": {ObjectID: "obj-c", DownloadCount: 1}, + }, + } + service := NewStorageAnalyticsService(backend) + + summary, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build storage summary: %v", err) + } + children, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "") + if err != nil { + t.Fatalf("build storage children: %v", err) + } + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", "", StorageFolderSummaryModeExact, false, nil) + if err != nil { + t.Fatalf("build storage folder: %v", err) + } + expectedSummary := *summary + expectedSummary.Source = StorageFolderSummarySourceExactJoin + if folder.Summary != expectedSummary { + t.Fatalf("expected folder summary to match summary endpoint\nfolder=%+v\nsummary=%+v", folder.Summary, *summary) + } + if len(folder.Children.Items) != len(children.Items) || folder.Children.HasMore != children.HasMore || folder.Children.NextCursor != children.NextCursor { + t.Fatalf("expected folder children page shape to match children endpoint\nfolder=%+v\nchildren=%+v", folder.Children, *children) + } + for i := range children.Items { + if folder.Children.Items[i].Path != children.Items[i].Path || folder.Children.Items[i].Type != children.Items[i].Type { + t.Fatalf("expected folder child ordering to match children endpoint\nfolder=%+v\nchildren=%+v", folder.Children, *children) + } + } + + next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, "bytes", "desc", folder.Children.NextCursor, StorageFolderSummaryModeExact, false, nil) + if err != nil { + t.Fatalf("build storage folder next page: %v", err) + } + if len(next.Children.Items) != 1 || next.Children.Items[0].Path != "data/c.txt" { + t.Fatalf("unexpected next folder page: %+v", next.Children) + } + if next.Children.HasMore || next.Children.NextCursor != "" { + t.Fatalf("expected final folder page without cursor, got %+v", next.Children) + } +} + +func TestBuildStorageFolderDefaultModeUsesGitIndexOnly(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "other/c.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 50), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj"}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj"}, + {ObjectID: "obj-c", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj"}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 1}, + "obj-b": {ObjectID: "obj-b", DownloadCount: 1}, + "obj-c": {ObjectID: "obj-c", DownloadCount: 1}, + }, + } + service := NewStorageAnalyticsService(backend) + + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", false, nil) + if err != nil { + t.Fatalf("build storage folder: %v", err) + } + if folder.Summary.Source != StorageFolderSummarySourceGitIndex { + t.Fatalf("expected default folder summary source %q, got %+v", StorageFolderSummarySourceGitIndex, folder.Summary) + } + if folder.Summary.FileCount != 3 || folder.Summary.RecordCount != 0 || folder.Summary.DownloadCount != 0 || folder.Summary.DuplicatePathCount != 0 { + t.Fatalf("git-index folder summary should not fake exact Syfon totals, got %+v", folder.Summary) + } + if len(folder.Children.Items) != 1 || !folder.Children.HasMore { + t.Fatalf("unexpected folder response: %+v", folder) + } + first := folder.Children.Items[0] + if first.Path != "data" || first.Type != "directory" || first.FileCount != 2 || first.TotalBytes != 300 { + t.Fatalf("expected first page to use Git directory aggregate, got %+v", folder.Children.Items) + } + if first.RecordCount != 0 || first.DownloadCount != 0 || first.LastDownloadTime != "" { + t.Fatalf("git-index child should not include Syfon/download enrichment, got %+v", first) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected default folder mode to skip Syfon record lookup, got %d", backend.bulkGetProjectRecordsCalls) + } + if len(backend.bulkChecksums) != 0 { + t.Fatalf("expected default folder mode to avoid descendant checksum expansion, got %v", backend.bulkChecksums) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected default folder mode to skip project usage lookup, got %d calls", backend.listProjectFileUsageCalls) + } + if backend.listProjectBucketObjectsCalls != 0 || backend.listProjectBucketSummaryCalls != 0 || backend.probeCalls != 0 { + t.Fatalf("expected default folder mode to skip bucket APIs, got objects=%d summary=%d probes=%d", backend.listProjectBucketObjectsCalls, backend.listProjectBucketSummaryCalls, backend.probeCalls) + } +} + +func TestBuildStorageFolderDefaultModeUsesPreSortedServingIndex(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/alpha.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), + "data/bravo.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 100), + "data/charlie.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 200), + }) + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + + tests := map[string]struct { + sortBy string + sortOrder string + expected []string + }{ + "bytes desc": {sortBy: "bytes", sortOrder: "desc", expected: []string{"data/alpha.txt", "data/charlie.txt", "data/bravo.txt"}}, + "bytes asc": {sortBy: "bytes", sortOrder: "asc", expected: []string{"data/bravo.txt", "data/charlie.txt", "data/alpha.txt"}}, + "name asc": {sortBy: "name", sortOrder: "asc", expected: []string{"data/alpha.txt", "data/bravo.txt", "data/charlie.txt"}}, + "name desc": {sortBy: "name", sortOrder: "desc", expected: []string{"data/charlie.txt", "data/bravo.txt", "data/alpha.txt"}}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, test.sortBy, test.sortOrder, "", "", false, nil) + if err != nil { + t.Fatalf("build storage folder: %v", err) + } + if got := storageChildResponsePaths(folder.Children.Items); strings.Join(got, ",") != strings.Join(test.expected[:2], ",") { + t.Fatalf("unexpected first page order: got %v want %v", got, test.expected[:2]) + } + if !folder.Children.HasMore || folder.Children.NextCursor == "" { + t.Fatalf("expected first page cursor, got %+v", folder.Children) + } + next, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 2, test.sortBy, test.sortOrder, folder.Children.NextCursor, "", false, nil) + if err != nil { + t.Fatalf("build storage folder next page: %v", err) + } + if got := storageChildResponsePaths(next.Children.Items); strings.Join(got, ",") != test.expected[2] { + t.Fatalf("unexpected next page order: got %v want %v", got, test.expected[2:]) + } + if next.Children.HasMore || next.Children.NextCursor != "" { + t.Fatalf("expected final page without cursor, got %+v", next.Children) + } + }) + } +} + +func TestBuildStorageFolderDefaultModeUsesSidecarServingIndex(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + if err := PersistRepoAnalyticsIndex(context.Background(), mirrorPath, repo, refName, hash); err != nil { + t.Fatalf("persist repo analytics index: %v", err) + } + repoAnalyticsIndexCache.mu.Lock() + repoAnalyticsIndexCache.entries = map[string]*repoAnalyticsIndex{} + repoAnalyticsIndexCache.inflight = map[string]*inflightRepoAnalyticsIndex{} + repoAnalyticsIndexCache.mu.Unlock() + + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + folder, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", "", "", false, nil) + if err != nil { + t.Fatalf("build storage folder from sidecar: %v", err) + } + if got := storageChildResponsePaths(folder.Children.Items); strings.Join(got, ",") != "data/b.txt" { + t.Fatalf("expected sidecar serving index to preserve bytes desc order, got %v", got) + } + if !folder.Children.HasMore || folder.Children.NextCursor == "" { + t.Fatalf("expected sidecar serving index cursor, got %+v", folder.Children) + } +} + +func storageChildResponsePaths(items []GitStorageChildResponseItem) []string { + paths := make([]string, 0, len(items)) + for _, item := range items { + paths = append(paths, item.Path) + } + return paths +} + +func TestBuildStorageChildrenRejectsStaleCursor(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 300), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + wrongHash := plumbing.NewHash("cccccccccccccccccccccccccccccccccccccccc") + staleCursors := map[string]string{ + "commit": buildStorageChildrenCursor(wrongHash, "data", "bytes", "desc", 1), + "path": buildStorageChildrenCursor(hash, "other", "bytes", "desc", 1), + "sort_by": buildStorageChildrenCursor(hash, "data", "name", "desc", 1), + "sort_order": buildStorageChildrenCursor(hash, "data", "bytes", "asc", 1), + "invalid": "not-base64", + } + for name, cursor := range staleCursors { + t.Run(name, func(t *testing.T) { + _, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1, "bytes", "desc", cursor) + if err == nil { + t.Fatalf("expected stale cursor %q to fail", name) + } + }) + } +} + +func TestApplyStorageCleanupMatchesSelectedRepoOrphanByAnyAccessURL(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + + applyResult, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"s3://bucket/orphan-selected"}, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "repo_orphan_stale_record", + NormalizedPath: "s3://bucket/orphan-primary", + ObjectIDs: []string{"obj-orphan"}, + AccessURLs: []string{"s3://bucket/orphan-primary", "s3://bucket/orphan-selected"}, + }}, + true, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply cleanup: %v", err) + } + if !contains(applyResult.DeletedRecordIDs, "obj-orphan") { + t.Fatalf("expected selected orphan to be deleted, got %+v", applyResult) + } + if len(applyResult.SkippedPaths) != 0 || len(applyResult.ManualPaths) != 0 { + t.Fatalf("expected selected orphan not to be skipped or manual, got %+v", applyResult) + } + if !contains(backend.deletedIDs, "obj-orphan") { + t.Fatalf("expected syfon delete call for selected orphan, got %+v", backend.deletedIDs) + } +} + +func TestApplyStorageCleanupRejectsUnmatchedSelectedPaths(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + + _, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"s3://other-bucket/missing"}, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "repo_orphan_stale_record", + NormalizedPath: "s3://bucket/a", + ObjectIDs: []string{"obj-a"}, + AccessURLs: []string{"s3://bucket/a"}, + }}, + true, + false, + false, + false, + false, + ) + if err == nil { + t.Fatalf("expected unmatched selected path to fail") + } + if !strings.Contains(err.Error(), "selected cleanup paths did not match provided cleanup findings") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestApplyStorageCleanupRepairsBrokenBucketMappings(t *testing.T) { + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://legacy/a", "s3://bucket/a"}, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{{URL: "s3://legacy/a"}, {URL: "s3://bucket/a"}}, + }, + { + ObjectID: "obj-b", + Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + Organization: "org", + Project: "proj", + Size: 200, + UpdatedAt: &now, + AccessURLs: []string{"s3://legacy/b"}, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{{URL: "s3://legacy/b"}}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://legacy/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://legacy/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://legacy/a", + Status: "error", + ErrorKind: "credential_missing", + }, + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey("s3://legacy/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://legacy/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://legacy/b", + Status: "error", + ErrorKind: "credential_missing", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + applyResult, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"data/a.txt", "data/b.txt"}, + nil, + []GitStorageCleanupApplyFinding{ + { + Kind: "broken_bucket_mapping", + NormalizedPath: "data/a.txt", + ObjectIDs: []string{"obj-a"}, + Records: []GitStorageCleanupRecordAudit{{ + ObjectID: "obj-a", + AccessURLs: []string{"s3://legacy/a", "s3://bucket/a"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/a"}, {URL: "s3://bucket/a"}}, + AccessProbes: []GitStorageCleanupAccessProbe{ + {URL: "s3://legacy/a", Status: "error", ErrorKind: "credential_missing"}, + {URL: "s3://bucket/a", Status: "present"}, + }, + }}, + }, + { + Kind: "broken_bucket_mapping", + NormalizedPath: "data/b.txt", + ObjectIDs: []string{"obj-b"}, + Records: []GitStorageCleanupRecordAudit{{ + ObjectID: "obj-b", + AccessURLs: []string{"s3://legacy/b"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/b"}}, + AccessProbes: []GitStorageCleanupAccessProbe{{URL: "s3://legacy/b", Status: "error", ErrorKind: "credential_missing"}}, + }}, + }, + }, + false, + false, + false, + true, + false, + ) + if err != nil { + t.Fatalf("apply cleanup broken bucket mapping repair: %v", err) + } + if len(applyResult.UpdatedRecordIDs) != 1 || applyResult.UpdatedRecordIDs[0] != "obj-a" { + t.Fatalf("expected obj-a access methods to be updated, got %+v", applyResult.UpdatedRecordIDs) + } + if len(applyResult.DeletedRecordIDs) != 1 || applyResult.DeletedRecordIDs[0] != "obj-b" { + t.Fatalf("expected obj-b to be deleted, got %+v", applyResult.DeletedRecordIDs) + } + updatedMethods := backend.updatedAccessMethods["obj-a"] + if len(updatedMethods) != 1 || updatedMethods[0].URL != "s3://bucket/a" { + t.Fatalf("expected obj-a to retain only good access method, got %+v", updatedMethods) + } + if len(backend.deletedIDs) != 1 || backend.deletedIDs[0] != "obj-b" { + t.Fatalf("expected obj-b to be deleted, got %+v", backend.deletedIDs) + } +} + +func TestApplyStorageCleanupDeletesBucketOnlyObjects(t *testing.T) { + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/a"}, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{{URL: "s3://bucket/a"}}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", SizeBytes: 100}, + {ObjectURL: "s3://bucket/orphan", Bucket: "bucket", Key: "orphan", SizeBytes: 25}, + }, + } + service := NewStorageAnalyticsService(backend) + + applyResult, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"s3://bucket/orphan"}, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "bucket_only_object", + NormalizedPath: "s3://bucket/orphan", + BucketObjectURL: "s3://bucket/orphan", + BucketObjectURLs: []string{"s3://bucket/orphan"}, + }}, + false, + false, + true, + false, + false, + ) + if err != nil { + t.Fatalf("apply cleanup bucket only delete: %v", err) + } + if len(applyResult.DeletedBucketObjectURLs) != 1 || applyResult.DeletedBucketObjectURLs[0] != "s3://bucket/orphan" { + t.Fatalf("expected orphan bucket object to be deleted, got %+v", applyResult.DeletedBucketObjectURLs) + } + if len(backend.deletedBucketObjects) != 1 || backend.deletedBucketObjects[0] != "s3://bucket/orphan" { + t.Fatalf("expected backend bucket delete call, got %+v", backend.deletedBucketObjects) + } +} + +func TestBuildStorageCleanupAuditReportsStorageProbeFailures(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/b"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Status: "not_found", + Exists: false, + ValidationStatus: "unverifiable", + }, + storageProbeRequestKey("s3://bucket/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://bucket/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://bucket/b", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + assertHasCleanupFinding(t, cleanup.Findings, "storage_object_missing", "data/a.txt") + assertHasCleanupFinding(t, cleanup.Findings, "storage_validation_mismatch", "data/b.txt") + var mismatchFinding GitStorageCleanupFinding + for _, finding := range cleanup.Findings { + if finding.Kind == "storage_validation_mismatch" && finding.NormalizedPath == "data/b.txt" { + mismatchFinding = finding + break + } + } + if len(mismatchFinding.Records) != 1 || len(mismatchFinding.Records[0].AccessProbes) != 1 { + t.Fatalf("expected probe details on mismatch finding, got %+v", mismatchFinding) + } + if mismatchFinding.Records[0].AccessProbes[0].ValidationStatus != "mismatched" { + t.Fatalf("expected mismatched validation status, got %+v", mismatchFinding.Records[0].AccessProbes[0]) + } + if mismatchFinding.Checksum != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" { + t.Fatalf("expected checksum on mismatch finding, got %+v", mismatchFinding) + } + if mismatchFinding.Evidence == nil || + len(mismatchFinding.Evidence.AccessURLs) != 1 || + mismatchFinding.Evidence.AccessURLs[0] != "s3://bucket/b" || + len(mismatchFinding.Evidence.ProbeStatuses) != 1 || + mismatchFinding.Evidence.ProbeStatuses[0] != "present" || + mismatchFinding.Evidence.BucketEvaluation != "probed" { + t.Fatalf("expected storage evidence on mismatch finding, got %+v", mismatchFinding) + } + if mismatchFinding.Actionability != "manual_choice" || + !contains(mismatchFinding.AvailableActions, "delete_syfon_record") || + !contains(mismatchFinding.AvailableActions, "delete_bucket_object") || + !contains(mismatchFinding.AvailableActions, "delete_both") || + mismatchFinding.DefaultAction != "" || + !mismatchFinding.SupportsDryRun { + t.Fatalf("expected mismatch action metadata, got %+v", mismatchFinding) + } +} + +func TestApplyStorageCleanupSupportsMetadataMismatchActions(t *testing.T) { + now := time.Date(2026, 7, 2, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/b"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://bucket/b", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://bucket/b", + Bucket: "bucket", + Key: "b", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + }, + } + service := NewStorageAnalyticsService(backend) + applyFindings := []GitStorageCleanupApplyFinding{{ + Kind: "storage_validation_mismatch", + NormalizedPath: "data/b.txt", + ObjectIDs: []string{"obj-b"}, + BucketObjectURL: "s3://bucket/b", + BucketObjectURLs: []string{"s3://bucket/b"}, + AccessURLs: []string{"s3://bucket/b"}, + }} + + deleteRecordOnly, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"data/b.txt"}, + []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/b.txt", Action: "delete_syfon_record"}}, + applyFindings, + false, + false, + false, + false, + true, + ) + if err != nil { + t.Fatalf("apply cleanup delete syfon record dry run: %v", err) + } + if len(deleteRecordOnly.DeletedRecordIDs) != 1 || deleteRecordOnly.DeletedRecordIDs[0] != "obj-b" { + t.Fatalf("expected mismatch delete-record action to select Syfon record, got %+v", deleteRecordOnly) + } + if len(deleteRecordOnly.DeletedBucketObjectURLs) != 0 { + t.Fatalf("expected record-only action to avoid bucket delete, got %+v", deleteRecordOnly) + } + + deleteBucketOnly, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"data/b.txt"}, + []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/b.txt", Action: "delete_bucket_object"}}, + applyFindings, + false, + false, + false, + false, + true, + ) + if err != nil { + t.Fatalf("apply cleanup delete bucket object dry run: %v", err) + } + if len(deleteBucketOnly.DeletedRecordIDs) != 0 { + t.Fatalf("expected bucket-only action to avoid Syfon delete, got %+v", deleteBucketOnly) + } + if len(deleteBucketOnly.DeletedBucketObjectURLs) != 1 || deleteBucketOnly.DeletedBucketObjectURLs[0] != "s3://bucket/b" { + t.Fatalf("expected mismatch delete-bucket action to select bucket object, got %+v", deleteBucketOnly) + } + + deleteBoth, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"data/b.txt"}, + []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/b.txt", Action: "delete_both"}}, + applyFindings, + false, + false, + false, + false, + true, + ) + if err != nil { + t.Fatalf("apply cleanup delete both dry run: %v", err) + } + if len(deleteBoth.DeletedRecordIDs) != 1 || deleteBoth.DeletedRecordIDs[0] != "obj-b" { + t.Fatalf("expected dual delete action to include Syfon record, got %+v", deleteBoth) + } + if len(deleteBoth.DeletedBucketObjectURLs) != 1 || deleteBoth.DeletedBucketObjectURLs[0] != "s3://bucket/b" { + t.Fatalf("expected dual delete action to include bucket object, got %+v", deleteBoth) + } +} + +func TestStorageRepairPolicyCoversFindingKinds(t *testing.T) { + tests := []struct { + kind string + defaultAction string + actions []string + actionability string + }{ + {"bucket_only_object", storageActionDeleteBucketObject, []string{storageActionDeleteBucketObject}, storageActionabilityAutoRepair}, + {"bucket_syfon_no_git", storageActionDeleteBoth, []string{storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject}, storageActionabilityManualChoice}, + {"repo_orphan_live_object", storageActionDeleteBoth, []string{storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject}, storageActionabilityAutoRepair}, + {"repo_orphan_stale_record", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"stale_duplicate_record", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"live_duplicate_conflict", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, + {"broken_access_url_error", storageActionRemoveBrokenAccessURLs, []string{storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"broken_bucket_mapping", storageActionRemoveBrokenAccessURLs, []string{storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"syfon_broken_bucket_mapping", storageActionRemoveBrokenAccessURLs, []string{storageActionRemoveBrokenAccessURLs, storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"storage_validation_mismatch", "", []string{storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionDeleteBoth}, storageActionabilityManualChoice}, + {"git_syfon_metadata_mismatch", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, + {"storage_object_missing", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"syfon_git_no_bucket", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"syfon_missing_bucket_object", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityAutoRepair}, + {"git_only_no_syfon", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, + {"storage_probe_error", storageActionInspectEvidence, []string{storageActionInspectEvidence}, storageActionabilityInspectOnly}, + {"probe_error", storageActionDeleteSyfonRecord, []string{storageActionDeleteSyfonRecord}, storageActionabilityManualChoice}, + } + for _, test := range tests { + t.Run(test.kind, func(t *testing.T) { + policy := storageRepairPolicyForKind(test.kind) + if policy.defaultAction != test.defaultAction { + t.Fatalf("expected default %q, got %q", test.defaultAction, policy.defaultAction) + } + if policy.actionability != test.actionability { + t.Fatalf("expected actionability %q, got %q", test.actionability, policy.actionability) + } + for _, action := range test.actions { + if !contains(policy.actions, action) { + t.Fatalf("expected action %q in %+v", action, policy.actions) + } + } + }) + } +} + +func TestFilterStorageChainSummaryPreservesCleanJoinCount(t *testing.T) { + summary := GitStorageChainAuditSummary{ + CountsByKind: map[string]int{ + "bucket_syfon_git_complete": 7, + "git_syfon_metadata_mismatch": 3, + "syfon_broken_bucket_mapping": 2, + }, + BucketObjectCount: 10, + SyfonRecordCount: 10, + GitTrackedFileCount: 10, + } + findings := []GitStorageChainFinding{ + {Kind: "git_syfon_metadata_mismatch", NormalizedPath: "data/a.txt"}, + } + + filtered := filterStorageChainSummary(summary, findings) + + if got := filtered.CountsByKind["bucket_syfon_git_complete"]; got != 7 { + t.Fatalf("expected clean-chain count to survive filtering, got %d in %+v", got, filtered) + } + if got := filtered.CountsByKind["git_syfon_metadata_mismatch"]; got != 1 { + t.Fatalf("expected filtered mismatch count from findings, got %d in %+v", got, filtered) + } + if got := filtered.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected filtered broken-mapping count to reset, got %d in %+v", got, filtered) + } +} + +func TestApplyStorageCleanupRepairMatrix(t *testing.T) { + tests := []struct { + name string + finding GitStorageCleanupApplyFinding + actions []GitStorageCleanupApplyAction + wantDeletedIDs []string + wantStorageIDs []string + wantBucketURLs []string + wantUpdatedRecordID string + wantManualPath string + }{ + {name: "bucket only", finding: applyFinding("bucket_only_object", "s3://bucket/only", nil, []string{"s3://bucket/only"}), wantBucketURLs: []string{"s3://bucket/only"}}, + {name: "bucket syfon no git", finding: applyFinding("bucket_syfon_no_git", "s3://bucket/no-git", []string{"obj-a"}, []string{"s3://bucket/no-git"}), wantDeletedIDs: []string{"obj-a"}, wantBucketURLs: []string{"s3://bucket/no-git"}}, + {name: "repo orphan live", finding: applyFinding("repo_orphan_live_object", "s3://bucket/live", []string{"obj-live"}, nil), wantDeletedIDs: []string{"obj-live"}, wantStorageIDs: []string{"obj-live"}}, + {name: "repo orphan stale", finding: applyFinding("repo_orphan_stale_record", "s3://bucket/stale", []string{"obj-stale"}, nil), wantDeletedIDs: []string{"obj-stale"}}, + {name: "stale duplicate", finding: applyFinding("stale_duplicate_record", "data/a.txt", []string{"obj-old"}, nil), wantDeletedIDs: []string{"obj-old"}}, + {name: "storage missing", finding: applyFinding("storage_object_missing", "data/missing.txt", []string{"obj-missing"}, []string{"s3://bucket/missing"}), wantDeletedIDs: []string{"obj-missing"}}, + {name: "syfon git no bucket", finding: applyFinding("syfon_git_no_bucket", "data/missing.txt", []string{"obj-missing"}, []string{"s3://bucket/missing"}), wantDeletedIDs: []string{"obj-missing"}}, + {name: "syfon missing bucket object", finding: applyFinding("syfon_missing_bucket_object", "s3://bucket/missing", []string{"obj-missing"}, []string{"s3://bucket/missing"}), wantDeletedIDs: []string{"obj-missing"}}, + {name: "git syfon mismatch requires manual review", finding: applyFinding("git_syfon_metadata_mismatch", "data/mismatch.txt", []string{"obj-mm"}, []string{"s3://bucket/mismatch"}), wantManualPath: "data/mismatch.txt"}, + {name: "storage mismatch delete both", finding: applyFinding("storage_validation_mismatch", "data/mismatch.txt", []string{"obj-mm"}, []string{"s3://bucket/mismatch"}), actions: []GitStorageCleanupApplyAction{{Kind: "storage_validation_mismatch", NormalizedPath: "data/mismatch.txt", Action: storageActionDeleteBoth}}, wantDeletedIDs: []string{"obj-mm"}, wantBucketURLs: []string{"s3://bucket/mismatch"}}, + {name: "live duplicate conflict", finding: applyFinding("live_duplicate_conflict", "data/dup.txt", []string{"obj-a", "obj-b"}, nil), wantManualPath: "data/dup.txt"}, + {name: "git only no syfon", finding: applyFinding("git_only_no_syfon", "data/git-only.txt", nil, nil), wantManualPath: "data/git-only.txt"}, + {name: "probe error", finding: applyFinding("probe_error", "data/probe.txt", []string{"obj-probe"}, nil), wantManualPath: "data/probe.txt"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{}} + action := test.finding.DefaultAction + if len(test.actions) > 0 { + action = test.actions[0].Action + } + if cleanupVerificationExpectation(test.finding.Kind, action) == cleanupExpectationMissing { + backend.probeResults["cleanup-revalidate-0"] = gintegrationsyfon.BulkStorageProbeResult{ + ID: "cleanup-revalidate-0", + Status: "not_found", + ErrorKind: "object_not_found", + } + } + service := NewStorageAnalyticsService(backend) + result, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", nil, test.actions, []GitStorageCleanupApplyFinding{test.finding}, false, false, false, false, false) + if err != nil { + t.Fatalf("apply cleanup: %v", err) + } + assertStringSet(t, "deleted IDs", result.DeletedRecordIDs, test.wantDeletedIDs) + assertStringSet(t, "storage IDs", backend.deletedStorageIDs, test.wantStorageIDs) + assertStringSet(t, "bucket URLs", result.DeletedBucketObjectURLs, test.wantBucketURLs) + if test.wantManualPath != "" && !contains(result.ManualPaths, test.wantManualPath) { + t.Fatalf("expected manual path %q, got %+v", test.wantManualPath, result.ManualPaths) + } + if test.wantUpdatedRecordID != "" && !contains(result.UpdatedRecordIDs, test.wantUpdatedRecordID) { + t.Fatalf("expected updated record %q, got %+v", test.wantUpdatedRecordID, result.UpdatedRecordIDs) + } + }) + } +} + +func TestApplyStorageCleanupBucketOnlyDeletionEvictsChainInputCache(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + } + service := NewStorageAnalyticsService(backend) + cacheKey := service.projectChainInputCacheKey("org", "proj") + service.chainInputCache[cacheKey] = cachedChainInputState{ + expiresAt: time.Now().Add(time.Minute), + projectScopes: backend.projectScopes, + } + service.chainInputCache[cacheKey+"::bucket-validation-inventory::"] = cachedChainInputState{ + expiresAt: time.Now().Add(time.Minute), + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: "s3://bucket/only", + Bucket: "bucket", + Key: "only", + }}, + bucketObjectsByURL: map[string]gintegrationsyfon.ProjectBucketObject{ + "s3://bucket/only": {ObjectURL: "s3://bucket/only", Bucket: "bucket", Key: "only"}, + }, + } + service.chainInputCache["org/other::bucket-validation-inventory::"] = cachedChainInputState{ + expiresAt: time.Now().Add(time.Minute), + } + + _, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + nil, + nil, + []GitStorageCleanupApplyFinding{applyFinding("bucket_only_object", "s3://bucket/only", nil, []string{"s3://bucket/only"})}, + false, + false, + true, + false, + false, + ) + if err != nil { + t.Fatalf("apply bucket-only cleanup: %v", err) + } + if _, ok := service.chainInputCache[cacheKey]; ok { + t.Fatalf("expected project chain input cache to be evicted, got %+v", service.chainInputCache) + } + if _, ok := service.chainInputCache[cacheKey+"::bucket-validation-inventory::"]; ok { + t.Fatalf("expected project bucket inventory cache to be evicted, got %+v", service.chainInputCache) + } + if _, ok := service.chainInputCache["org/other::bucket-validation-inventory::"]; !ok { + t.Fatalf("expected another project's cache entry to remain, got %+v", service.chainInputCache) + } +} + +func TestApplyStorageCleanupPrunesBrokenAccessMethods(t *testing.T) { + tests := []struct { + name string + record GitStorageCleanupRecordAudit + wantDeleted []string + wantRemaining []string + wantErrSnippet string + }{ + { + name: "keeps good access method", + record: GitStorageCleanupRecordAudit{ + ObjectID: "obj-a", + AccessURLs: []string{"s3://legacy/a", "s3://bucket/a"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/a"}, {URL: "s3://bucket/a"}}, + AccessProbes: []GitStorageCleanupAccessProbe{ + {URL: "s3://legacy/a", Status: "error", ErrorKind: "credential_missing"}, + {URL: "s3://bucket/a", Status: "present"}, + }, + }, + wantRemaining: []string{"s3://bucket/a"}, + }, + { + name: "deletes record when only access method is broken", + record: GitStorageCleanupRecordAudit{ + ObjectID: "obj-b", + AccessURLs: []string{"s3://legacy/b"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/b"}}, + AccessProbes: []GitStorageCleanupAccessProbe{{URL: "s3://legacy/b", Status: "error", ErrorKind: "credential_missing"}}, + }, + wantDeleted: []string{"obj-b"}, + }, + { + name: "missing evidence fails", + record: GitStorageCleanupRecordAudit{ + ObjectID: "obj-c", + AccessURLs: []string{"s3://legacy/c"}, + AccessMethods: []GitStorageCleanupAccessMethod{{URL: "s3://legacy/c"}}, + }, + wantErrSnippet: "missing broken access URL evidence", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + _, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + nil, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "broken_access_url_error", + NormalizedPath: "data/a.txt", + ObjectIDs: []string{test.record.ObjectID}, + Records: []GitStorageCleanupRecordAudit{test.record}, + }}, + false, + false, + false, + false, + false, + ) + if test.wantErrSnippet != "" { + if err == nil || !strings.Contains(err.Error(), test.wantErrSnippet) { + t.Fatalf("expected error containing %q, got %v", test.wantErrSnippet, err) + } + return + } + if err != nil { + t.Fatalf("apply cleanup: %v", err) + } + assertStringSet(t, "deleted IDs", backend.deletedIDs, test.wantDeleted) + if len(test.wantRemaining) > 0 { + methods := backend.updatedAccessMethods[test.record.ObjectID] + got := make([]string, 0, len(methods)) + for _, method := range methods { + got = append(got, method.URL) + } + assertStringSet(t, "remaining access methods", got, test.wantRemaining) + } + }) + } +} + +func TestApplyStorageCleanupValidation(t *testing.T) { + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + tests := []struct { + name string + finding GitStorageCleanupApplyFinding + actions []GitStorageCleanupApplyAction + wantErr string + }{ + {name: "unknown kind", finding: applyFinding("unknown_kind", "data/a.txt", []string{"obj-a"}, nil), wantErr: "unsupported cleanup finding kind"}, + {name: "unsupported action", finding: applyFinding("bucket_only_object", "s3://bucket/a", nil, []string{"s3://bucket/a"}), actions: []GitStorageCleanupApplyAction{{Kind: "bucket_only_object", NormalizedPath: "s3://bucket/a", Action: storageActionDeleteSyfonRecord}}, wantErr: "not supported"}, + {name: "missing object ids", finding: applyFinding("repo_orphan_stale_record", "s3://bucket/a", nil, nil), wantErr: "missing object_ids"}, + {name: "metadata mismatch disallows deletion", finding: applyFinding("git_syfon_metadata_mismatch", "data/a.txt", []string{"obj-a"}, nil), actions: []GitStorageCleanupApplyAction{{Kind: "git_syfon_metadata_mismatch", NormalizedPath: "data/a.txt", Action: storageActionDeleteBucketObject}}, wantErr: "not supported"}, + {name: "missing revalidation urls", finding: applyFinding("syfon_git_no_bucket", "data/a.txt", []string{"obj-a"}, nil), wantErr: "required for exact revalidation"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := service.ApplyStorageCleanup(context.Background(), "Bearer token", "org", "proj", nil, test.actions, []GitStorageCleanupApplyFinding{test.finding}, false, false, false, false, false) + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("expected error containing %q, got %v", test.wantErr, err) + } + }) + } +} + +func TestApplyStorageCleanupUsesServerPolicyWhenAuditActionsAreStale(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{}} + service := NewStorageAnalyticsService(backend) + finding := applyFinding("bucket_syfon_no_git", "s3://bucket/no-git", []string{"obj-no-git"}, []string{"s3://bucket/no-git"}) + // An audit may have been rendered before Gecko advertised this manual + // repair. The server policy, rather than this stale client snapshot, must + // decide whether the requested action is allowed. + finding.AvailableActions = []string{storageActionInspectEvidence} + response, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{finding.NormalizedPath}, + []GitStorageCleanupApplyAction{{ + Kind: finding.Kind, + NormalizedPath: finding.NormalizedPath, + Action: storageActionDeleteBoth, + }}, + []GitStorageCleanupApplyFinding{finding}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply stale bucket + Syfon audit finding: %v", err) + } + assertStringSet(t, "deleted IDs", response.DeletedRecordIDs, []string{"obj-no-git"}) + assertStringSet(t, "deleted bucket URLs", response.DeletedBucketObjectURLs, []string{"s3://bucket/no-git"}) +} + +func TestBuildStorageCleanupAuditAcceptsRecordWhenAnyAccessProbeIsLive(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + missingKey := storageProbeRequestKey("s3://bucket/legacy-a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + presentKey := storageProbeRequestKey("s3://bucket/current-a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/legacy-a", "s3://bucket/current-a"}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + missingKey: { + ID: missingKey, + ObjectURL: "s3://bucket/legacy-a", + Status: "not_found", + Exists: false, + ValidationStatus: "unverifiable", + }, + presentKey: { + ID: presentKey, + ObjectURL: "s3://bucket/current-a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + for _, finding := range cleanup.Findings { + if finding.NormalizedPath == "data/a.txt" && finding.Kind == "storage_object_missing" { + t.Fatalf("a live canonical locator must suppress a missing-record finding: %+v", finding) + } + } +} + +func TestBuildStorageCleanupAuditReportsBrokenBucketMappingSeparately(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + probeKey := storageProbeRequestKey("s3://bforepc-prod/path/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc-prod/path/a.txt"}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + probeKey: { + ID: probeKey, + ObjectURL: "s3://bforepc-prod/path/a.txt", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "bforepc-prod"`, + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + assertHasCleanupFinding(t, cleanup.Findings, "broken_bucket_mapping", "data/a.txt") + for _, finding := range cleanup.Findings { + if finding.Kind == "broken_bucket_mapping" && finding.NormalizedPath == "data/a.txt" { + if len(finding.Records) != 1 || finding.Records[0].Error != "Syfon access URL did not resolve through a configured bucket mapping" { + t.Fatalf("expected broken bucket mapping detail, got %+v", finding) + } + if finding.Evidence == nil || + len(finding.Evidence.AccessURLs) != 1 || + finding.Evidence.AccessURLs[0] != "s3://bforepc-prod/path/a.txt" || + len(finding.Evidence.ErrorKinds) != 1 || + finding.Evidence.ErrorKinds[0] != "credential_missing" || + finding.Evidence.BucketEvaluation != "probed" { + t.Fatalf("expected broken bucket mapping evidence, got %+v", finding) + } + if finding.Actionability != "auto_repair" || + !contains(finding.AvailableActions, "remove_broken_access_urls") || + finding.DefaultAction != "remove_broken_access_urls" || + !finding.SupportsDryRun { + t.Fatalf("expected broken mapping action metadata, got %+v", finding) + } + return + } + } + t.Fatalf("missing broken bucket mapping detail in %+v", cleanup.Findings) +} + +func TestBuildStorageCleanupAuditStartsFromBucketInventory(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-orphan", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj", Size: 50, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/orphan"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + {ObjectURL: "s3://bucket/orphan", Bucket: "bucket", Key: "orphan", Path: "orphan", SizeBytes: 50}, + {ObjectURL: "s3://bucket/loose", Bucket: "bucket", Key: "loose", Path: "loose", SizeBytes: 25}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey("s3://bucket/orphan", 50, "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"): { + ID: storageProbeRequestKey("s3://bucket/orphan", 50, "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"), + ObjectURL: "s3://bucket/orphan", + Bucket: "bucket", + Key: "orphan", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + assertHasCleanupFinding(t, cleanup.Findings, "repo_orphan_live_object", "s3://bucket/orphan") + assertHasCleanupFinding(t, cleanup.Findings, "bucket_only_object", "s3://bucket/loose") +} + +func TestBuildStorageChainAuditUsesBucketFirstFindingKinds(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/missing.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + "data/git-only.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 300), + "data/bad-map.txt": lfsPointer("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", 400), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-missing", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/missing"}}, + {ObjectID: "obj-no-git", Checksum: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", Organization: "org", Project: "proj", Size: 150, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/no-git"}}, + {ObjectID: "obj-bad-map", Checksum: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Organization: "org", Project: "proj", Size: 400, UpdatedAt: &now, AccessURLs: []string{"s3://legacy-bucket/bad-map"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + {ObjectURL: "s3://bucket/no-git", Bucket: "bucket", Key: "no-git", Path: "no-git", SizeBytes: 150}, + {ObjectURL: "s3://bucket/loose", Bucket: "bucket", Key: "loose", Path: "loose", SizeBytes: 25}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey("s3://bucket/missing", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://bucket/missing", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://bucket/missing", + Bucket: "bucket", + Key: "missing", + Status: "not_found", + Exists: false, + ValidationStatus: "unverifiable", + }, + storageProbeRequestKey("s3://bucket/no-git", 150, "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"): { + ID: storageProbeRequestKey("s3://bucket/no-git", 150, "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"), + ObjectURL: "s3://bucket/no-git", + Bucket: "bucket", + Key: "no-git", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey("s3://legacy-bucket/bad-map", 400, "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"): { + ID: storageProbeRequestKey("s3://legacy-bucket/bad-map", 400, "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"), + ObjectURL: "s3://legacy-bucket/bad-map", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "legacy-bucket"`, + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + findings := loadAllChainFindings(t, service, "org", "proj", chain) + assertHasChainFinding(t, findings, "bucket_only_object", "s3://bucket/loose") + assertHasChainFinding(t, findings, "bucket_syfon_no_git", "s3://bucket/no-git") + assertHasChainFinding(t, findings, "syfon_git_no_bucket", "data/missing.txt") + assertHasChainFinding(t, findings, "syfon_broken_bucket_mapping", "data/bad-map.txt") + assertHasChainFinding(t, findings, "git_only_no_syfon", "data/git-only.txt") + if chain.Summary.BucketObjectCount != 3 || chain.Summary.SyfonRecordCount != 4 || chain.Summary.GitTrackedFileCount != 4 { + t.Fatalf("unexpected chain summary totals: %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditUsesProjectAuditSourcesAndTargetsProbes(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/missing.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/present"}}, + {ObjectID: "obj-missing", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/missing"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present", Bucket: "bucket", Key: "present", Path: "present", SizeBytes: 100, MetaSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/missing", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://bucket/missing", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://bucket/missing", + Bucket: "bucket", + Key: "missing", + Status: "not_found", + Exists: false, + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + findings := loadAllChainFindings(t, service, "org", "proj", chain) + assertHasChainFinding(t, findings, "syfon_git_no_bucket", "data/missing.txt") + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected one project audit record call, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.listProjectAuditRecordsPathPrefix != "" { + t.Fatalf("expected audit to load project-scoped records for cache reuse, got %q", backend.listProjectAuditRecordsPathPrefix) + } + if backend.listProjectScopesCalls != 1 { + t.Fatalf("expected one project scope call, got %d", backend.listProjectScopesCalls) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected no project file usage calls, got %d", backend.listProjectFileUsageCalls) + } + if backend.listBucketsCalls != 0 || backend.listBucketScopesCalls != 0 { + t.Fatalf("expected no bucket-wide scope discovery, got listBuckets=%d listBucketScopes=%d", backend.listBucketsCalls, backend.listBucketScopesCalls) + } + if backend.probeCalls != 1 || len(backend.probeItems) != 1 || backend.probeItems[0].ObjectURL != "s3://bucket/missing" { + t.Fatalf("expected one targeted probe for missing object, got calls=%d items=%+v", backend.probeCalls, backend.probeItems) + } +} + +func TestBuildStorageChainAuditInventoryOnlySkipsTargetedProbes(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/present"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present", Bucket: "bucket", Key: "present", Path: "present", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly}, + ) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if backend.probeCalls != 0 || len(backend.probeItems) != 0 { + t.Fatalf("expected inventory-only audit to skip targeted probes, got calls=%d items=%+v", backend.probeCalls, backend.probeItems) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected inventory-derived complete chain, got summary %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditValidateModeUsesListValidation(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/mismatch.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Name: "present.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/present.txt"}}, + {ObjectID: "obj-mismatch", Name: "expected.txt", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, AccessURLs: []string{"s3://bucket/mismatch.txt"}}, + {ObjectID: "obj-syfon-only", Name: "syfon-only.txt", Checksum: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Organization: "org", Project: "proj", Size: 300, AccessURLs: []string{"s3://bucket/syfon-only.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present.txt", Bucket: "bucket", Key: "present.txt", Path: "present.txt", SizeBytes: 100}, + {ObjectURL: "s3://bucket/mismatch.txt", Bucket: "bucket", Key: "mismatch.txt", Path: "mismatch.txt", SizeBytes: 200}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build validate chain audit: %v", err) + } + if backend.listProjectBucketObjectsCalls != 0 { + t.Fatalf("expected validate mode to skip browse bucket inventory endpoint, got %d calls", backend.listProjectBucketObjectsCalls) + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected validate mode to load project inventory once, got %d calls", backend.listProjectBucketInventoryCalls) + } + if backend.listProjectBucketSummaryCalls != 0 { + t.Fatalf("expected validate mode to skip bucket summary preflight, got %d calls", backend.listProjectBucketSummaryCalls) + } + if backend.probeCalls != 0 { + t.Fatalf("expected validate mode to skip HEAD probes, got %d calls", backend.probeCalls) + } + if backend.listProbeCalls != 0 { + t.Fatalf("expected complete Syfon inventory to skip exact LIST fallback, got calls=%d items=%+v", backend.listProbeCalls, backend.listProbeItems) + } + if chain.Summary.BucketPathExists != nil || chain.Summary.BucketSummaryMode != "" { + t.Fatalf("expected validate mode not to claim bucket prefix summary, got %+v", chain.Summary) + } + if chain.Summary.ValidationMode != StorageChainValidationModeList { + t.Fatalf("expected LIST validation mode, got %+v", chain.Summary) + } + if chain.Summary.BucketObjectCount != 2 || chain.Summary.CountsByKind["bucket_only_object"] != 0 { + t.Fatalf("expected validate mode to preserve the complete inventory, got summary %+v", chain.Summary) + } + if chain.Summary.CountsByKind["bucket_syfon_git_complete"] != 2 { + t.Fatalf("expected validate mode to count clean Syfon/Git/bucket join, got summary %+v", chain.Summary) + } + if chain.Summary.CountsByKind["probe_error"] != 1 || chain.Summary.CountsByKind["bucket_syfon_no_git"] != 0 { + t.Fatalf("expected complete-inventory miss to remain a probe error, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected basename mismatch not to be a broken bucket mapping, got summary %+v", chain.Summary) + } + assertNoChainFinding(t, chain.Findings, "bucket_only_object") + assertNoChainFinding(t, chain.Findings, "syfon_broken_bucket_mapping") + finding := assertHasChainFinding(t, chain.Findings, "probe_error", "s3://bucket/syfon-only.txt") + if finding.EvidenceStatus != "unknown" || finding.Actionability != storageActionabilityManualChoice || finding.DefaultAction != storageActionDeleteSyfonRecord { + t.Fatalf("expected inventory-miss probe error with a manual remediation choice, got %+v", finding) + } +} + +func TestBuildStorageChainAuditDefaultListExposesBucketOnlyObjects(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-present", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + AccessURLs: []string{"s3://bucket/present.txt"}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present.txt", Bucket: "bucket", Key: "present.txt", SizeBytes: 100}, + {ObjectURL: "s3://bucket/loose.txt", Bucket: "bucket", Key: "loose.txt", SizeBytes: 25}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build default chain audit: %v", err) + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected default list audit to load bucket inventory, got %d calls", backend.listProjectBucketInventoryCalls) + } + if got := chain.Summary.CountsByKind["bucket_only_object"]; got != 1 { + t.Fatalf("expected one bucket-only finding, got summary %+v", chain.Summary) + } + finding := assertHasChainFinding(t, chain.Findings, "bucket_only_object", "s3://bucket/loose.txt") + if finding.DefaultAction != storageActionDeleteBucketObject || len(finding.ObjectIDs) != 0 { + t.Fatalf("expected bucket-only deletion evidence without Syfon IDs, got %+v", finding) + } +} + +func TestBuildStorageChainAuditMetadataModeUsesMetadataValidation(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a.txt", + Bucket: "bucket", + Key: "a.txt", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"sha256_mismatch"}, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "proj", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + ValidationMode: StorageChainValidationModeMetadata, + }, + ) + if err != nil { + t.Fatalf("build metadata chain audit: %v", err) + } + if backend.listProjectBucketObjectsCalls != 0 { + t.Fatalf("expected metadata validation to skip recursive bucket inventory, got %d calls", backend.listProjectBucketObjectsCalls) + } + if backend.listProbeCalls != 0 { + t.Fatalf("expected metadata validation to skip LIST validation, got %d calls", backend.listProbeCalls) + } + if backend.probeCalls != 1 || len(backend.probeItems) != 1 { + t.Fatalf("expected one metadata probe call, got calls=%d items=%+v", backend.probeCalls, backend.probeItems) + } + if chain.Summary.ValidationMode != StorageChainValidationModeMetadata { + t.Fatalf("expected metadata validation mode, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_only_object"]; got != 0 { + t.Fatalf("metadata validation must not claim bucket-only coverage, got summary %+v", chain.Summary) + } + finding := assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/a.txt") + if finding.Evidence == nil || !contains(finding.Evidence.StorageOperations, StorageChainValidationModeMetadata) { + t.Fatalf("expected metadata operation evidence, got %+v", finding.Evidence) + } +} + +func TestBuildStorageChainAuditIgnoresOneByteListSizeDrift(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 44), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Name: "a.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 44, AccessURLs: []string{"s3://bucket/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/a.txt", Bucket: "bucket", Key: "data/a.txt", Path: "data/a.txt", SizeBytes: 43}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + ValidationMode: StorageChainValidationModeList, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["git_syfon_metadata_mismatch"]; got != 0 { + t.Fatalf("expected one-byte size drift to be ignored, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected one-byte size drift to count as complete, got summary %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditSuggestsFixForLargeSizeDrift(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 44), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Name: "a.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 44, AccessURLs: []string{"s3://bucket/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/a.txt", Bucket: "bucket", Key: "data/a.txt", Path: "data/a.txt", SizeBytes: 40}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + ValidationMode: StorageChainValidationModeList, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + finding := assertHasChainFinding(t, chain.Findings, "git_syfon_metadata_mismatch", "data/a.txt") + if !strings.Contains(finding.SuggestedFix, "Syfon record size is 44 B, bucket inventory reports 40 B") || + !strings.Contains(finding.SuggestedFix, "bucket object cannot have the current Git/Syfon SHA-256") { + t.Fatalf("expected size-drift suggested fix, got %+v", finding) + } + if finding.SuggestedAction != "" { + t.Fatalf("did not expect fake suggested action for size drift, got %+v", finding) + } +} + +func TestBuildStorageChainAuditIgnoresBucketBasenameMismatch(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/right.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Name: "right.txt", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + AccessURLs: []string{"s3://bucket/data/wrong.txt"}, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{ + {AccessID: "s3", Type: "s3", URL: "s3://bucket/data/wrong.txt"}, + }, + }, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/wrong.txt", Bucket: "bucket", Key: "data/wrong.txt", Path: "data/wrong.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + ValidationMode: StorageChainValidationModeList, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["git_syfon_metadata_mismatch"]; got != 0 { + t.Fatalf("expected basename-only mismatch to be ignored, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected basename-only mismatch not to be a broken mapping, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected basename-only mismatch to count as complete, got summary %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditCachesProjectChainInputs(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/present"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/present", Bucket: "bucket", Key: "present", Path: "present", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + options := StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly} + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build first chain audit: %v", err) + } + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build second chain audit: %v", err) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected cached project records, got %d calls", backend.listProjectAuditRecordsCalls) + } + if backend.listProjectAuditRecordsPathPrefix != "" { + t.Fatalf("expected cached project records to load the full project, got %q", backend.listProjectAuditRecordsPathPrefix) + } + if backend.getProjectMetricsSummaryCalls != 2 { + t.Fatalf("expected one cheap summary check per audit, got %d", backend.getProjectMetricsSummaryCalls) + } + if backend.listProjectScopesCalls != 1 { + t.Fatalf("expected cached project scopes, got %d calls", backend.listProjectScopesCalls) + } + if backend.listProjectBucketObjectsCalls != 1 { + t.Fatalf("expected cached project bucket inventory, got %d calls", backend.listProjectBucketObjectsCalls) + } +} + +func TestBuildStorageChainAuditCachesFullResponseAcrossFindingViews(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Name: "present.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + {ObjectID: "obj-syfon-only", Name: "syfon-only.txt", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, AccessURLs: []string{"s3://bucket/data/syfon-only.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + {ObjectURL: "s3://bucket/data/syfon-only.txt", Bucket: "bucket", Key: "data/syfon-only.txt", Path: "data/syfon-only.txt", SizeBytes: 200}, + }, + } + service := NewStorageAnalyticsService(backend) + service.chainAuditResponseCache = newMemoryStorageChainAuditResponseCache() + + first, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + FindingKind: "git_only_no_syfon", + FindingLimit: -1, + }) + if err != nil { + t.Fatalf("build first chain audit: %v", err) + } + if first.Summary.AuditCacheHit { + t.Fatalf("expected first response to build fresh, got %+v", first.Summary) + } + if first.Summary.ReturnedFindings != 0 || first.Summary.FindingsTruncated { + t.Fatalf("expected first response to apply request finding kind, got %+v", first.Summary) + } + + second, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + FindingKind: "bucket_syfon_no_git", + FindingLimit: -1, + }) + if err != nil { + t.Fatalf("build second chain audit: %v", err) + } + if !second.Summary.AuditCacheHit { + t.Fatalf("expected second response to come from audit response cache, got %+v", second.Summary) + } + if len(second.Findings) != 1 || second.Findings[0].Kind != "bucket_syfon_no_git" { + t.Fatalf("expected cached full response to serve issue-kind view, got %+v", second.Findings) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected cached full response to skip Syfon record reload, got %d calls", backend.listProjectAuditRecordsCalls) + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected cached full response to skip bucket inventory reload, got %d calls", backend.listProjectBucketInventoryCalls) + } +} + +func TestBuildStorageChainAuditProjectsGitSubpathFromRootResponseCache(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "other/clean.txt": lfsPointer("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", 300), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Name: "present.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + {ObjectID: "obj-clean", Name: "clean.txt", Checksum: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Organization: "org", Project: "proj", Size: 300, AccessURLs: []string{"s3://bucket/other/clean.txt"}}, + {ObjectID: "obj-syfon-only", Name: "syfon-only.txt", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, AccessURLs: []string{"s3://bucket/data/syfon-only.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + {ObjectURL: "s3://bucket/other/clean.txt", Bucket: "bucket", Key: "other/clean.txt", Path: "other/clean.txt", SizeBytes: 300}, + {ObjectURL: "s3://bucket/data/syfon-only.txt", Bucket: "bucket", Key: "data/syfon-only.txt", Path: "data/syfon-only.txt", SizeBytes: 200}, + }, + } + service := NewStorageAnalyticsService(backend) + cache := newMemoryStorageChainAuditResponseCache() + service.chainAuditResponseCache = cache + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + root, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options) + if err != nil { + t.Fatalf("build root chain audit: %v", err) + } + if root.Summary.AuditCacheHit { + t.Fatalf("expected root audit to build fresh, got %+v", root.Summary) + } + if backend.listProjectAuditRecordsCalls != 1 || backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected root audit to load Syfon records and bucket inventory once, got records=%d bucket=%d", backend.listProjectAuditRecordsCalls, backend.listProjectBucketInventoryCalls) + } + cache.mu.Lock() + for key, entry := range cache.entries { + entry.value.RefreshDurationMillis = 123 + cache.entries[key] = entry + } + cache.mu.Unlock() + cachedRoot, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options) + if err != nil { + t.Fatalf("build cached root chain audit: %v", err) + } + if !cachedRoot.Summary.AuditCacheHit || cachedRoot.Summary.AuditRefreshDurationMs != 123 { + t.Fatalf("expected cached root audit to include refresh duration, got %+v", cachedRoot.Summary) + } + subpath, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options) + if err != nil { + t.Fatalf("build subpath chain audit: %v", err) + } + if !subpath.Summary.AuditCacheHit || subpath.Summary.AuditCacheSource != "memory:root" { + t.Fatalf("expected subpath audit to project from root response cache, got %+v", subpath.Summary) + } + if subpath.Summary.AuditRefreshDurationMs != 123 { + t.Fatalf("expected subpath audit to include root refresh duration, got %+v", subpath.Summary) + } + if subpath.PathPrefix != "data" || subpath.Summary.GitTrackedFileCount != 1 { + t.Fatalf("expected data-scoped audit projection, got path=%q summary=%+v", subpath.PathPrefix, subpath.Summary) + } + if got := subpath.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected subpath complete count from Git inventory, got summary %+v", subpath.Summary) + } + if got := subpath.Summary.CountsByKind["bucket_syfon_no_git"]; got != 1 { + t.Fatalf("expected subpath Syfon-only finding from root cache, got summary %+v findings=%+v", subpath.Summary, subpath.Findings) + } + if backend.listProjectAuditRecordsCalls != 1 || backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected subpath projection to skip Syfon records and bucket inventory, got records=%d bucket=%d", backend.listProjectAuditRecordsCalls, backend.listProjectBucketInventoryCalls) + } + cache.mu.RLock() + cacheEntryCount := len(cache.entries) + cache.mu.RUnlock() + if cacheEntryCount != 1 { + t.Fatalf("expected only root response cache entry, got %d", cacheEntryCount) + } +} + +func TestBuildStorageChainAuditForceRefreshBypassesResponseCache(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Name: "present.txt", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + service.chainAuditResponseCache = newMemoryStorageChainAuditResponseCache() + service.projectBucketCache = newMemoryProjectBucketInventoryCache() + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build first chain audit: %v", err) + } + options.ForceAuditRefresh = true + refreshed, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, options) + if err != nil { + t.Fatalf("build forced chain audit: %v", err) + } + if refreshed.Summary.AuditCacheHit { + t.Fatalf("expected forced response to rebuild fresh, got %+v", refreshed.Summary) + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected forced refresh to reuse the bucket inventory during its cooldown, got %d calls", backend.listProjectBucketInventoryCalls) + } +} + +func TestProjectBucketInventoryUsesStaleCacheWhenRefreshFails(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{ + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: "s3://bucket/data/present.txt", + Bucket: "bucket", + Key: "data/present.txt", + Path: "data/present.txt", + SizeBytes: 100, + }}, + } + service := NewStorageAnalyticsService(backend) + cache := newMemoryProjectBucketInventoryCache() + service.projectBucketCache = cache + + objects, _, err := service.loadCachedProjectBucketValidationInventory(context.Background(), "Bearer token", "org", "proj", "") + if err != nil || len(objects) != 1 { + t.Fatalf("seed cached bucket inventory: objects=%d err=%v", len(objects), err) + } + key := projectBucketInventoryCacheKey("org", "proj", "") + cache.mu.Lock() + entry := cache.entries[key] + entry.value.CachedAt = time.Now().Add(-projectBucketInventoryRefreshInterval - time.Second) + cache.entries[key] = entry + cache.mu.Unlock() + service.chainInputMu.Lock() + chainKey := service.projectChainInputCacheKey("org", "proj") + "::bucket-validation-inventory::" + chainEntry := service.chainInputCache[chainKey] + chainEntry.expiresAt = time.Now().Add(-time.Second) + service.chainInputCache[chainKey] = chainEntry + service.chainInputMu.Unlock() + backend.listProjectBucketObjectsErr = fmt.Errorf("provider returned an incomplete listing") + + objects, lookup, err := service.loadCachedProjectBucketValidationInventory(context.Background(), "Bearer token", "org", "proj", "") + if err != nil { + t.Fatalf("expected stale inventory fallback, got %v", err) + } + if len(objects) != 1 || len(lookup) != 1 { + t.Fatalf("expected cached inventory after failed refresh, got objects=%d lookup=%d", len(objects), len(lookup)) + } + if backend.listProjectBucketInventoryCalls != 2 { + t.Fatalf("expected one seed scan and one failed refresh, got %d calls", backend.listProjectBucketInventoryCalls) + } +} + +func TestStorageFolderExactReusesAuditProjectRecordCache(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-present": {ObjectID: "obj-present", DownloadCount: 3}, + }, + } + service := NewStorageAnalyticsService(backend) + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly}); err != nil { + t.Fatalf("build chain audit: %v", err) + } + summary, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + if err != nil { + t.Fatalf("build exact storage summary: %v", err) + } + if summary.RecordCount != 1 || summary.DownloadCount != 3 { + t.Fatalf("expected exact summary to join cached record and usage, got %+v", summary) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected audit and exact summary to share one project record enumeration, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.getProjectMetricsSummaryCalls != 2 { + t.Fatalf("expected one cheap metrics validation per request, got %d", backend.getProjectMetricsSummaryCalls) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to avoid checksum bulk record lookup, got %d", backend.bulkGetProjectRecordsCalls) + } +} + +func TestStorageFolderExactWarmsAuditProjectRecordCache(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + + if _, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash); err != nil { + t.Fatalf("build exact storage summary: %v", err) + } + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly}); err != nil { + t.Fatalf("build chain audit: %v", err) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected exact summary and audit to share one project record enumeration, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.getProjectMetricsSummaryCalls != 2 { + t.Fatalf("expected one cheap metrics validation per request, got %d", backend.getProjectMetricsSummaryCalls) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to avoid checksum bulk record lookup, got %d", backend.bulkGetProjectRecordsCalls) + } +} + +func TestStorageFolderExactAndAuditCoalesceProjectRecordRefresh(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/present.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + listProjectAuditRecordsDelay: 50 * time.Millisecond, + projectMetricsSummary: &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 1, + RecordLatestUpdatedTime: "2026-07-01T00:00:00Z", + }, + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-present", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/present.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/data/present.txt", Bucket: "bucket", Key: "data/present.txt", Path: "data/present.txt", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + errs := make(chan error, 2) + + go func() { + _, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + errs <- err + }() + go func() { + _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ProbeMode: StorageChainProbeModeInventoryOnly}) + errs <- err + }() + for i := 0; i < 2; i++ { + if err := <-errs; err != nil { + t.Fatalf("expected concurrent exact summary and audit to succeed, got %v", err) + } + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected concurrent exact summary and audit to coalesce one project record refresh, got %d", backend.listProjectAuditRecordsCalls) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to avoid checksum bulk record lookup, got %d", backend.bulkGetProjectRecordsCalls) + } +} + +func TestBuildStorageChainAuditCachesProjectRecordsPerPathPrefix(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "CONFIG/a.json": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "DATA/b.json": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/root/CONFIG/a.json"}}, + {ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/root/DATA/b.json"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket/root"}, + }, + } + service := NewStorageAnalyticsService(backend) + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "CONFIG", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build first chain audit: %v", err) + } + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "DATA", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build second chain audit: %v", err) + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected project-scoped record cache to serve both prefixes, got %d calls", backend.listProjectAuditRecordsCalls) + } +} + +func TestBuildStorageChainAuditCoalescesConcurrentProjectRecordRefresh(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + listProjectAuditRecordsDelay: 50 * time.Millisecond, + projectMetricsSummary: &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 1, + RecordLatestUpdatedTime: "2026-07-01T00:00:00Z", + }, + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + } + service := NewStorageAnalyticsService(backend) + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + var wg sync.WaitGroup + errs := make(chan error, 2) + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options) + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("build concurrent chain audit: %v", err) + } + } + if backend.listProjectAuditRecordsCalls != 1 { + t.Fatalf("expected concurrent audits to share one project record refresh, got %d calls", backend.listProjectAuditRecordsCalls) + } + if backend.getProjectMetricsSummaryCalls != 2 { + t.Fatalf("expected each audit to use the cheap metrics validator, got %d calls", backend.getProjectMetricsSummaryCalls) + } +} + +func TestBuildStorageChainAuditRefreshesProjectRecordsWhenMetricsValidatorChanges(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + backend := &fakeStorageAnalyticsBackend{ + projectMetricsSummary: &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 1, + RecordLatestUpdatedTime: "2026-07-01T00:00:00Z", + }, + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/data/a.txt"}}, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + } + service := NewStorageAnalyticsService(backend) + options := StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate} + + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build first chain audit: %v", err) + } + backend.projectMetricsSummary = &gintegrationsyfon.ProjectMetricsSummary{ + RecordCount: 2, + RecordLatestUpdatedTime: "2026-07-02T00:00:00Z", + } + backend.projectRecords = append(backend.projectRecords, gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-b", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, AccessURLs: []string{"s3://bucket/data/b.txt"}, + }) + if _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, options); err != nil { + t.Fatalf("build second chain audit: %v", err) + } + if backend.listProjectAuditRecordsCalls != 2 { + t.Fatalf("expected changed metrics validator to refresh project records, got %d calls", backend.listProjectAuditRecordsCalls) + } +} + +func TestStorageChildrenSkipsSummaryJoinUsageWork(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 1}, + }, + bulkGetDelay: 75 * time.Millisecond, + } + service := NewStorageAnalyticsService(backend) + errCh := make(chan error, 2) + go func() { + _, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash) + errCh <- err + }() + go func() { + _, err := service.BuildStorageChildren(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, 1000, "bytes", "desc", "") + errCh <- err + }() + for i := 0; i < 2; i++ { + if err := <-errCh; err != nil { + t.Fatalf("expected concurrent storage analytics request to succeed, got %v", err) + } + } + if backend.bulkGetProjectRecordsCalls != 1 { + t.Fatalf("expected only page-scoped children to use checksum lookup, got %d", backend.bulkGetProjectRecordsCalls) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected paginated usage lookup to remain unused, got %d", backend.listProjectFileUsageCalls) + } + if backend.listProjectFileUsageByObjectIDsCalls != 1 { + t.Fatalf("expected only summary to perform bulk usage lookup, got %d", backend.listProjectFileUsageByObjectIDsCalls) + } +} + +func TestListProjectFileUsageByObjectIDsChunksRequests(t *testing.T) { + objectIDs := make([]string, 0, projectFileUsageBulkChunkSize+1) + for i := 0; i < projectFileUsageBulkChunkSize+1; i++ { + objectIDs = append(objectIDs, fmt.Sprintf("obj-%04d", i)) + } + backend := &fakeStorageAnalyticsBackend{ + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-0000": {ObjectID: "obj-0000", DownloadCount: 1}, + "obj-5000": {ObjectID: "obj-5000", DownloadCount: 2}, + }, + } + service := NewStorageAnalyticsService(backend) + + usage, err := service.listProjectFileUsageByObjectIDs(context.Background(), "Bearer token", "org", "proj", objectIDs, cleanupInactiveDays) + if err != nil { + t.Fatalf("list bulk usage: %v", err) + } + if backend.listProjectFileUsageByObjectIDsCalls != 2 { + t.Fatalf("expected two bulk usage chunks, got %d", backend.listProjectFileUsageByObjectIDsCalls) + } + if backend.listProjectFileUsageCalls != 0 { + t.Fatalf("expected no paginated fallback, got %d", backend.listProjectFileUsageCalls) + } + if len(backend.listProjectFileUsageObjectIDs) != projectFileUsageBulkChunkSize+1 { + t.Fatalf("expected all ids to be requested, got %d", len(backend.listProjectFileUsageObjectIDs)) + } + if usage["obj-0000"].DownloadCount != 1 || usage["obj-5000"].DownloadCount != 2 { + t.Fatalf("unexpected usage map: %+v", usage) + } +} + +func TestListProjectFileUsageByObjectIDsFallsBackWhenBulkEndpointMissing(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{ + listProjectFileUsageByObjectIDsErr: &gintegrationsyfon.HTTPError{ + Method: "POST", + Path: "/index/v1/metrics/files/bulk", + Status: http.StatusNotFound, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{ + "obj-a": {ObjectID: "obj-a", DownloadCount: 9}, + }, + } + service := NewStorageAnalyticsService(backend) + + usage, err := service.listProjectFileUsageByObjectIDs(context.Background(), "Bearer token", "org", "proj", []string{"obj-a"}, cleanupInactiveDays) + if err != nil { + t.Fatalf("list fallback usage: %v", err) + } + if backend.listProjectFileUsageByObjectIDsCalls != 1 || backend.listProjectFileUsageCalls != 1 { + t.Fatalf("expected one bulk attempt and one paginated fallback, got bulk=%d paginated=%d", backend.listProjectFileUsageByObjectIDsCalls, backend.listProjectFileUsageCalls) + } + if usage["obj-a"].DownloadCount != 9 { + t.Fatalf("unexpected fallback usage: %+v", usage) + } +} + +func TestLoadProjectJoinCacheFiltersSharedRecordsByDeduplicatedGitChecksums(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/b.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-orphan", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 100, AccessURLs: []string{"s3://bucket/orphan"}}, + }, + } + service := NewStorageAnalyticsService(backend) + if _, err := service.BuildStorageSummary(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash); err != nil { + t.Fatalf("build storage summary: %v", err) + } + if backend.bulkGetProjectRecordsCalls != 0 { + t.Fatalf("expected exact summary to avoid checksum lookup, got %d calls", backend.bulkGetProjectRecordsCalls) + } + if len(backend.listProjectFileUsageObjectIDs) != 1 || backend.listProjectFileUsageObjectIDs[0] != "obj-a" { + t.Fatalf("expected duplicate Git checksums to request one matched object usage row, got %+v", backend.listProjectFileUsageObjectIDs) + } +} + +func TestBuildStorageChainAuditForwardsBucketPathPrefixForExplicitInventory(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "README.md": "fixture", + }) + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/root/CONFIG/a.json", Bucket: "bucket", Key: "root/CONFIG/a.json", Path: "CONFIG/a.json", SizeBytes: 10}, + {ObjectURL: "s3://bucket/root/CONFIG/nested/b.json", Bucket: "bucket", Key: "root/CONFIG/nested/b.json", Path: "CONFIG/nested/b.json", SizeBytes: 15}, + }, + } + service := NewStorageAnalyticsService(backend) + response, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeItems, + BucketPathPrefix: "/CONFIG/", + ProbeMode: StorageChainProbeModeInventoryOnly, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if backend.listProjectBucketObjectsCalls != 1 { + t.Fatalf("expected one recursive bucket inventory call, got %d", backend.listProjectBucketObjectsCalls) + } + if backend.listProjectBucketObjectsPathPrefix != "CONFIG" { + t.Fatalf("expected bucket path prefix to be forwarded, got %q", backend.listProjectBucketObjectsPathPrefix) + } + if response.BucketPathPrefix != "CONFIG" { + t.Fatalf("expected response bucket path prefix, got %q", response.BucketPathPrefix) + } + if response.Summary.BucketObjectCount != 2 { + t.Fatalf("expected recursive LIST rows to drive bucket count, got %+v", response.Summary) + } +} + +func TestBuildStorageChainAuditSurfacesMetadataMismatchAndEvidence(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 999}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + findings := loadAllChainFindings(t, service, "org", "proj", chain) + finding := assertHasChainFinding(t, findings, "git_syfon_metadata_mismatch", "data/a.txt") + if finding.Evidence == nil || + len(finding.Evidence.BucketObjectURLs) != 1 || + finding.Evidence.BucketObjectURLs[0] != "s3://bucket/a" || + len(finding.Evidence.ValidationStates) != 1 || + finding.Evidence.ValidationStates[0] != "mismatched" { + t.Fatalf("expected metadata mismatch evidence, got %+v", finding) + } + if finding.Actionability != "manual_choice" || + !contains(finding.AvailableActions, "delete_syfon_record") || + !contains(finding.AvailableActions, "delete_bucket_object") || + !contains(finding.AvailableActions, "delete_both") || + finding.DefaultAction != "" || + !finding.SupportsDryRun { + t.Fatalf("expected metadata mismatch chain actions, got %+v", finding) + } +} + +func TestBuildStorageChainAuditFiltersFindingsByKind(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/bad-map.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200), + }) + now := time.Date(2026, 7, 2, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + {ObjectID: "obj-bad-map", Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Organization: "org", Project: "proj", Size: 200, UpdatedAt: &now, AccessURLs: []string{"s3://legacy-bucket/bad-map.txt"}}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 999}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + storageProbeRequestKey("s3://legacy-bucket/bad-map.txt", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"): { + ID: storageProbeRequestKey("s3://legacy-bucket/bad-map.txt", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ObjectURL: "s3://legacy-bucket/bad-map.txt", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "legacy-bucket"`, + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{ + FindingKind: "syfon_broken_bucket_mapping", + FindingLimit: 1, + }) + if err != nil { + t.Fatalf("build filtered chain audit: %v", err) + } + if chain.Summary.TotalFindings != 1 || chain.Summary.ReturnedFindings != 1 || chain.Summary.FindingsTruncated { + t.Fatalf("expected one filtered finding without truncation, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 1 { + t.Fatalf("expected filtered broken mapping count, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["git_syfon_metadata_mismatch"]; got != 0 { + t.Fatalf("expected mismatch count to be excluded from filtered response, got %+v", chain.Summary) + } + if len(chain.Groups) != 1 || chain.Groups[0].Kind != "syfon_broken_bucket_mapping" { + t.Fatalf("expected one filtered group, got %+v", chain.Groups) + } + if len(chain.Findings) != 1 || chain.Findings[0].Kind != "syfon_broken_bucket_mapping" || chain.Findings[0].NormalizedPath != "data/bad-map.txt" { + t.Fatalf("expected filtered broken mapping detail row, got %+v", chain.Findings) + } +} + +func TestBuildStorageChainAuditDefaultTruncationKeepsRowsPerIssueKind(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/unrelated.txt": lfsPointer("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 1), + }) + now := time.Date(2026, 7, 4, 12, 0, 0, 0, time.UTC) + records := make([]gintegrationsyfon.ProjectRecord, 0, 2) + probeResults := make(map[string]gintegrationsyfon.BulkStorageProbeResult) + for i := 0; i < 2; i++ { + checksum := fmt.Sprintf("%064x", i+1) + objectID := fmt.Sprintf("obj-broken-%03d", i) + objectURL := fmt.Sprintf("s3://legacy-bucket/broken-%03d.txt", i) + records = append(records, gintegrationsyfon.ProjectRecord{ + ObjectID: objectID, + Checksum: checksum, + Organization: "org", + Project: "proj", + Size: 100 + int64(i), + UpdatedAt: &now, + AccessURLs: []string{objectURL}, + }) + probeResults[storageProbeRequestKey(objectURL, 100+int64(i), checksum)] = gintegrationsyfon.BulkStorageProbeResult{ + ID: storageProbeRequestKey(objectURL, 100+int64(i), checksum), + ObjectURL: objectURL, + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "legacy-bucket"`, + ValidationStatus: "unverifiable", + } + } + bucketObjects := make([]gintegrationsyfon.ProjectBucketObject, 0, 501) + for i := 0; i < 501; i++ { + bucketObjects = append(bucketObjects, gintegrationsyfon.ProjectBucketObject{ + ObjectURL: fmt.Sprintf("s3://bucket/loose-%03d.txt", i), + Bucket: "bucket", + Key: fmt.Sprintf("loose-%03d.txt", i), + Path: fmt.Sprintf("loose-%03d.txt", i), + SizeBytes: 10, + }) + } + backend := &fakeStorageAnalyticsBackend{ + projectRecords: records, + bucketObjects: bucketObjects, + probeResults: probeResults, + } + service := NewStorageAnalyticsService(backend) + + truncated, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + FindingLimit: 500, + }) + if err != nil { + t.Fatalf("build truncated chain audit: %v", err) + } + if !truncated.Summary.FindingsTruncated || truncated.Summary.ReturnedFindings != 503 { + t.Fatalf("expected per-kind truncated response, got %+v", truncated.Summary) + } + if got := truncated.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 2 { + t.Fatalf("expected summary to retain broken mapping count, got %+v", truncated.Summary) + } + assertHasChainFinding(t, truncated.Findings, "syfon_broken_bucket_mapping", "syfon/obj-broken-000") + assertHasChainFinding(t, truncated.Findings, "syfon_broken_bucket_mapping", "syfon/obj-broken-001") + + filtered, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + FindingKind: "syfon_broken_bucket_mapping", + FindingLimit: -1, + }) + if err != nil { + t.Fatalf("build filtered chain audit: %v", err) + } + if filtered.Summary.FindingsTruncated || filtered.Summary.ReturnedFindings != 2 { + t.Fatalf("expected full filtered response, got %+v", filtered.Summary) + } + if len(filtered.Findings) != 2 { + t.Fatalf("expected filtered broken mapping detail rows, got %+v", filtered.Findings) + } + assertHasChainFinding(t, filtered.Findings, "syfon_broken_bucket_mapping", "syfon/obj-broken-000") + assertHasChainFinding(t, filtered.Findings, "syfon_broken_bucket_mapping", "syfon/obj-broken-001") +} + +func TestBuildStorageChainAuditUsesScopedProjectRecordsForGitJoin(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + bulkRecords: map[string][]gintegrationsyfon.ProjectRecord{}, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + for _, finding := range loadAllChainFindings(t, service, "org", "proj", chain) { + if finding.Kind == "git_only_no_syfon" && finding.NormalizedPath == "data/a.txt" { + t.Fatalf("did not expect scoped Syfon record to be reclassified as git-only when bulk checksum lookup is empty: %+v", finding) + } + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected one fully connected bucket->syfon->git chain, got summary %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditCanonicalizesScopedLegacyAccessURLs(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/slide.ome.tiff": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "HTAN_INT", + Project: "BForePC", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc-prod/OHSU/slide.ome.tiff"}, + }, + }, + buckets: map[string]domain.StorageBucket{ + "bforepc": {Bucket: "bforepc", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bforepc": {{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", Bucket: "bforepc", Key: "bforepc-prod/OHSU/slide.ome.tiff", Path: "slide.ome.tiff", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bforepc-prod/OHSU/slide.ome.tiff", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bforepc-prod/OHSU/slide.ome.tiff", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bforepc-prod/OHSU/slide.ome.tiff", + Status: "not_found", + Exists: false, + ErrorKind: "credential_missing", + Error: `no stored bucket credential found for bucket "bforepc-prod"`, + ValidationStatus: "unverifiable", + }, + storageProbeRequestKey("s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", + Bucket: "bforepc", + Key: "bforepc-prod/OHSU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected mapped bucket match to suppress stale raw URL misclassification, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected mapped bucket match to preserve clean-chain count, got %+v", chain.Summary) + } + findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) + for _, finding := range findings { + if finding.NormalizedPath == "data/slide.ome.tiff" { + t.Fatalf("did not expect stale raw URL to produce a chain finding once mapped object exists: %+v", finding) + } + } + if len(backend.probeItems) != 1 { + t.Fatalf("expected only the canonical scoped probe, got %+v", backend.probeItems) + } + if containsProbeTarget(backend.probeItems, "s3://bforepc-prod/OHSU/slide.ome.tiff") { + t.Fatalf("did not expect raw alias URL to be probed once scoped URL resolves: %+v", backend.probeItems) + } + if !containsProbeTarget(backend.probeItems, "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff") { + t.Fatalf("expected probe targets to include canonical scoped URL, got %+v", backend.probeItems) + } +} + +func TestBuildStorageChainAuditClassifiesSameBucketPathChecksumConflictAsMetadataMismatch(t *testing.T) { + const repoPath = "JHU/ashley_kiemen/hematoxylin_eosin_stain/Level_2/HTA201_3/HTA201_3_ometif/HTA201_3_1_0022.offsets.json" + const gitChecksum = "99ec07eaecf97dc1cd7b7c4fb717c0661062eb57013d51b81780cbd3a302e323" + const syfonChecksum = "6b33300205aa4730ace8a04147c9223d547b1bf4f54001602d4ec147b7841a4f" + const size = int64(43) + const canonicalURL = "s3://bforepc/bforepc-prod/" + repoPath + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + repoPath: lfsPointer(gitChecksum, size), + }) + now := time.Date(2026, 7, 6, 18, 27, 23, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{{ + ObjectID: "93fd2da3-7b66-5232-bb77-002b78cb8ade", + Name: "HTA201_3_1_0022.offsets.json", + Checksum: syfonChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: size, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc-prod/" + repoPath}, + }}, + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: canonicalURL, + Bucket: "bforepc", + Key: "bforepc-prod/" + repoPath, + Path: repoPath, + SizeBytes: size, + }}, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) + mismatch := assertHasChainFinding(t, findings, "git_syfon_metadata_mismatch", repoPath) + if mismatch.Actionability != storageActionabilityInspectOnly || mismatch.DefaultAction != storageActionInspectEvidence || !contains(mismatch.AvailableActions, storageActionInspectEvidence) { + t.Fatalf("expected checksum conflict to be inspect-only, got %+v", mismatch) + } + if contains(mismatch.AvailableActions, storageActionDeleteBoth) || contains(mismatch.AvailableActions, storageActionDeleteBucketObject) { + t.Fatalf("checksum conflict must not advertise bucket deletion, got %+v", mismatch.AvailableActions) + } + if !strings.Contains(mismatch.SuggestedFix, gitChecksum) || !strings.Contains(mismatch.SuggestedFix, syfonChecksum) { + t.Fatalf("expected both conflicting checksums in the suggested fix, got %q", mismatch.SuggestedFix) + } + for _, kind := range []string{"bucket_only_object", "bucket_syfon_no_git", "git_only_no_syfon"} { + if got := chain.Summary.CountsByKind[kind]; got != 0 { + t.Fatalf("expected same-path checksum conflict to suppress %s, got summary %+v", kind, chain.Summary) + } + } +} + +func TestBuildStorageChainAuditAcceptsMappedNameAndSourcePathInventoryObjects(t *testing.T) { + sourceChecksum := "610cad76a473c4b0bf43f923b3bb907aac9ac2630d9c3f21aba1c531953c43e4" + mappedChecksum := "b71c3fcf28a6d2cf6e432b21a1734786708517f4808c22f9114abb73a60f4cc7" + sourcePath := "OHSU/koei_chin/visium_hd/1-R1/outs/binned_outputs/square_002um/spatial/tissue_positions.parquet" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + sourcePath: lfsPointer(sourceChecksum, 2048), + "META/Research.ndjson": lfsPointer(mappedChecksum, 4722), + }) + now := time.Date(2026, 7, 8, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-source-path", + Name: "tissue_positions.parquet", + Checksum: sourceChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: 2048, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc/" + sourceChecksum}, + }, + { + ObjectID: "obj-mapped-name", + Name: "ResearchSubject.ndjson", + Checksum: mappedChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: 4722, + UpdatedAt: &now, + AccessURLs: []string{"s3://bforepc/" + mappedChecksum}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + { + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://bforepc/bforepc-prod/" + sourcePath, + Bucket: "bforepc", + Key: "bforepc-prod/" + sourcePath, + Path: sourcePath, + SizeBytes: 2048, + }, + { + ObjectURL: "s3://bforepc/" + mappedChecksum, + Bucket: "bforepc", + Key: mappedChecksum, + Path: mappedChecksum, + SizeBytes: 4722, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 2 { + t.Fatalf("expected both source-path and mapped-name objects to complete chains, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 0 { + t.Fatalf("expected no missing bucket findings when either candidate exists, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_missing_bucket_object"]; got != 0 { + t.Fatalf("expected no Syfon-only missing bucket findings when either candidate exists, got summary %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditUsesCanonicalInventoryCandidatesBeforeMissingBucket(t *testing.T) { + sourceChecksum := "db458f223d03fddc541e2ad3c93ff3c152a20ecaa636cdfcb031cff6fe0c6b3a" + mappedChecksum := "b8639eca6eed65224f3af4c70425f578b949ee7202051388c331404cf35073f6" + sourcePath := "OHSU/koei_chin/visium_hd/4-R2/outs/segmented_outputs/analysis/diffexp/gene_expression_kmeans_10_clusters/differential_expression.csv" + mappedRepoPath := "OHSU/koei_chin/visium_hd/4-R2/outs/segmented_outputs/analysis/diffexp/gene_expression_kmeans_2_clusters/differential_expression.csv" + mappedRawURL := "s3://bforepc-prod/97a32a3a-1a30-5e14-bd94-7bf65ac98f27/" + mappedChecksum + sourceRawURL := "s3://bforepc-prod/" + sourcePath + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + sourcePath: lfsPointer(sourceChecksum, 1200), + mappedRepoPath: lfsPointer(mappedChecksum, 1300), + }) + now := time.Date(2026, 7, 9, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-source", + Name: "differential_expression.csv", + Checksum: sourceChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: 1200, + UpdatedAt: &now, + AccessURLs: []string{sourceRawURL}, + }, + { + ObjectID: "obj-mapped", + Name: "differential_expression.csv", + Checksum: mappedChecksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: 1300, + UpdatedAt: &now, + AccessURLs: []string{mappedRawURL}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + { + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://bforepc/bforepc-prod/" + sourcePath, + Bucket: "bforepc", + Key: "bforepc-prod/" + sourcePath, + Path: sourcePath, + SizeBytes: 1200, + }, + { + ObjectURL: "s3://bforepc/bforepc-prod/" + mappedRepoPath, + Bucket: "bforepc", + Key: "bforepc-prod/" + mappedRepoPath, + Path: mappedRepoPath, + SizeBytes: 1300, + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{ + BucketInventoryMode: StorageChainBucketModeValidate, + }) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 2 { + t.Fatalf("expected canonical inventory candidates to complete both chains, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 0 { + t.Fatalf("expected no Git+Syfon missing bucket findings after canonical inventory matching, got summary %+v", chain.Summary) + } + if backend.listProbeCalls != 0 { + t.Fatalf("expected no Syfon bulk-list fallback call, got %d calls with items %+v", backend.listProbeCalls, backend.listProbeItems) + } +} + +func TestBuildStorageChainAuditBForePCListMissIsProbeErrorWithoutMetadataConfirmation(t *testing.T) { + const repoPath = "OHSU/koei_chin/visium_hd/4-R2/outs/binned_outputs/square_002um/filtered_feature_bc_matrix/barcodes.tsv.gz" + const checksum = "a9809ffd53130272f775ad1bbea3166dbb4853bc6fa73bfa7a51535c4bc4f599" + const size = int64(46063992) + const canonicalURL = "s3://bforepc/bforepc-prod/" + repoPath + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + repoPath: lfsPointer(checksum, size), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{{ + ObjectID: "b5162717-bf49-528a-a2a8-47fbd3b0f090", + Name: "barcodes.tsv.gz", + Checksum: checksum, + Organization: "HTAN_INT", + Project: "BForePC", + Size: size, + AccessURLs: []string{"s3://bforepc-prod/" + repoPath}, + }}, + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{}, + listProbeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageListValidationRequestKey(canonicalURL, size, "barcodes.tsv.gz"): { + ID: storageListValidationRequestKey(canonicalURL, size, "barcodes.tsv.gz"), + ObjectURL: canonicalURL, + Status: "not_found", + ErrorKind: "object_not_found", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "HTAN_INT", + "BForePC", + refName, + "", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build BForePC chain audit: %v", err) + } + if backend.listProbeCalls != 1 || len(backend.listProbeItems) != 1 || backend.listProbeItems[0].ObjectURL != canonicalURL { + t.Fatalf("expected one exact LIST for %q, got calls=%d items=%+v", canonicalURL, backend.listProbeCalls, backend.listProbeItems) + } + if backend.probeCalls != 0 { + t.Fatalf("expected no metadata confirmation for the LIST miss at %q, got calls=%d items=%+v", canonicalURL, backend.probeCalls, backend.probeItems) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 0 { + t.Fatalf("LIST miss must not become a missing-bucket finding, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["probe_error"]; got != 1 { + t.Fatalf("expected the exact LIST miss to remain a probe error, got summary %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditCanonicalizesStaleRecordBucketUsingAuditProjectScopes(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/file.bin": lfsPointer("dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + Organization: "Ellrott_Lab", + Project: "hla2vec", + Size: 100, + AccessURLs: []string{ + "s3://EllrottLab/calypr/ff4b54c2-fc50-5850-9fc3-9d662e1f9e44/dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + }, + }, + }, + buckets: map[string]domain.StorageBucket{ + "gdc-mirror-bucket": {Bucket: "gdc-mirror-bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "gdc-mirror-bucket": {{ + Bucket: "gdc-mirror-bucket", + Organization: "gdc_mirror", + ProjectID: "gdc_mirror", + Path: "s3://gdc-mirror-bucket/gdc_mirror", + }}, + }, + projectScopes: []domain.StorageBucketScope{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://gdc-mirror-bucket/gdc_mirror/calypr/ff4b54c2-fc50-5850-9fc3-9d662e1f9e44/dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + Bucket: "gdc-mirror-bucket", + Key: "gdc_mirror/calypr/ff4b54c2-fc50-5850-9fc3-9d662e1f9e44/dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + Path: "dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + SizeBytes: 100, + Provider: "s3", + MetaSHA256: "dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "gdc_mirror", + "gdc_mirror", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if backend.listBucketScopesCalls == 0 { + t.Fatalf("expected audit to fall back to bucket scope enumeration") + } + if got := chain.Summary.CountsByKind["syfon_missing_bucket_object"]; got != 0 { + t.Fatalf("expected stale EllrottLab URL to be canonicalized into the audit project bucket, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected current project bucket object to complete the chain, got summary %+v", chain.Summary) + } + findings := loadAllChainFindings(t, service, "gdc_mirror", "gdc_mirror", chain) + for _, finding := range findings { + if strings.Contains(finding.BucketObjectURL, "EllrottLab") || strings.Contains(finding.NormalizedPath, "EllrottLab") { + t.Fatalf("did not expect stale EllrottLab bucket to appear in findings: %+v", finding) + } + } +} + +func TestBuildStorageChainAuditErrorsWhenRecordBucketCannotMapToAuditProjectScope(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/file.bin": lfsPointer("dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", 100), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "dc4e842bde2d06c161ad96dfcc58084677e9e376989f5b3567c4217b108a0935", + Organization: "Ellrott_Lab", + Project: "hla2vec", + Size: 100, + AccessURLs: []string{"s3://EllrottLab/calypr/file.bin"}, + }, + }, + projectScopes: []domain.StorageBucketScope{}, + } + service := NewStorageAnalyticsService(backend) + + _, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "gdc_mirror", + "gdc_mirror", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err == nil { + t.Fatalf("expected unmapped S3 access URL to fail audit") + } + if !strings.Contains(err.Error(), `storage access URL "s3://EllrottLab/calypr/file.bin" could not be mapped into bucket scopes for project gdc_mirror/gdc_mirror`) { + t.Fatalf("expected explicit unmapped URL error, got %v", err) + } +} + +func TestBuildStorageChainAuditErrorsWhenProjectScopeBucketDisagreesWithInventory(t *testing.T) { + checksum := "ba49c9465b911b950abd60fb333e76032db677b29194ae2268a2a6797402cb4f" + did := "0005c5a4-d3cb-57fc-ad83-78a52c950851" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/file.tsv": lfsPointer(checksum, 4247846), + }) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: did, + Checksum: checksum, + Organization: "gdc_mirror", + Project: "gdc_mirror", + Size: 4247846, + Name: "186b06a9-0b6c-48fe-83fa-451514703728.rna_seq.augmented_star_gene_counts.tsv", + AccessURLs: []string{"s3://gdcdata/" + did + "/" + checksum}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + { + Bucket: "EllrottLab", + Organization: "gdc_mirror", + ProjectID: "gdc_mirror", + Path: "s3://EllrottLab/calypr", + }, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://gdcdata/" + did + "/" + checksum, + Bucket: "gdcdata", + Key: did + "/" + checksum, + Path: "186b06a9-0b6c-48fe-83fa-451514703728.rna_seq.augmented_star_gene_counts.tsv", + SizeBytes: 4247846, + Provider: "s3", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + _, err := service.BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "gdc_mirror", + "gdc_mirror", + refName, + "data", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err == nil { + t.Fatalf("expected scope bucket mismatch to fail audit") + } + if !strings.Contains(err.Error(), `storage access URL mapped to "s3://EllrottLab/calypr/0005c5a4-d3cb-57fc-ad83-78a52c950851/ba49c9465b911b950abd60fb333e76032db677b29194ae2268a2a6797402cb4f"`) { + t.Fatalf("expected mapped EllrottLab URL in error, got %v", err) + } + if !strings.Contains(err.Error(), `bucket "EllrottLab" is not present in project bucket inventory for project gdc_mirror/gdc_mirror`) { + t.Fatalf("expected inventory bucket mismatch error, got %v", err) + } +} + +func TestCanonicalizeScopedStorageURLMatchesSyfonDownloadScopeResolution(t *testing.T) { + tests := []struct { + name string + accessURL string + scopes []domain.StorageBucketScope + org string + project string + want string + }{ + { + name: "project scope rewrites alias bucket to physical bucket", + accessURL: "s3://bforepc-prod/OHSU/koei_chin/visium_hd/barcodes.tsv.gz", + scopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + org: "HTAN_INT", + project: "BForePC", + want: "s3://bforepc/bforepc-prod/OHSU/koei_chin/visium_hd/barcodes.tsv.gz", + }, + { + name: "root scope does not require slash in path", + accessURL: "s3://alias/path/file.txt", + scopes: []domain.StorageBucketScope{{ + Bucket: "physical", + Organization: "org", + ProjectID: "proj", + Path: "s3://physical", + }}, + org: "org", + project: "proj", + want: "s3://physical/path/file.txt", + }, + { + name: "explicit bucket with prefix path", + accessURL: "s3://alias/path/file.txt", + scopes: []domain.StorageBucketScope{{ + Bucket: "physical", + Organization: "org", + ProjectID: "proj", + Path: "root/prefix", + }}, + org: "org", + project: "proj", + want: "s3://physical/root/prefix/path/file.txt", + }, + { + name: "already scoped url is not double prefixed", + accessURL: "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", + scopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + org: "HTAN_INT", + project: "BForePC", + want: "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff", + }, + { + name: "unrelated scope is ignored", + accessURL: "s3://alias/path/file.txt", + scopes: []domain.StorageBucketScope{{ + Bucket: "physical", + Organization: "other", + ProjectID: "proj", + Path: "s3://physical/root", + }}, + org: "org", + project: "proj", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := canonicalizeScopedStorageURL(tt.accessURL, tt.scopes, tt.org, tt.project) + if got != tt.want { + t.Fatalf("unexpected scoped URL: got %q want %q", got, tt.want) + } + }) + } +} + +func TestBuildStorageChainAuditDoesNotInferMappingMismatchFromRelocatedBasename(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "CONFIG/cbds-BForePC.json": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 23739), + }) + now := time.Date(2026, 7, 2, 12, 0, 0, 0, time.UTC) + objectHash := "ec4b068cb42b52449dd44052c3bfb2a459b00336a9cd42cd29c22ca1d1b26cb0" + rawURL := "s3://cbds/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/" + objectHash + canonicalURL := "s3://bforepc/bforepc-prod/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/" + objectHash + relocatedURL := "s3://bforepc/bforepc-prod/2532ab27-4961-57da-8bef-e9774093bf56/" + objectHash + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Name: "CONFIG/cbds-BForePC.json", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "HTAN_INT", + Project: "BForePC", + Size: 23739, + UpdatedAt: &now, + AccessURLs: []string{rawURL}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + { + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: relocatedURL, + Bucket: "bforepc", + Key: "bforepc-prod/2532ab27-4961-57da-8bef-e9774093bf56/" + objectHash, + Path: objectHash, + SizeBytes: 23739, + }, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey(rawURL, 23739, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey(rawURL, 23739, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: rawURL, + Bucket: "cbds", + Key: "0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/" + objectHash, + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + storageProbeRequestKey(canonicalURL, 23739, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey(canonicalURL, 23739, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: canonicalURL, + Bucket: "bforepc", + Key: "bforepc-prod/0b76f9ee-3c82-58e5-8ae2-47addb5d6d79/" + objectHash, + Status: "not_found", + Exists: false, + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "CONFIG", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["syfon_broken_bucket_mapping"]; got != 0 { + t.Fatalf("expected relocated basename not to imply a broken bucket mapping, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_only_object"]; got != 0 { + t.Fatalf("expected relocated hash not to be double-counted as bucket-only, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 1 { + t.Fatalf("expected exact mapped-key miss to remain a missing-object finding, got %+v", chain.Summary) + } + findings := loadAllChainFindings(t, service, "HTAN_INT", "BForePC", chain) + assertHasChainFinding(t, findings, "syfon_git_no_bucket", "CONFIG/cbds-BForePC.json") +} + +func TestClassifyStorageFindingDoesNotTreatRepeatedBasenameAsBrokenMapping(t *testing.T) { + record := projectRecordState{ + ProjectRecord: gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Size: 100, + AccessURLs: []string{"s3://bucket-alias/path/features.tsv.gz"}, + }, + CanonicalAccessURLs: []string{"s3://bucket/root/path/features.tsv.gz"}, + AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ + { + ObjectURL: "s3://bucket/root/path/features.tsv.gz", + Operation: StorageChainValidationModeList, + Bucket: "bucket", + Key: "root/path/features.tsv.gz", + Status: "not_found", + Exists: false, + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }, + }, + } + bucketObjects := map[string]gintegrationsyfon.ProjectBucketObject{ + "s3://bucket/root/unrelated/features.tsv.gz": { + ObjectURL: "s3://bucket/root/unrelated/features.tsv.gz", + Bucket: "bucket", + Key: "root/unrelated/features.tsv.gz", + SizeBytes: 100, + }, + } + + if got := classifyStorageFinding(record, bucketObjects); got != storageFindingObjectMissing { + t.Fatalf("expected a repeated basename without a verified raw object to remain missing, got %s", got) + } +} + +func TestClassifyStorageFindingSuppressesRawURLFailuresWhenScopedProbeMatches(t *testing.T) { + record := projectRecordState{ + ProjectRecord: gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Size: 100, + AccessURLs: []string{"s3://bforepc-prod/JHU/slide.ome.tiff"}, + }, + CanonicalAccessURLs: []string{"s3://bforepc/bforepc-prod/JHU/slide.ome.tiff"}, + CanonicalAccessURLByRaw: map[string]string{ + "s3://bforepc-prod/JHU/slide.ome.tiff": "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + }, + AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ + { + ObjectURL: "s3://bforepc-prod/JHU/slide.ome.tiff", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + ValidationStatus: "unverifiable", + }, + { + ObjectURL: "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + Bucket: "bforepc", + Key: "bforepc-prod/JHU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + + if got := classifyStorageFinding(record, nil); got != storageFindingNone { + t.Fatalf("expected scoped matched probe to suppress raw URL credential miss, got %s", got) + } +} + +func TestClassifyStorageFindingSuppressesRawURLMismatchWhenScopedProbeMatches(t *testing.T) { + record := projectRecordState{ + ProjectRecord: gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Size: 100, + AccessURLs: []string{"s3://bforepc-prod/JHU/slide.ome.tiff"}, + }, + CanonicalAccessURLs: []string{"s3://bforepc/bforepc-prod/JHU/slide.ome.tiff"}, + CanonicalAccessURLByRaw: map[string]string{ + "s3://bforepc-prod/JHU/slide.ome.tiff": "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + }, + AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ + { + ObjectURL: "s3://bforepc-prod/JHU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "mismatched", + ValidationMismatches: []string{"size_mismatch"}, + }, + { + ObjectURL: "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + Bucket: "bforepc", + Key: "bforepc-prod/JHU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + + if got := classifyStorageFinding(record, nil); got != storageFindingNone { + t.Fatalf("expected scoped matched probe to suppress raw URL mismatch, got %s", got) + } +} + +func TestClassifyStorageFindingSurfacesRepairableBrokenAccessMethod(t *testing.T) { + record := projectRecordState{ + ProjectRecord: gintegrationsyfon.ProjectRecord{ + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Size: 100, + AccessURLs: []string{ + "s3://retired-bucket/JHU/slide.ome.tiff", + "s3://bforepc-prod/JHU/slide.ome.tiff", + }, + AccessMethods: []gintegrationsyfon.ProjectAccessMethod{ + {AccessID: "s3", Type: "s3", URL: "s3://retired-bucket/JHU/slide.ome.tiff"}, + {AccessID: "s3", Type: "s3", URL: "s3://bforepc-prod/JHU/slide.ome.tiff"}, + }, + }, + CanonicalAccessURLs: []string{"s3://retired-bucket/JHU/slide.ome.tiff", "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff"}, + CanonicalAccessURLByRaw: map[string]string{ + "s3://retired-bucket/JHU/slide.ome.tiff": "s3://retired-bucket/JHU/slide.ome.tiff", + "s3://bforepc-prod/JHU/slide.ome.tiff": "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + }, + AccessProbes: []gintegrationsyfon.BulkStorageProbeResult{ + { + ObjectURL: "s3://retired-bucket/JHU/slide.ome.tiff", + Status: "error", + Exists: false, + ErrorKind: "credential_missing", + ValidationStatus: "unverifiable", + }, + { + ObjectURL: "s3://bforepc/bforepc-prod/JHU/slide.ome.tiff", + Bucket: "bforepc", + Key: "bforepc-prod/JHU/slide.ome.tiff", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + + if got := classifyStorageFinding(record, nil); got != storageFindingNone { + t.Fatalf("expected the live canonical locator to keep the record connected, got %s", got) + } + broken := repairableBrokenAccessRecord(record) + if len(broken.AccessProbes) != 1 || broken.AccessProbes[0].ObjectURL != "s3://retired-bucket/JHU/slide.ome.tiff" { + t.Fatalf("expected only broken access probe in repair record, got %+v", broken.AccessProbes) + } + remaining, shouldDelete, ok := repairBrokenBucketMappingRecord(broken) + if !ok || shouldDelete { + t.Fatalf("expected repair update, got ok=%v shouldDelete=%v remaining=%+v", ok, shouldDelete, remaining) + } + if len(remaining) != 1 || remaining[0].URL != "s3://bforepc-prod/JHU/slide.ome.tiff" { + t.Fatalf("expected working access method to remain, got %+v", remaining) + } +} + +func TestBuildStorageChainAuditNormalizesChecksumJoinKeys(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/a.txt"}, + }, + }, + bulkRecords: map[string][]gintegrationsyfon.ProjectRecord{ + "sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {{ + ObjectID: "obj-a", + Checksum: "SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/a.txt"}, + }}, + }, + buckets: map[string]domain.StorageBucket{ + "bucket": {Bucket: "bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bucket": {{ + Bucket: "bucket", + Organization: "org", + ProjectID: "proj", + Path: "s3://bucket", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a.txt", Bucket: "bucket", Key: "a.txt", Path: "a.txt", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a.txt", + Bucket: "bucket", + Key: "a.txt", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["git_only_no_syfon"]; got != 0 { + t.Fatalf("expected normalized checksum join to avoid git-only false positive, got %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["bucket_syfon_git_complete"]; got != 1 { + t.Fatalf("expected one normalized complete chain, got %+v", chain.Summary) + } +} + +func TestBuildStorageChainAuditFailsWhenProjectBucketListDenied(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/prefix/a.txt"}, + }, + }, + buckets: map[string]domain.StorageBucket{ + "bucket": {Bucket: "bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bucket": {{ + Bucket: "bucket", + Organization: "org", + ProjectID: "proj", + Path: "s3://bucket/prefix", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + listProjectBucketObjectsErr: fmt.Errorf("list syfon project bucket inventory: syfon POST /data/inspect/project-bucket/inventory failed with status 409: provider rejected bucket inventory request for s3://bucket/prefix; mapped bucket target may be missing or inaccessible"), + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/prefix/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/prefix/a.txt", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/prefix/a.txt", + Bucket: "bucket", + Key: "prefix/a.txt", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + _, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}) + if err == nil { + t.Fatal("expected bucket inventory failure to hard-error chain audit") + } + if !strings.Contains(err.Error(), "mapped bucket target may be missing or inaccessible") { + t.Fatalf("expected bucket inventory error detail, got %v", err) + } + if backend.listProjectBucketInventoryCalls != 1 { + t.Fatalf("expected one bucket validation inventory attempt, got %d", backend.listProjectBucketInventoryCalls) + } + if backend.probeCalls != 0 { + t.Fatalf("expected no probe fallback after bucket inventory failure, got %d", backend.probeCalls) + } +} + +func TestBuildStorageChainAuditReturnsFullFindings(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "data/git-only.txt": lfsPointer("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 50), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + buckets: map[string]domain.StorageBucket{ + "bucket": {Bucket: "bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bucket": {{ + Bucket: "bucket", + Organization: "org", + ProjectID: "proj", + Path: "s3://bucket", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): { + ID: storageProbeRequestKey("s3://bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ObjectURL: "s3://bucket/a", + Bucket: "bucket", + Key: "a", + Status: "present", + Exists: true, + ValidationStatus: "matched", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + chain, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "org", "proj", refName, "data", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build chain audit: %v", err) + } + if len(chain.Findings) == 0 { + t.Fatalf("expected chain audit findings in response, got %+v", chain) + } + if len(chain.Groups) == 0 { + t.Fatalf("expected grouped summary rows, got %+v", chain) + } + if finding := assertHasChainFinding(t, chain.Findings, "git_only_no_syfon", "data/git-only.txt"); finding.NormalizedPath != "data/git-only.txt" { + t.Fatalf("unexpected chain finding: %+v", finding) + } +} + +func TestApplyStorageCleanupDeletesSelectedBucketSyfonNoGitChainFinding(t *testing.T) { + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-no-git", + Checksum: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + Organization: "org", + Project: "proj", + Size: 150, + UpdatedAt: &now, + AccessURLs: []string{"s3://bucket/no-git"}, + }, + }, + projectScopes: []domain.StorageBucketScope{ + {Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket"}, + }, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + { + ObjectURL: "s3://bucket/no-git", + Bucket: "bucket", + Key: "no-git", + Path: "no-git", + SizeBytes: 150, + MetaSHA256: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + response, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"s3://bucket/no-git"}, + nil, + []GitStorageCleanupApplyFinding{{ + Kind: "bucket_syfon_no_git", + NormalizedPath: "s3://bucket/no-git", + ObjectIDs: []string{"obj-no-git"}, + BucketObjectURL: "s3://bucket/no-git", + BucketObjectURLs: []string{"s3://bucket/no-git"}, + AccessURLs: []string{"s3://bucket/no-git"}, + }}, + true, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply cleanup: %v", err) + } + if !contains(response.DeletedRecordIDs, "obj-no-git") || !contains(response.DeletedBucketObjectURLs, "s3://bucket/no-git") { + t.Fatalf("expected selected record and bucket object deletion, got %+v", response) + } + if !contains(backend.deletedIDs, "obj-no-git") || !contains(backend.deletedBucketObjects, "s3://bucket/no-git") { + t.Fatalf("expected backend deletion, IDs=%v bucket=%v", backend.deletedIDs, backend.deletedBucketObjects) + } +} + +func TestBuildStorageChainAuditMarksBucketSyfonNoGitManualRepair(t *testing.T) { + actionability, availableActions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence("bucket_syfon_no_git", "verified") + if actionability != storageActionabilityManualChoice { + t.Fatalf("expected manual-choice actionability, got %q", actionability) + } + if defaultAction != storageActionDeleteBoth { + t.Fatalf("expected delete-both default action, got %q", defaultAction) + } + if !supportsDryRun { + t.Fatal("manual repair must advertise a dry-run") + } + for _, action := range []string{storageActionDeleteBoth, storageActionDeleteSyfonRecord, storageActionDeleteBucketObject} { + if !contains(availableActions, action) { + t.Fatalf("expected action %q, got %+v", action, availableActions) + } + } +} + +func TestBuildStorageCleanupAuditTreatsMissingProbeEvidenceAsProbeError(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + { + ObjectID: "obj-a", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Organization: "org", + Project: "proj", + Size: 100, + UpdatedAt: &now, + AccessURLs: []string{"s3://legacy-bucket/a"}, + }, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/other", Bucket: "bucket", Key: "other", Path: "other", SizeBytes: 100}, + }, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + storageProbeRequestKey("s3://legacy-bucket/a", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"): {}, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, true) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + assertHasCleanupFinding(t, cleanup.Findings, "storage_probe_error", "data/a.txt") + for _, finding := range cleanup.Findings { + if finding.NormalizedPath == "data/a.txt" && finding.Kind == "storage_object_missing" { + t.Fatalf("did not expect missing-object classification without a real not_found probe, got %+v", finding) + } + } +} + +func TestBuildStorageCleanupAuditSkipsBucketStagesWhenCheckStorageDisabled(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + }) + now := time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{ + {ObjectID: "obj-a", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Organization: "org", Project: "proj", Size: 100, UpdatedAt: &now, AccessURLs: []string{"s3://bucket/a"}}, + }, + buckets: map[string]domain.StorageBucket{ + "bucket": {Bucket: "bucket", Provider: "s3"}, + }, + bucketScopes: map[string][]domain.StorageBucketScope{ + "bucket": {{ + Bucket: "bucket", + Organization: "org", + ProjectID: "proj", + Path: "s3://bucket", + }}, + }, + usageByObject: map[string]gintegrationsyfon.FileUsage{}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{ + {ObjectURL: "s3://bucket/a", Bucket: "bucket", Key: "a", Path: "a", SizeBytes: 100}, + }, + } + service := NewStorageAnalyticsService(backend) + + cleanup, _, err := service.BuildStorageCleanupAudit(context.Background(), "Bearer token", "org", "proj", refName, "data", nil, mirrorPath, repo, hash, false) + if err != nil { + t.Fatalf("build cleanup audit: %v", err) + } + if len(cleanup.Findings) != 0 { + t.Fatalf("expected no cleanup findings without storage checks, got %+v", cleanup.Findings) + } + if backend.listProjectBucketObjectsCalls != 0 { + t.Fatalf("expected bucket inventory to be skipped when check_storage=false, got %d calls", backend.listProjectBucketObjectsCalls) + } + if backend.probeCalls != 0 { + t.Fatalf("expected storage probes to be skipped when check_storage=false, got %d calls", backend.probeCalls) + } +} + +func TestPersistRepoAnalyticsIndexAndLoadExistingDirectoryWithoutLFSFiles(t *testing.T) { + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.txt": lfsPointer("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 100), + "plain/notes.txt": "plain text only\n", + "plain/nested/x.md": "still plain\n", + }) + if err := PersistRepoAnalyticsIndex(context.Background(), mirrorPath, repo, refName, hash); err != nil { + t.Fatalf("persist repo analytics index: %v", err) + } + sidecar, err := readRepoAnalyticsIndexSidecar(mirrorPath) + if err != nil { + t.Fatalf("read repo analytics sidecar: %v", err) + } + if sidecar.CommitHash != hash.String() { + t.Fatalf("unexpected sidecar hash: %+v", sidecar) + } + index, err := loadOrBuildRepoAnalyticsIndex(context.Background(), mirrorPath, refName, repo, hash) + if err != nil { + t.Fatalf("load repo analytics index: %v", err) + } + directory, err := repoDirectoryAggregate(index, "plain") + if err != nil { + t.Fatalf("lookup plain directory aggregate: %v", err) + } + if directory.FileCount != 0 || directory.DirectChildCount != 0 { + t.Fatalf("expected zero-lfs directory aggregate, got %+v", directory) + } + filtered, err := filterRepoInventoryFiles(index, "plain") + if err != nil { + t.Fatalf("filter plain directory inventory: %v", err) + } + if len(filtered) != 0 { + t.Fatalf("expected no lfs files under plain directory, got %+v", filtered) + } +} + +func BenchmarkBuildStorageFolderDefaultModeLargeDirectoryPage(b *testing.B) { + children := make([]GitRepoAnalyticsChild, 1000) + for index := 0; index < 1000; index++ { + children[index] = GitRepoAnalyticsChild{ + Name: fmt.Sprintf("file-%04d.txt", index), + Path: fmt.Sprintf("data/file-%04d.txt", index), + Type: "file", + FileCount: 1, + TotalBytes: int64(index + 1), + } + } + hash := plumbing.NewHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + mirrorPath := filepath.Join(b.TempDir(), "mirror.git") + index := repoAnalyticsIndexFromSidecar(GitRepoAnalyticsIndexSidecar{ + SchemaVersion: repoAnalyticsIndexSchemaVersion, + CommitHash: hash.String(), + RefName: "main", + GeneratedAt: time.Now().UTC(), + Directories: []GitRepoAnalyticsDirectory{ + { + Path: "data", + DirectChildCount: len(children), + FileCount: len(children), + TotalBytes: 500500, + Children: children, + }, + }, + }) + repoAnalyticsIndexCache.put(mirrorPath, hash, index) + service := NewStorageAnalyticsService(&fakeStorageAnalyticsBackend{}) + if _, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", "main", "data", mirrorPath, nil, hash, 100, "bytes", "desc", "", "", false, nil); err != nil { + b.Fatalf("warm storage folder index: %v", err) + } + + b.ReportAllocs() + b.ResetTimer() + for index := 0; index < b.N; index++ { + if _, err := service.BuildStorageFolder(context.Background(), "Bearer token", "org", "proj", "main", "data", mirrorPath, nil, hash, 100, "bytes", "desc", "", "", false, nil); err != nil { + b.Fatalf("build storage folder: %v", err) + } + } +} + +func buildAnalyticsMirror(t *testing.T, files map[string]string) (*gogit.Repository, string, string, plumbing.Hash) { + t.Helper() + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + for filePath, content := range files { + fullPath := filepath.Join(sourcePath, filepath.FromSlash(filePath)) + if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filePath, err) + } + if err := os.WriteFile(fullPath, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", filePath, err) + } + if _, err := worktree.Add(filePath); err != nil { + t.Fatalf("add %s: %v", filePath, err) + } + } + if _, err := worktree.Commit("seed analytics repo", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit analytics repo: %v", err) + } + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve ref: %v", err) + } + return mirrorRepo, mirrorPath, refName, hash +} + +func lfsPointer(checksum string, size int64) string { + return strings.Join([]string{ + "version https://git-lfs.github.com/spec/v1", + "oid sha256:" + checksum, + "size " + strconv.FormatInt(size, 10), + "", + }, "\n") +} + +func ptrTime(value time.Time) *time.Time { + copyValue := value + return ©Value +} + +func ptrBool(value bool) *bool { + copyValue := value + return ©Value +} + +func int64Ptr(value int64) *int64 { + copyValue := value + return ©Value +} + +func applyFinding(kind string, normalizedPath string, objectIDs []string, bucketURLs []string) GitStorageCleanupApplyFinding { + policy := storageRepairPolicyForKind(kind) + return GitStorageCleanupApplyFinding{ + Kind: kind, + NormalizedPath: normalizedPath, + ObjectIDs: append([]string(nil), objectIDs...), + BucketObjectURL: firstString(bucketURLs), + BucketObjectURLs: append([]string(nil), bucketURLs...), + AccessURLs: append([]string(nil), bucketURLs...), + AvailableActions: append([]string(nil), policy.actions...), + DefaultAction: policy.defaultAction, + Evidence: &GitAuditEvidence{ + ObjectIDs: append([]string(nil), objectIDs...), + AccessURLs: append([]string(nil), bucketURLs...), + BucketObjectURLs: append([]string(nil), bucketURLs...), + }, + } +} + +func firstString(values []string) string { + if len(values) == 0 { + return "" + } + return values[0] +} + +func assertStringSet(t *testing.T, label string, got []string, want []string) { + t.Helper() + got = uniqueStrings(got) + want = uniqueStrings(want) + if len(got) != len(want) { + t.Fatalf("%s: expected %v, got %v", label, want, got) + } + for _, value := range want { + if !contains(got, value) { + t.Fatalf("%s: expected %v, got %v", label, want, got) + } + } +} + +func contains(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func containsProbeTarget(values []gintegrationsyfon.BulkStorageProbeItem, target string) bool { + for _, value := range values { + if value.ObjectURL == target { + return true + } + } + return false +} + +func assertHasDiffFinding(t *testing.T, findings []GitProjectDiffFinding, kind string, path string) GitProjectDiffFinding { + t.Helper() + for _, finding := range findings { + if finding.Kind == kind && finding.NormalizedPath == path { + return finding + } + } + t.Fatalf("missing diff finding kind=%s path=%s in %+v", kind, path, findings) + return GitProjectDiffFinding{} +} + +func assertHasCleanupFinding(t *testing.T, findings []GitStorageCleanupFinding, kind string, path string) GitStorageCleanupFinding { + t.Helper() + for _, finding := range findings { + if finding.Kind == kind && finding.NormalizedPath == path { + return finding + } + } + t.Fatalf("missing cleanup finding kind=%s path=%s in %+v", kind, path, findings) + return GitStorageCleanupFinding{} +} + +func loadAllChainFindings(t *testing.T, service *StorageAnalyticsService, organization string, project string, chain *GitStorageChainAuditResponse) []GitStorageChainFinding { + t.Helper() + _, _ = service, organization + _ = project + if chain == nil { + t.Fatalf("expected chain audit response, got %+v", chain) + } + return append([]GitStorageChainFinding(nil), chain.Findings...) +} + +func assertHasChainFinding(t *testing.T, findings []GitStorageChainFinding, kind string, path string) GitStorageChainFinding { + t.Helper() + for _, finding := range findings { + if finding.Kind == kind && finding.NormalizedPath == path { + return finding + } + } + t.Fatalf("missing chain finding kind=%s path=%s in %+v", kind, path, findings) + return GitStorageChainFinding{} +} + +func assertNoChainFinding(t *testing.T, findings []GitStorageChainFinding, kind string) { + t.Helper() + for _, finding := range findings { + if finding.Kind == kind { + t.Fatalf("unexpected chain finding kind=%s in %+v", kind, findings) + } + } +} diff --git a/internal/git/storage_analytics_timing.go b/internal/git/storage_analytics_timing.go new file mode 100644 index 0000000..3a36eb8 --- /dev/null +++ b/internal/git/storage_analytics_timing.go @@ -0,0 +1,172 @@ +package git + +import ( + "fmt" + "runtime" + "strings" + "sync" + "time" +) + +const ( + StorageChainProbeModeFull = "full" + StorageChainProbeModeInventoryOnly = "inventory_only" + + StorageChainBucketModeItems = "items" + StorageChainBucketModeValidate = "validate" + + StorageChainValidationModeList = "list" + StorageChainValidationModeMetadata = "metadata" + StorageChainValidationModeInventory = "inventory" +) + +type StorageChainAuditOptions struct { + ProbeMode string + ValidationMode string + BucketInventoryMode string + BucketPathPrefix string + FindingKind string + FindingLimit int + ForceAuditRefresh bool + Timings *StorageChainAuditTimings +} + +type StorageChainAuditStageTiming struct { + Stage string + Duration time.Duration +} + +type StorageChainAuditTimings struct { + mu sync.Mutex + stages []StorageChainAuditStageTiming + Logf func(format string, args ...any) + DebugPrefix string +} + +func (timings *StorageChainAuditTimings) StageStart(stage string) { + if timings == nil || timings.Logf == nil { + return + } + timings.Logf("storage_chain_audit_stage_start %s stage=%s", strings.TrimSpace(timings.DebugPrefix), strings.TrimSpace(stage)) +} + +func (timings *StorageChainAuditTimings) Record(stage string, duration time.Duration) { + if timings == nil { + return + } + normalizedStage := strings.TrimSpace(stage) + timings.mu.Lock() + timings.stages = append(timings.stages, StorageChainAuditStageTiming{ + Stage: normalizedStage, + Duration: duration, + }) + logf := timings.Logf + prefix := strings.TrimSpace(timings.DebugPrefix) + timings.mu.Unlock() + if logf != nil { + logf("storage_chain_audit_stage_done %s stage=%s duration_ms=%d", prefix, normalizedStage, duration.Milliseconds()) + } +} + +func (timings *StorageChainAuditTimings) RecordMemory(stage string, fields ...any) { + if timings == nil || timings.Logf == nil { + return + } + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + prefix := strings.TrimSpace(timings.DebugPrefix) + normalizedStage := strings.TrimSpace(stage) + extra := formatStorageChainMemoryFields(fields...) + timings.Logf( + "storage_chain_audit_memory %s stage=%s alloc_mib=%d heap_alloc_mib=%d heap_inuse_mib=%d heap_sys_mib=%d stack_sys_mib=%d sys_mib=%d next_gc_mib=%d num_gc=%d%s", + prefix, + normalizedStage, + bytesToMiB(stats.Alloc), + bytesToMiB(stats.HeapAlloc), + bytesToMiB(stats.HeapInuse), + bytesToMiB(stats.HeapSys), + bytesToMiB(stats.StackSys), + bytesToMiB(stats.Sys), + bytesToMiB(stats.NextGC), + stats.NumGC, + extra, + ) +} + +func bytesToMiB(bytes uint64) uint64 { + return bytes / 1024 / 1024 +} + +func formatStorageChainMemoryFields(fields ...any) string { + if len(fields) == 0 { + return "" + } + parts := make([]string, 0, len(fields)/2) + for i := 0; i+1 < len(fields); i += 2 { + key := strings.TrimSpace(fmt.Sprint(fields[i])) + if key == "" { + continue + } + parts = append(parts, fmt.Sprintf("%s=%v", key, fields[i+1])) + } + if len(parts) == 0 { + return "" + } + return " " + strings.Join(parts, " ") +} + +func (timings *StorageChainAuditTimings) Snapshot() []StorageChainAuditStageTiming { + if timings == nil { + return nil + } + timings.mu.Lock() + defer timings.mu.Unlock() + out := make([]StorageChainAuditStageTiming, len(timings.stages)) + copy(out, timings.stages) + return out +} + +func NormalizeStorageChainProbeMode(raw string) (string, bool) { + switch strings.TrimSpace(raw) { + case "", StorageChainProbeModeFull: + return StorageChainProbeModeFull, true + case StorageChainProbeModeInventoryOnly: + return StorageChainProbeModeInventoryOnly, true + default: + return "", false + } +} + +func NormalizeStorageChainValidationMode(raw string) (string, bool) { + switch strings.TrimSpace(raw) { + case "", StorageChainValidationModeList: + return StorageChainValidationModeList, true + case StorageChainValidationModeMetadata, StorageChainProbeModeFull: + return StorageChainValidationModeMetadata, true + case StorageChainValidationModeInventory, StorageChainProbeModeInventoryOnly: + return StorageChainValidationModeInventory, true + default: + return "", false + } +} + +func DefaultStorageChainValidationMode(probeMode string, bucketMode string) string { + switch strings.TrimSpace(probeMode) { + case StorageChainProbeModeInventoryOnly: + return StorageChainValidationModeInventory + case StorageChainProbeModeFull: + return StorageChainValidationModeMetadata + } + return StorageChainValidationModeList +} + +func NormalizeStorageChainBucketInventoryMode(raw string) (string, bool) { + switch strings.TrimSpace(raw) { + case "", StorageChainBucketModeItems: + return StorageChainBucketModeItems, true + case StorageChainBucketModeValidate: + return StorageChainBucketModeValidate, true + default: + return "", false + } +} diff --git a/internal/git/storage_audit_evidence.go b/internal/git/storage_audit_evidence.go new file mode 100644 index 0000000..a931798 --- /dev/null +++ b/internal/git/storage_audit_evidence.go @@ -0,0 +1,62 @@ +package git + +import ( + "context" + "fmt" + "strings" + + "github.com/calypr/gecko/internal/storageaudit" +) + +func (service *StorageAnalyticsService) loadStorageAuditSyfonRevision(ctx context.Context, authorizationHeader string, organization string, project string) (string, error) { + summary, err := service.storage.GetProjectMetricsSummary(ctx, authorizationHeader, organization, project) + if err != nil { + return "", fmt.Errorf("get Syfon revision for storage audit: %w", err) + } + if summary == nil { + return "", fmt.Errorf("get Syfon revision for storage audit: response summary is missing") + } + return fmt.Sprintf( + "%d:%s:%s", + summary.RecordCount, + strings.TrimSpace(summary.RecordLatestUpdatedTime), + strings.TrimSpace(summary.RecordRevision), + ), nil +} + +func assessStorageRecordEvidence(record projectRecordState, bucketObserved bool) storageaudit.Assessment { + probes := make([]storageaudit.Probe, 0, len(record.AccessProbes)) + for _, probe := range record.AccessProbes { + probes = append(probes, storageaudit.Probe{ + Operation: strings.TrimSpace(probe.Operation), + Status: strings.TrimSpace(probe.Status), + ErrorKind: strings.TrimSpace(probe.ErrorKind), + ValidationStatus: strings.TrimSpace(probe.ValidationStatus), + }) + } + return storageaudit.Assess(probes, bucketObserved) +} + +func storageEvidenceStatus(record projectRecordState, bucketObserved bool) string { + return string(assessStorageRecordEvidence(record, bucketObserved).Status) +} + +func storageChainActionSupportForEvidence(kind string, evidenceStatus string) (string, []string, string, bool) { + if strings.TrimSpace(kind) == "git_only_no_syfon" && strings.TrimSpace(evidenceStatus) == string(storageaudit.EvidenceVerified) { + return storageActionabilityManualChoice, + []string{storageActionCreateSyfonRecord, storageActionInspectEvidence}, + storageActionCreateSyfonRecord, + false + } + if strings.TrimSpace(kind) == "git_syfon_metadata_mismatch" && strings.TrimSpace(evidenceStatus) == string(storageaudit.EvidenceVerified) { + return storageActionabilityManualChoice, + []string{storageActionDeleteSyfonRecord, storageActionDeleteBucketObject, storageActionDeleteBoth, storageActionInspectEvidence}, + "", + true + } + policy := storageRepairPolicyForKind(kind) + if policy.actionability == storageActionabilityAutoRepair && !storageaudit.AllowsAutomaticRepair(kind, storageaudit.EvidenceStatus(evidenceStatus)) { + policy = inspectOnlyStorageRepairPolicy() + } + return policy.actionability, append([]string(nil), policy.actions...), policy.defaultAction, policy.supportsDryRun +} diff --git a/internal/git/storage_audit_evidence_policy_test.go b/internal/git/storage_audit_evidence_policy_test.go new file mode 100644 index 0000000..071cb35 --- /dev/null +++ b/internal/git/storage_audit_evidence_policy_test.go @@ -0,0 +1,81 @@ +package git + +import ( + "context" + "testing" + + "github.com/calypr/gecko/internal/storageaudit" +) + +func TestStorageChainActionSupportAllowsManualProbeErrorDeletion(t *testing.T) { + actionability, actions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence( + "probe_error", + string(storageaudit.EvidenceUnknown), + ) + + if actionability != storageActionabilityManualChoice { + t.Fatalf("expected manual probe-error deletion, got %q", actionability) + } + if defaultAction != storageActionDeleteSyfonRecord { + t.Fatalf("expected delete-Syfon default, got %q", defaultAction) + } + if !supportsDryRun { + t.Fatal("expected probe-error deletion to support a dry run") + } + if len(actions) != 2 || actions[0] != storageActionDeleteSyfonRecord || actions[1] != storageActionInspectEvidence { + t.Fatalf("unexpected probe-error actions: %+v", actions) + } +} + +func TestStorageChainActionSupportKeepsUnverifiedMissingFindingInspectOnly(t *testing.T) { + actionability, actions, defaultAction, supportsDryRun := storageChainActionSupportForEvidence( + "syfon_git_no_bucket", + string(storageaudit.EvidenceUnknown), + ) + + if actionability != storageActionabilityInspectOnly { + t.Fatalf("expected unverified missing finding to remain inspect-only, got %q", actionability) + } + if defaultAction != storageActionInspectEvidence || supportsDryRun { + t.Fatalf("unexpected unverified finding defaults: action=%q dry_run=%t", defaultAction, supportsDryRun) + } + if len(actions) != 1 || actions[0] != storageActionInspectEvidence { + t.Fatalf("unexpected unverified finding actions: %+v", actions) + } +} + +func TestApplyStorageCleanupDeletesSelectedProbeErrorRecords(t *testing.T) { + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + response, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "project", + []string{"probe-row"}, + []GitStorageCleanupApplyAction{{ + Kind: "probe_error", + NormalizedPath: "probe-row", + Action: storageActionDeleteSyfonRecord, + }}, + []GitStorageCleanupApplyFinding{{ + Kind: "probe_error", + NormalizedPath: "probe-row", + ObjectIDs: []string{"probe-object"}, + }}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("delete selected probe-error record: %v", err) + } + if len(response.DeletedRecordIDs) != 1 || response.DeletedRecordIDs[0] != "probe-object" { + t.Fatalf("unexpected cleanup response: %+v", response) + } + if len(backend.deletedIDs) != 1 || backend.deletedIDs[0] != "probe-object" { + t.Fatalf("expected Syfon deletion only, got %+v", backend) + } +} diff --git a/internal/git/storage_audit_list_confirmation_test.go b/internal/git/storage_audit_list_confirmation_test.go new file mode 100644 index 0000000..07493aa --- /dev/null +++ b/internal/git/storage_audit_list_confirmation_test.go @@ -0,0 +1,71 @@ +package git + +import ( + "context" + "testing" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" +) + +func TestBuildStorageChainAuditListMissDoesNotTriggerMetadataConfirmation(t *testing.T) { + const ( + repoPath = "data/file.bin" + checksum = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + canonicalURL = "s3://bucket/root/data/file.bin" + ) + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + repoPath: lfsPointer(checksum, 100), + }) + listKey := storageListValidationRequestKey(canonicalURL, 100, "file.bin") + backend := &fakeStorageAnalyticsBackend{ + projectRecords: []gintegrationsyfon.ProjectRecord{{ + ObjectID: "object-id", + Name: "file.bin", + Checksum: checksum, + Organization: "org", + Project: "project", + Size: 100, + AccessURLs: []string{canonicalURL}, + }}, + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bucket", + Organization: "org", + ProjectID: "project", + Path: "s3://bucket/root", + }}, + listProbeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + listKey: { + ID: listKey, + ObjectURL: canonicalURL, + Status: "not_found", + ErrorKind: "object_not_found", + }, + }, + } + + chain, err := NewStorageAnalyticsService(backend).BuildStorageChainAuditWithOptions( + context.Background(), + "Bearer token", + "org", + "project", + refName, + "", + mirrorPath, + repo, + hash, + StorageChainAuditOptions{BucketInventoryMode: StorageChainBucketModeValidate}, + ) + if err != nil { + t.Fatalf("build storage chain audit: %v", err) + } + if got := chain.Summary.CountsByKind["syfon_git_no_bucket"]; got != 0 { + t.Fatalf("LIST misses must not become missing-object findings, got summary %+v", chain.Summary) + } + if got := chain.Summary.CountsByKind["probe_error"]; got != 1 { + t.Fatalf("expected a LIST miss to remain a probe error, got summary %+v", chain.Summary) + } + if backend.listProbeCalls != 1 || backend.probeCalls != 0 { + t.Fatalf("expected one exact LIST and no metadata confirmation, got list=%d metadata=%d", backend.listProbeCalls, backend.probeCalls) + } +} diff --git a/internal/git/storage_audit_registration.go b/internal/git/storage_audit_registration.go new file mode 100644 index 0000000..8d32755 --- /dev/null +++ b/internal/git/storage_audit_registration.go @@ -0,0 +1,316 @@ +package git + +import ( + "context" + "fmt" + "path" + "sort" + "strings" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +type gitOnlyRegistrationCandidate struct { + checksum string + name string + size int64 + accessURLs []string + resultIndexes []int +} + +type gitOnlyBucketCandidate struct { + url string +} + +// RegisterGitOnlySyfonRecords restores only missing Syfon records. It always +// re-reads the Git LFS pointer, confirms the current scoped bucket object, and +// rejects size drift before sending a DRS registration request to Syfon. +func (service *StorageAnalyticsService) RegisterGitOnlySyfonRecords(ctx context.Context, authorizationHeader string, organization string, project string, ref string, mirrorPath string, repo *gogit.Repository, hash plumbing.Hash, request GitOnlySyfonRegistrationRequest) (*GitOnlySyfonRegistrationResponse, error) { + expectedRevision := strings.TrimSpace(request.ExpectedGitRevision) + if expectedRevision == "" { + return nil, fmt.Errorf("git-only Syfon registration requires expected_git_revision") + } + if expectedRevision != hash.String() { + return nil, fmt.Errorf("git-only Syfon registration refused because Git revision changed from %s to %s; refresh the audit and try again", expectedRevision, hash.String()) + } + paths := normalizedGitOnlyRegistrationPaths(request.RepoPaths) + if len(paths) == 0 { + return nil, fmt.Errorf("git-only Syfon registration requires at least one repo path") + } + inventory, err := service.loadStorageChainInventory(ctx, ref, "", mirrorPath, repo, hash) + if err != nil { + return nil, fmt.Errorf("load Git inventory for Syfon registration: %w", err) + } + byPath := make(map[string]RepoInventoryFile, len(inventory)) + for _, item := range inventory { + byPath[normalizeRepoSubpath(item.RepoPath)] = item + } + results := make([]GitOnlySyfonRegistrationResult, len(paths)) + checksums := make([]string, 0, len(paths)) + for index, repoPath := range paths { + results[index].NormalizedPath = repoPath + item, ok := byPath[repoPath] + if !ok { + results[index].Status = "skipped" + results[index].Reason = "path is no longer a Git LFS pointer at the audited revision" + continue + } + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" || item.Size < 0 { + results[index].Status = "skipped" + results[index].Reason = "Git LFS pointer does not contain a valid SHA-256 and size" + continue + } + results[index].Checksum = checksum + results[index].GitSizeBytes = item.Size + checksums = append(checksums, checksum) + } + existing, err := service.storage.BulkGetProjectRecordsByChecksum(ctx, authorizationHeader, organization, project, uniqueStrings(checksums)) + if err != nil { + return nil, fmt.Errorf("recheck Syfon records before registration: %w", err) + } + scopes, err := service.loadProjectChainScopeMappings(ctx, authorizationHeader, organization, project) + if err != nil { + return nil, fmt.Errorf("load project storage scopes before registration: %w", err) + } + bucketObjects, bucketObjectsByURL, err := service.loadCachedProjectBucketValidationInventory(ctx, authorizationHeader, organization, project, "") + if err != nil { + return nil, fmt.Errorf("load project bucket inventory before registration: %w", err) + } + probes := make([]gintegrationsyfon.BulkStorageProbeItem, 0, len(results)) + probeResultIndexes := make(map[string]int, len(results)) + for index := range results { + result := &results[index] + if result.Status != "" { + continue + } + if len(existing[result.Checksum]) > 0 { + result.Status = "skipped" + result.Reason = "a Syfon record for this SHA-256 already exists" + continue + } + candidates := gitOnlyBucketCandidates(result.NormalizedPath, result.Checksum, scopes, organization, project, bucketObjects, bucketObjectsByURL) + if len(candidates) == 0 { + result.Status = "skipped" + result.Reason = "no bucket object matched the audited Git path or SHA-256 in the configured project mappings" + continue + } + size := result.GitSizeBytes + for candidateIndex, candidate := range candidates { + probeID := fmt.Sprintf("git-only-register-%d-%d", index, candidateIndex) + probes = append(probes, gintegrationsyfon.BulkStorageProbeItem{ + ID: probeID, + ObjectURL: candidate.url, + ExpectedSizeBytes: &size, + }) + probeResultIndexes[probeID] = index + } + } + if len(probes) > 0 { + observed, err := service.storage.BulkProbeStorageObjects(ctx, authorizationHeader, probes) + if err != nil { + return nil, fmt.Errorf("recheck bucket objects before registration: %w", err) + } + byProbeID := make(map[string]gintegrationsyfon.BulkStorageProbeResult, len(observed)) + for _, probe := range observed { + byProbeID[strings.TrimSpace(probe.ID)] = probe + } + for _, item := range probes { + index := probeResultIndexes[item.ID] + result := &results[index] + if result.Status == "eligible" { + continue + } + probe, ok := byProbeID[item.ID] + if !ok { + result.Status = "skipped" + result.Reason = "bucket verification did not return a result" + continue + } + result.BucketObjectURL = canonicalStorageURL(probe.Bucket, probe.Key, item.ObjectURL) + if result.BucketObjectURL == "" { + result.BucketObjectURL = item.ObjectURL + } + result.BucketSizeBytes = derefInt64(probe.SizeBytes) + if !probe.Exists || strings.TrimSpace(probe.Status) != "present" { + continue + } + if probe.SizeBytes == nil || *probe.SizeBytes != result.GitSizeBytes { + continue + } + result.Status = "eligible" + result.Reason = "bucket object matched the audited Git path or SHA-256 and passed size verification" + } + for index := range results { + if results[index].Status == "" { + results[index].Status = "skipped" + results[index].Reason = fmt.Sprintf("Git LFS size is %d B but no matched bucket object is present at that size", results[index].GitSizeBytes) + } + } + } + candidates := buildGitOnlyRegistrationCandidates(results) + if len(candidates) > 0 { + registrations := make([]gintegrationsyfon.ProjectObjectRegistration, 0, len(candidates)) + controlledAccess := []string{"/organization/" + strings.TrimSpace(organization) + "/project/" + strings.TrimSpace(project)} + for _, candidate := range candidates { + registrations = append(registrations, gintegrationsyfon.ProjectObjectRegistration{ + Name: candidate.name, + Checksum: candidate.checksum, + Size: candidate.size, + ControlledAccess: controlledAccess, + AccessURLs: candidate.accessURLs, + }) + } + registrationResults, err := service.storage.RegisterProjectObjects(ctx, authorizationHeader, registrations) + if err != nil { + return nil, fmt.Errorf("register missing Syfon records: %w", err) + } + for candidateIndex, candidate := range candidates { + objectID := registrationResults[candidateIndex].ObjectID + for _, resultIndex := range candidate.resultIndexes { + results[resultIndex].Status = "created" + results[resultIndex].ObjectID = objectID + results[resultIndex].Reason = "Syfon record created after Git pointer and bucket object verification" + } + } + service.evictProjectJoinCache(organization, project) + service.evictProjectAuditRecordCache(organization, project) + } + return &GitOnlySyfonRegistrationResponse{GitRevision: hash.String(), Results: results}, nil +} + +func gitOnlyBucketCandidates(repoPath string, checksum string, scopes []domain.StorageBucketScope, organization string, project string, bucketObjects []gintegrationsyfon.ProjectBucketObject, bucketObjectsByURL map[string]gintegrationsyfon.ProjectBucketObject) []gitOnlyBucketCandidate { + pathURLs := projectScopeRepoPathObjectURLs([]string{repoPath}, scopes, organization, project) + checksumURLs := projectScopeRepoPathObjectURLs([]string{checksum}, scopes, organization, project) + auditCandidates := make([]gitOnlyBucketCandidate, 0) + seen := make(map[string]struct{}) + addAuditCandidate := func(objectURL string) { + canonical := canonicalStorageURL("", "", objectURL) + if canonical == "" { + return + } + if _, ok := bucketObjectsByURL[canonical]; !ok { + return + } + if _, ok := seen[canonical]; ok { + return + } + seen[canonical] = struct{}{} + auditCandidates = append(auditCandidates, gitOnlyBucketCandidate{url: canonical}) + } + for _, objectURL := range pathURLs { + addAuditCandidate(objectURL) + } + for _, objectURL := range checksumURLs { + addAuditCandidate(objectURL) + } + for _, object := range bucketObjects { + objectURL := canonicalStorageURL(object.Bucket, object.Key, object.ObjectURL) + if objectURL == "" || normalizeAnalyticsChecksum(object.MetaSHA256) != checksum || !objectBelongsToProjectScope(objectURL, scopes, organization, project) { + continue + } + addAuditCandidate(objectURL) + } + if len(auditCandidates) > 0 { + return auditCandidates + } + + // Inventory may be unavailable for a newly uploaded object. Keep the + // recheck conservative by probing both supported storage layouts rather + // than assuming only the Git-path layout. + generated := make([]gitOnlyBucketCandidate, 0, len(pathURLs)+len(checksumURLs)) + for _, objectURL := range append(pathURLs, checksumURLs...) { + canonical := canonicalStorageURL("", "", objectURL) + if canonical == "" { + continue + } + if _, ok := seen[canonical]; ok { + continue + } + seen[canonical] = struct{}{} + generated = append(generated, gitOnlyBucketCandidate{url: canonical}) + } + return generated +} + +func objectBelongsToProjectScope(objectURL string, scopes []domain.StorageBucketScope, organization string, project string) bool { + bucket, _, ok := parseStorageURL(objectURL) + if !ok { + return false + } + for _, scope := range scopes { + if storageScopeApplies(scope, organization, project) && strings.EqualFold(strings.TrimSpace(scope.Bucket), bucket) { + return true + } + } + return false +} + +func normalizedGitOnlyRegistrationPaths(paths []string) []string { + out := make([]string, 0, len(paths)) + seen := make(map[string]struct{}, len(paths)) + for _, repoPath := range paths { + normalized := normalizeRepoSubpath(repoPath) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + sort.Strings(out) + return out +} + +func gitOnlyRegistrationScopeReason(urls []string, scopes []domain.StorageBucketScope, organization string, project string) string { + if len(urls) == 0 { + return fmt.Sprintf("no configured bucket mapping applies to project %s/%s", organization, project) + } + return fmt.Sprintf("multiple configured bucket mappings apply (%d); choose a single storage mapping before creating a Syfon record", len(urls)) +} + +func buildGitOnlyRegistrationCandidates(results []GitOnlySyfonRegistrationResult) []gitOnlyRegistrationCandidate { + byChecksum := make(map[string]*gitOnlyRegistrationCandidate) + for index := range results { + result := &results[index] + if result.Status != "eligible" { + continue + } + candidate := byChecksum[result.Checksum] + if candidate == nil { + candidate = &gitOnlyRegistrationCandidate{ + checksum: result.Checksum, + name: path.Base(result.NormalizedPath), + size: result.GitSizeBytes, + } + byChecksum[result.Checksum] = candidate + } + if candidate.size != result.GitSizeBytes { + result.Status = "skipped" + result.Reason = "matching Git SHA-256 has inconsistent pointer sizes" + continue + } + candidate.accessURLs = append(candidate.accessURLs, result.BucketObjectURL) + candidate.resultIndexes = append(candidate.resultIndexes, index) + } + checksums := make([]string, 0, len(byChecksum)) + for checksum := range byChecksum { + checksums = append(checksums, checksum) + } + sort.Strings(checksums) + out := make([]gitOnlyRegistrationCandidate, 0, len(checksums)) + for _, checksum := range checksums { + candidate := byChecksum[checksum] + candidate.accessURLs = uniqueStrings(candidate.accessURLs) + if len(candidate.resultIndexes) == 0 { + continue + } + out = append(out, *candidate) + } + return out +} diff --git a/internal/git/storage_audit_registration_test.go b/internal/git/storage_audit_registration_test.go new file mode 100644 index 0000000..dc126a4 --- /dev/null +++ b/internal/git/storage_audit_registration_test.go @@ -0,0 +1,129 @@ +package git + +import ( + "context" + "testing" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" +) + +func TestRegisterGitOnlySyfonRecordsUsesScopedBucketEvidence(t *testing.T) { + checksum := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "OHSU/slide.ome.tiff": lfsPointer(checksum, 100), + }) + objectURL := "s3://bforepc/bforepc-prod/OHSU/slide.ome.tiff" + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: objectURL, + Bucket: "bforepc", + Key: "bforepc-prod/OHSU/slide.ome.tiff", + SizeBytes: 100, + }}, + } + service := NewStorageAnalyticsService(backend) + + audit, err := service.BuildStorageChainAuditWithOptions(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, "", mirrorPath, repo, hash, StorageChainAuditOptions{}) + if err != nil { + t.Fatalf("build audit: %v", err) + } + finding := assertHasChainFinding(t, audit.Findings, "git_only_no_syfon", "OHSU/slide.ome.tiff") + if finding.BucketObjectURL != objectURL || finding.BucketSizeBytes != 100 || finding.Evidence == nil || finding.Evidence.BucketEvaluation != "present" { + t.Fatalf("expected audit to expose mapped bucket evidence, got %+v", finding) + } + + response, err := service.RegisterGitOnlySyfonRecords(context.Background(), "Bearer token", "HTAN_INT", "BForePC", refName, mirrorPath, repo, hash, GitOnlySyfonRegistrationRequest{ + ExpectedGitRevision: hash.String(), + RepoPaths: []string{"OHSU/slide.ome.tiff"}, + }) + if err != nil { + t.Fatalf("register missing Syfon record: %v", err) + } + if len(response.Results) != 1 || response.Results[0].Status != "created" || response.Results[0].BucketObjectURL != objectURL { + t.Fatalf("unexpected registration response: %+v", response) + } + if len(backend.registeredObjects) != 1 { + t.Fatalf("expected one Syfon registration, got %+v", backend.registeredObjects) + } + registered := backend.registeredObjects[0] + if registered.Checksum != checksum || registered.Size != 100 || registered.Name != "slide.ome.tiff" { + t.Fatalf("unexpected Syfon candidate: %+v", registered) + } + if len(registered.AccessURLs) != 1 || registered.AccessURLs[0] != objectURL { + t.Fatalf("expected canonical scoped access URL, got %+v", registered.AccessURLs) + } +} + +func TestRegisterGitOnlySyfonRecordsRefusesBucketSizeMismatch(t *testing.T) { + checksum := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.bin": lfsPointer(checksum, 100), + }) + objectURL := "s3://bucket/root/data/a.bin" + actualSize := int64(99) + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{{Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket/root"}}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: objectURL, + Bucket: "bucket", + Key: "root/data/a.bin", + Path: "data/a.bin", + SizeBytes: 99, + }}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + "git-only-register-0-0": {ID: "git-only-register-0-0", ObjectURL: objectURL, Exists: true, Status: "present", SizeBytes: &actualSize}, + }, + } + service := NewStorageAnalyticsService(backend) + response, err := service.RegisterGitOnlySyfonRecords(context.Background(), "Bearer token", "org", "proj", refName, mirrorPath, repo, hash, GitOnlySyfonRegistrationRequest{ + ExpectedGitRevision: hash.String(), + RepoPaths: []string{"data/a.bin"}, + }) + if err != nil { + t.Fatalf("register missing Syfon record: %v", err) + } + if len(backend.registeredObjects) != 0 || len(response.Results) != 1 || response.Results[0].Status != "skipped" { + t.Fatalf("expected size mismatch to block registration, got response=%+v registrations=%+v", response, backend.registeredObjects) + } +} + +func TestRegisterGitOnlySyfonRecordsFindsChecksumKeyedBucketObjectFromInventory(t *testing.T) { + checksum := "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + repo, mirrorPath, refName, hash := buildAnalyticsMirror(t, map[string]string{ + "data/a.bin": lfsPointer(checksum, 100), + }) + objectURL := "s3://bucket/root/" + checksum + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{{Bucket: "bucket", Organization: "org", ProjectID: "proj", Path: "s3://bucket/root"}}, + bucketObjects: []gintegrationsyfon.ProjectBucketObject{{ + ObjectURL: objectURL, + Bucket: "bucket", + Key: "root/" + checksum, + Path: checksum, + SizeBytes: 100, + MetaSHA256: checksum, + }}, + } + service := NewStorageAnalyticsService(backend) + + response, err := service.RegisterGitOnlySyfonRecords(context.Background(), "Bearer token", "org", "proj", refName, mirrorPath, repo, hash, GitOnlySyfonRegistrationRequest{ + ExpectedGitRevision: hash.String(), + RepoPaths: []string{"data/a.bin"}, + }) + if err != nil { + t.Fatalf("register missing Syfon record: %v", err) + } + if len(response.Results) != 1 || response.Results[0].Status != "created" || response.Results[0].BucketObjectURL != objectURL { + t.Fatalf("expected checksum-keyed inventory object to be registered, got %+v", response) + } + if len(backend.probeItems) != 1 || backend.probeItems[0].ObjectURL != objectURL { + t.Fatalf("expected registration to probe the inventory-resolved checksum object, got %+v", backend.probeItems) + } +} diff --git a/internal/git/storage_audit_revalidation.go b/internal/git/storage_audit_revalidation.go new file mode 100644 index 0000000..caf39aa --- /dev/null +++ b/internal/git/storage_audit_revalidation.go @@ -0,0 +1,147 @@ +package git + +import ( + "context" + "fmt" + "strings" + + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" +) + +const ( + cleanupExpectationPresent = "present" + cleanupExpectationMissing = "missing" +) + +type storageCleanupVerification struct { + FindingKind string + NormalizedPath string + ObjectURL string + Expectation string +} + +func appendStorageCleanupVerification(plan *storageCleanupApplyPlan, finding GitStorageCleanupApplyFinding, action string) error { + if plan == nil || action == storageActionInspectEvidence { + return nil + } + expectation := cleanupVerificationExpectation(finding.Kind, action) + if expectation == "" { + return nil + } + objectURLs := storageApplyFindingBucketObjectURLs(finding) + if len(objectURLs) == 0 { + return fmt.Errorf("cleanup finding %q at %q is missing bucket object URLs required for exact revalidation", finding.Kind, finding.NormalizedPath) + } + for _, objectURL := range objectURLs { + plan.Verifications = append(plan.Verifications, storageCleanupVerification{ + FindingKind: strings.TrimSpace(finding.Kind), + NormalizedPath: strings.TrimSpace(finding.NormalizedPath), + ObjectURL: strings.TrimSpace(objectURL), + Expectation: expectation, + }) + } + return nil +} + +func cleanupVerificationExpectation(kind string, action string) string { + switch action { + case storageActionDeleteBucketObject, storageActionDeleteBoth: + return cleanupExpectationPresent + case storageActionDeleteSyfonRecord: + switch strings.TrimSpace(kind) { + case "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object": + return cleanupExpectationMissing + } + } + return "" +} + +func (service *StorageAnalyticsService) verifyStorageCleanupApplyPlan(ctx context.Context, authorizationHeader string, plan *storageCleanupApplyPlan) error { + if plan == nil { + return nil + } + verifications := uniqueStorageCleanupVerifications(plan.Verifications) + if len(verifications) == 0 { + return nil + } + items := make([]gintegrationsyfon.BulkStorageProbeItem, 0, len(verifications)) + for index, verification := range verifications { + items = append(items, gintegrationsyfon.BulkStorageProbeItem{ + ID: fmt.Sprintf("cleanup-revalidate-%d", index), + ObjectURL: verification.ObjectURL, + }) + } + results, err := service.storage.BulkProbeStorageObjects(ctx, authorizationHeader, items) + if err != nil { + return fmt.Errorf("revalidate cleanup storage evidence: %w", err) + } + if len(results) != len(items) { + return fmt.Errorf("revalidate cleanup storage evidence: expected %d results, received %d", len(items), len(results)) + } + for index, result := range results { + if cleanupObjectVanishedAfterAudit(verifications[index], result) { + plan.DeleteBucketObjects = differenceStrings(plan.DeleteBucketObjects, []string{verifications[index].ObjectURL}) + plan.SkippedPaths = append(plan.SkippedPaths, verifications[index].NormalizedPath) + continue + } + if err := validateStorageCleanupResult(verifications[index], result); err != nil { + return err + } + } + return nil +} + +func cleanupObjectVanishedAfterAudit(verification storageCleanupVerification, result gintegrationsyfon.BulkStorageProbeResult) bool { + if verification.Expectation != cleanupExpectationPresent { + return false + } + if strings.TrimSpace(result.Status) != "not_found" || strings.TrimSpace(result.ErrorKind) != "object_not_found" { + return false + } + switch strings.TrimSpace(verification.FindingKind) { + case "bucket_only_object", "bucket_syfon_no_git": + return true + default: + return false + } +} + +func uniqueStorageCleanupVerifications(input []storageCleanupVerification) []storageCleanupVerification { + seen := make(map[string]struct{}, len(input)) + out := make([]storageCleanupVerification, 0, len(input)) + for _, verification := range input { + key := verification.Expectation + "\x00" + verification.ObjectURL + if verification.ObjectURL == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, verification) + } + return out +} + +func validateStorageCleanupResult(verification storageCleanupVerification, result gintegrationsyfon.BulkStorageProbeResult) error { + status := strings.TrimSpace(result.Status) + switch verification.Expectation { + case cleanupExpectationPresent: + if status == "present" && result.Exists { + return nil + } + case cleanupExpectationMissing: + if status == "not_found" && strings.TrimSpace(result.ErrorKind) == "object_not_found" { + return nil + } + } + return fmt.Errorf( + "revalidate cleanup finding %q for %q: expected %s, got status=%q error_kind=%q error=%q", + verification.FindingKind, + verification.ObjectURL, + verification.Expectation, + status, + strings.TrimSpace(result.ErrorKind), + strings.TrimSpace(result.Error), + ) +} diff --git a/internal/git/storage_audit_revalidation_test.go b/internal/git/storage_audit_revalidation_test.go new file mode 100644 index 0000000..c49cc9e --- /dev/null +++ b/internal/git/storage_audit_revalidation_test.go @@ -0,0 +1,191 @@ +package git + +import ( + "context" + "strings" + "testing" + + "github.com/calypr/gecko/internal/git/domain" + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" +) + +func TestApplyStorageCleanupRejectsStaleMissingObjectEvidence(t *testing.T) { + finding := applyFinding( + "syfon_git_no_bucket", + "data/file.bin", + []string{"object-id"}, + []string{"s3://bucket/data/file.bin"}, + ) + backend := &fakeStorageAnalyticsBackend{} + service := NewStorageAnalyticsService(backend) + + _, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "project", + nil, + nil, + []GitStorageCleanupApplyFinding{finding}, + false, + false, + false, + false, + false, + ) + if err == nil || !strings.Contains(err.Error(), "expected missing, got status=\"present\"") { + t.Fatalf("expected stale missing evidence to abort cleanup, got %v", err) + } + if len(backend.deletedIDs) != 0 || len(backend.deletedBucketObjects) != 0 { + t.Fatalf("revalidation failure must not mutate storage, deleted IDs=%v bucket objects=%v", backend.deletedIDs, backend.deletedBucketObjects) + } +} + +func TestApplyStorageCleanupAcceptsFreshExactMissingEvidence(t *testing.T) { + finding := applyFinding( + "syfon_git_no_bucket", + "data/file.bin", + []string{"object-id"}, + []string{"s3://bucket/data/file.bin"}, + ) + backend := &fakeStorageAnalyticsBackend{ + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + "cleanup-revalidate-0": { + ID: "cleanup-revalidate-0", + Status: "not_found", + ErrorKind: "object_not_found", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + result, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "project", + nil, + nil, + []GitStorageCleanupApplyFinding{finding}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply cleanup with fresh exact evidence: %v", err) + } + if len(result.DeletedRecordIDs) != 1 || result.DeletedRecordIDs[0] != "object-id" { + t.Fatalf("expected exact-missing record deletion, got %+v", result) + } +} + +func TestApplyStorageCleanupCanonicalizesScopedBucketURLsBeforeRevalidation(t *testing.T) { + rawURL := "s3://bforepc-prod/02bd1708-9071-5950-aed4-38a7f3090900/file.bin" + canonicalURL := "s3://bforepc/bforepc-prod/02bd1708-9071-5950-aed4-38a7f3090900/file.bin" + finding := applyFinding( + "bucket_syfon_no_git", + rawURL, + []string{"object-id"}, + []string{rawURL}, + ) + backend := &fakeStorageAnalyticsBackend{ + projectScopes: []domain.StorageBucketScope{{ + Bucket: "bforepc", + Organization: "HTAN_INT", + ProjectID: "BForePC", + Path: "s3://bforepc/bforepc-prod", + }}, + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + "cleanup-revalidate-0": { + ID: "cleanup-revalidate-0", + ObjectURL: canonicalURL, + Bucket: "bforepc", + Key: "bforepc-prod/02bd1708-9071-5950-aed4-38a7f3090900/file.bin", + Exists: true, + Status: "present", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + result, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "HTAN_INT", + "BForePC", + []string{rawURL}, + []GitStorageCleanupApplyAction{{ + Kind: finding.Kind, + NormalizedPath: finding.NormalizedPath, + Action: storageActionDeleteBoth, + }}, + []GitStorageCleanupApplyFinding{finding}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply canonicalized cleanup: %v", err) + } + if len(backend.probeItems) != 1 || backend.probeItems[0].ObjectURL != canonicalURL { + t.Fatalf("expected canonical revalidation target %q, got %+v", canonicalURL, backend.probeItems) + } + if len(backend.deletedBucketObjects) != 1 || backend.deletedBucketObjects[0] != canonicalURL { + t.Fatalf("expected canonical bucket delete target %q, got %+v", canonicalURL, backend.deletedBucketObjects) + } + if len(result.DeletedRecordIDs) != 1 || result.DeletedRecordIDs[0] != "object-id" { + t.Fatalf("expected Syfon record deletion, got %+v", result) + } +} + +func TestApplyStorageCleanupContinuesWhenBucketOrphanVanishesAfterAudit(t *testing.T) { + missingURL := "s3://bucket/orphan-gone" + presentURL := "s3://bucket/orphan-present" + missingFinding := applyFinding("bucket_syfon_no_git", missingURL, []string{"object-gone"}, []string{missingURL}) + presentFinding := applyFinding("bucket_syfon_no_git", presentURL, []string{"object-present"}, []string{presentURL}) + backend := &fakeStorageAnalyticsBackend{ + probeResults: map[string]gintegrationsyfon.BulkStorageProbeResult{ + "cleanup-revalidate-0": { + ID: "cleanup-revalidate-0", + ObjectURL: missingURL, + Status: "not_found", + ErrorKind: "object_not_found", + }, + "cleanup-revalidate-1": { + ID: "cleanup-revalidate-1", + ObjectURL: presentURL, + Exists: true, + Status: "present", + }, + }, + } + service := NewStorageAnalyticsService(backend) + + result, err := service.ApplyStorageCleanup( + context.Background(), + "Bearer token", + "org", + "project", + []string{missingURL, presentURL}, + []GitStorageCleanupApplyAction{ + {Kind: missingFinding.Kind, NormalizedPath: missingFinding.NormalizedPath, Action: storageActionDeleteBoth}, + {Kind: presentFinding.Kind, NormalizedPath: presentFinding.NormalizedPath, Action: storageActionDeleteBoth}, + }, + []GitStorageCleanupApplyFinding{missingFinding, presentFinding}, + false, + false, + false, + false, + false, + ) + if err != nil { + t.Fatalf("apply cleanup after one bucket object vanished: %v", err) + } + assertStringSet(t, "deleted Syfon records", result.DeletedRecordIDs, []string{"object-gone", "object-present"}) + assertStringSet(t, "deleted bucket objects", result.DeletedBucketObjectURLs, []string{presentURL}) + assertStringSet(t, "skipped paths", result.SkippedPaths, []string{missingURL}) +} diff --git a/internal/git/storage_chain_audit_cache.go b/internal/git/storage_chain_audit_cache.go new file mode 100644 index 0000000..5ba56d6 --- /dev/null +++ b/internal/git/storage_chain_audit_cache.go @@ -0,0 +1,526 @@ +package git + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "sync" + "time" + + gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + "github.com/redis/go-redis/v9" +) + +const ( + defaultStorageChainAuditCacheTTL = 5 * time.Minute + // projectBucketInventoryRefreshInterval controls when Gecko asks Syfon for + // a newer inventory. The cache retains the last successful result longer so + // an intermittent provider failure does not turn into a user-facing 502. + projectBucketInventoryRefreshInterval = 10 * time.Minute + projectBucketInventoryStaleTTL = 24 * time.Hour + storageChainAuditCacheKeyPrefix = "gecko:storage_chain_audit:v2:" + storageExactJoinCacheKeyPrefix = "gecko:storage_exact_join:v1:" + projectBucketInventoryKeyPrefix = "gecko:project_bucket_inventory:v1:" +) + +type storageChainAuditResponseCache interface { + Get(ctx context.Context, key string) (cachedStorageChainAuditResponse, bool, error) + Set(ctx context.Context, key string, value cachedStorageChainAuditResponse, ttl time.Duration) error + Source() string +} + +type cachedStorageChainAuditResponse struct { + CachedAt time.Time `json:"cached_at"` + RefreshDurationMillis int64 `json:"refresh_duration_ms,omitempty"` + Response GitStorageChainAuditResponse `json:"response"` +} + +type inflightStorageChainAuditRefresh struct { + done chan struct{} + value cachedStorageChainAuditResponse + err error +} + +// coalesceStorageChainAuditRefresh ensures a root audit has one authoritative +// refresh result per cache key. The UI can issue overlapping forced refreshes; +// without this guard, a partial bucket LIST from the later request can replace +// the completed response from the earlier request. +func (service *StorageAnalyticsService) coalesceStorageChainAuditRefresh(ctx context.Context, key string, refresh func() (cachedStorageChainAuditResponse, error)) (cachedStorageChainAuditResponse, bool, error) { + service.chainAuditRefreshMu.Lock() + if work := service.chainAuditRefreshWork[key]; work != nil { + service.chainAuditRefreshMu.Unlock() + select { + case <-ctx.Done(): + return cachedStorageChainAuditResponse{}, true, ctx.Err() + case <-work.done: + return cloneCachedStorageChainAuditResponse(work.value), true, work.err + } + } + work := &inflightStorageChainAuditRefresh{done: make(chan struct{})} + service.chainAuditRefreshWork[key] = work + service.chainAuditRefreshMu.Unlock() + + value, err := refresh() + service.chainAuditRefreshMu.Lock() + work.value = cloneCachedStorageChainAuditResponse(value) + work.err = err + delete(service.chainAuditRefreshWork, key) + close(work.done) + service.chainAuditRefreshMu.Unlock() + return value, false, err +} + +type storageExactProjectJoinCache interface { + Get(ctx context.Context, key string) (cachedExactProjectJoinState, bool, error) + Set(ctx context.Context, key string, value cachedExactProjectJoinState, ttl time.Duration) error + Source() string +} + +type cachedExactProjectJoinState struct { + CachedAt time.Time `json:"cached_at"` + RecordsByChecksum map[string][]projectRecordState `json:"records_by_checksum"` + UsageByObjectID map[string]gintegrationsyfon.FileUsage `json:"usage_by_object_id"` +} + +type projectBucketInventoryCache interface { + Get(ctx context.Context, key string) (cachedProjectBucketInventory, bool, error) + Set(ctx context.Context, key string, value cachedProjectBucketInventory, ttl time.Duration) error + Source() string +} + +type cachedProjectBucketInventory struct { + CachedAt time.Time `json:"cached_at"` + Objects []gintegrationsyfon.ProjectBucketObject `json:"objects"` +} + +type memoryStorageChainAuditResponseCache struct { + mu sync.RWMutex + entries map[string]memoryStorageChainAuditResponseEntry +} + +type memoryStorageChainAuditResponseEntry struct { + value cachedStorageChainAuditResponse + expiresAt time.Time +} + +func newMemoryStorageChainAuditResponseCache() *memoryStorageChainAuditResponseCache { + return &memoryStorageChainAuditResponseCache{entries: map[string]memoryStorageChainAuditResponseEntry{}} +} + +func (cache *memoryStorageChainAuditResponseCache) Get(_ context.Context, key string) (cachedStorageChainAuditResponse, bool, error) { + cache.mu.RLock() + entry, ok := cache.entries[key] + cache.mu.RUnlock() + if !ok || time.Now().After(entry.expiresAt) { + if ok { + cache.mu.Lock() + delete(cache.entries, key) + cache.mu.Unlock() + } + return cachedStorageChainAuditResponse{}, false, nil + } + return cloneCachedStorageChainAuditResponse(entry.value), true, nil +} + +func (cache *memoryStorageChainAuditResponseCache) Set(_ context.Context, key string, value cachedStorageChainAuditResponse, ttl time.Duration) error { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.entries[key] = memoryStorageChainAuditResponseEntry{ + value: cloneCachedStorageChainAuditResponse(value), + expiresAt: time.Now().Add(ttl), + } + return nil +} + +func (cache *memoryStorageChainAuditResponseCache) Source() string { + return "memory" +} + +type memoryStorageExactProjectJoinCache struct { + mu sync.RWMutex + entries map[string]memoryStorageExactProjectJoinEntry +} + +type memoryProjectBucketInventoryCache struct { + mu sync.RWMutex + entries map[string]memoryProjectBucketInventoryEntry +} + +type memoryProjectBucketInventoryEntry struct { + value cachedProjectBucketInventory + expiresAt time.Time +} + +func newMemoryProjectBucketInventoryCache() *memoryProjectBucketInventoryCache { + return &memoryProjectBucketInventoryCache{entries: map[string]memoryProjectBucketInventoryEntry{}} +} + +func (cache *memoryProjectBucketInventoryCache) Get(_ context.Context, key string) (cachedProjectBucketInventory, bool, error) { + cache.mu.RLock() + entry, ok := cache.entries[key] + cache.mu.RUnlock() + if !ok || time.Now().After(entry.expiresAt) { + if ok { + cache.mu.Lock() + delete(cache.entries, key) + cache.mu.Unlock() + } + return cachedProjectBucketInventory{}, false, nil + } + return cloneCachedProjectBucketInventory(entry.value), true, nil +} + +func (cache *memoryProjectBucketInventoryCache) Set(_ context.Context, key string, value cachedProjectBucketInventory, ttl time.Duration) error { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.entries[key] = memoryProjectBucketInventoryEntry{value: cloneCachedProjectBucketInventory(value), expiresAt: time.Now().Add(ttl)} + return nil +} + +func (cache *memoryProjectBucketInventoryCache) Source() string { return "memory" } + +type memoryStorageExactProjectJoinEntry struct { + value cachedExactProjectJoinState + expiresAt time.Time +} + +func newMemoryStorageExactProjectJoinCache() *memoryStorageExactProjectJoinCache { + return &memoryStorageExactProjectJoinCache{entries: map[string]memoryStorageExactProjectJoinEntry{}} +} + +func (cache *memoryStorageExactProjectJoinCache) Get(_ context.Context, key string) (cachedExactProjectJoinState, bool, error) { + cache.mu.RLock() + entry, ok := cache.entries[key] + cache.mu.RUnlock() + if !ok || time.Now().After(entry.expiresAt) { + if ok { + cache.mu.Lock() + delete(cache.entries, key) + cache.mu.Unlock() + } + return cachedExactProjectJoinState{}, false, nil + } + return cloneCachedExactProjectJoinState(entry.value), true, nil +} + +func (cache *memoryStorageExactProjectJoinCache) Set(_ context.Context, key string, value cachedExactProjectJoinState, ttl time.Duration) error { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.entries[key] = memoryStorageExactProjectJoinEntry{ + value: cloneCachedExactProjectJoinState(value), + expiresAt: time.Now().Add(ttl), + } + return nil +} + +func (cache *memoryStorageExactProjectJoinCache) Source() string { + return "memory" +} + +type redisStorageChainAuditResponseCache struct { + client *redis.Client +} + +type redisStorageExactProjectJoinCache struct { + client *redis.Client +} + +type redisProjectBucketInventoryCache struct { + client *redis.Client +} + +func newRedisStorageChainAuditResponseCache(redisURL string) (*redisStorageChainAuditResponseCache, error) { + options, err := redis.ParseURL(strings.TrimSpace(redisURL)) + if err != nil { + return nil, err + } + cache := &redisStorageChainAuditResponseCache{client: redis.NewClient(options)} + if err := cache.client.Ping(context.Background()).Err(); err != nil { + _ = cache.client.Close() + return nil, err + } + return cache, nil +} + +func newRedisStorageExactProjectJoinCache(redisURL string) (*redisStorageExactProjectJoinCache, error) { + options, err := redis.ParseURL(strings.TrimSpace(redisURL)) + if err != nil { + return nil, err + } + cache := &redisStorageExactProjectJoinCache{client: redis.NewClient(options)} + if err := cache.client.Ping(context.Background()).Err(); err != nil { + _ = cache.client.Close() + return nil, err + } + return cache, nil +} + +func newRedisProjectBucketInventoryCache(redisURL string) (*redisProjectBucketInventoryCache, error) { + options, err := redis.ParseURL(strings.TrimSpace(redisURL)) + if err != nil { + return nil, err + } + cache := &redisProjectBucketInventoryCache{client: redis.NewClient(options)} + if err := cache.client.Ping(context.Background()).Err(); err != nil { + _ = cache.client.Close() + return nil, err + } + return cache, nil +} + +func (cache *redisStorageChainAuditResponseCache) Get(ctx context.Context, key string) (cachedStorageChainAuditResponse, bool, error) { + raw, err := cache.client.Get(ctx, key).Bytes() + if err == redis.Nil { + return cachedStorageChainAuditResponse{}, false, nil + } + if err != nil { + return cachedStorageChainAuditResponse{}, false, err + } + var value cachedStorageChainAuditResponse + if err := json.Unmarshal(raw, &value); err != nil { + return cachedStorageChainAuditResponse{}, false, err + } + if value.CachedAt.IsZero() { + value.CachedAt = time.Now() + } + return value, true, nil +} + +func (cache *redisStorageChainAuditResponseCache) Set(ctx context.Context, key string, value cachedStorageChainAuditResponse, ttl time.Duration) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + return cache.client.Set(ctx, key, raw, ttl).Err() +} + +func (cache *redisStorageChainAuditResponseCache) Source() string { + return "redis" +} + +func (cache *redisStorageExactProjectJoinCache) Get(ctx context.Context, key string) (cachedExactProjectJoinState, bool, error) { + raw, err := cache.client.Get(ctx, key).Bytes() + if err == redis.Nil { + return cachedExactProjectJoinState{}, false, nil + } + if err != nil { + return cachedExactProjectJoinState{}, false, err + } + var value cachedExactProjectJoinState + if err := json.Unmarshal(raw, &value); err != nil { + return cachedExactProjectJoinState{}, false, err + } + if value.CachedAt.IsZero() { + value.CachedAt = time.Now() + } + return value, true, nil +} + +func (cache *redisStorageExactProjectJoinCache) Set(ctx context.Context, key string, value cachedExactProjectJoinState, ttl time.Duration) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + return cache.client.Set(ctx, key, raw, ttl).Err() +} + +func (cache *redisStorageExactProjectJoinCache) Source() string { + return "redis" +} + +func (cache *redisProjectBucketInventoryCache) Get(ctx context.Context, key string) (cachedProjectBucketInventory, bool, error) { + raw, err := cache.client.Get(ctx, key).Bytes() + if err == redis.Nil { + return cachedProjectBucketInventory{}, false, nil + } + if err != nil { + return cachedProjectBucketInventory{}, false, err + } + var value cachedProjectBucketInventory + if err := json.Unmarshal(raw, &value); err != nil { + return cachedProjectBucketInventory{}, false, err + } + if value.CachedAt.IsZero() { + value.CachedAt = time.Now() + } + return cloneCachedProjectBucketInventory(value), true, nil +} + +func (cache *redisProjectBucketInventoryCache) Set(ctx context.Context, key string, value cachedProjectBucketInventory, ttl time.Duration) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + return cache.client.Set(ctx, key, raw, ttl).Err() +} + +func (cache *redisProjectBucketInventoryCache) Source() string { return "redis" } + +func NewStorageChainAuditResponseCacheFromEnv() storageChainAuditResponseCache { + if !storageChainAuditCacheEnabled() { + return nil + } + redisURL := storageChainAuditRedisURLFromEnv() + if redisURL != "" { + cache, err := newRedisStorageChainAuditResponseCache(redisURL) + if err == nil { + return cache + } + } + return newMemoryStorageChainAuditResponseCache() +} + +func NewStorageExactProjectJoinCacheFromEnv() storageExactProjectJoinCache { + if !storageChainAuditCacheEnabled() { + return nil + } + redisURL := storageChainAuditRedisURLFromEnv() + if redisURL != "" { + cache, err := newRedisStorageExactProjectJoinCache(redisURL) + if err == nil { + return cache + } + } + return newMemoryStorageExactProjectJoinCache() +} + +func NewProjectBucketInventoryCacheFromEnv() projectBucketInventoryCache { + if !storageChainAuditCacheEnabled() { + return nil + } + redisURL := storageChainAuditRedisURLFromEnv() + if redisURL != "" { + cache, err := newRedisProjectBucketInventoryCache(redisURL) + if err == nil { + return cache + } + } + return newMemoryProjectBucketInventoryCache() +} + +func storageChainAuditRedisURLFromEnv() string { + redisURL := strings.TrimSpace(os.Getenv("STORAGE_CHAIN_AUDIT_CACHE_REDIS_URL")) + if redisURL == "" { + redisURL = strings.TrimSpace(os.Getenv("REDIS_URL")) + } + return redisURLWithPassword(redisURL, strings.TrimSpace(os.Getenv("STORAGE_CHAIN_AUDIT_CACHE_REDIS_PASSWORD"))) +} + +func redisURLWithPassword(redisURL string, password string) string { + redisURL = strings.TrimSpace(redisURL) + password = strings.TrimSpace(password) + if redisURL == "" || password == "" || !strings.HasPrefix(redisURL, "redis://") || strings.Contains(redisURL, "@") { + return redisURL + } + return "redis://:" + password + "@" + strings.TrimPrefix(redisURL, "redis://") +} + +func storageChainAuditCacheEnabled() bool { + raw := strings.TrimSpace(os.Getenv("STORAGE_CHAIN_AUDIT_CACHE_ENABLED")) + if raw == "" { + return true + } + enabled, err := strconv.ParseBool(raw) + return err == nil && enabled +} + +func storageChainAuditCacheTTL() time.Duration { + raw := strings.TrimSpace(os.Getenv("STORAGE_CHAIN_AUDIT_CACHE_TTL_SECONDS")) + if raw == "" { + return defaultStorageChainAuditCacheTTL + } + seconds, err := strconv.Atoi(raw) + if err != nil || seconds < 60 { + return defaultStorageChainAuditCacheTTL + } + return time.Duration(seconds) * time.Second +} + +func storageChainAuditResponseCacheKey(organization string, project string, ref string, gitSubpath string, probeMode string, validationMode string, bucketMode string, bucketPathPrefix string, hash string, syfonRevision string) string { + body := fmt.Sprintf( + "org=%s\nproject=%s\nref=%s\nhash=%s\nsyfon_revision=%s\ngit_subpath=%s\nprobe_mode=%s\nvalidation_mode=%s\nbucket_mode=%s\nbucket_path_prefix=%s", + strings.TrimSpace(organization), + strings.TrimSpace(project), + strings.TrimSpace(ref), + strings.TrimSpace(hash), + strings.TrimSpace(syfonRevision), + normalizeRepoSubpath(gitSubpath), + strings.TrimSpace(probeMode), + strings.TrimSpace(validationMode), + strings.TrimSpace(bucketMode), + normalizeRepoSubpath(bucketPathPrefix), + ) + sum := sha256.Sum256([]byte(body)) + return storageChainAuditCacheKeyPrefix + hex.EncodeToString(sum[:]) +} + +func storageExactProjectJoinCacheKey(organization string, project string, hash string) string { + body := fmt.Sprintf( + "org=%s\nproject=%s\nhash=%s", + strings.TrimSpace(organization), + strings.TrimSpace(project), + strings.TrimSpace(hash), + ) + sum := sha256.Sum256([]byte(body)) + return storageExactJoinCacheKeyPrefix + hex.EncodeToString(sum[:]) +} + +func projectBucketInventoryCacheKey(organization string, project string, bucketPathPrefix string) string { + body := fmt.Sprintf("org=%s\nproject=%s\nbucket_path_prefix=%s", strings.TrimSpace(organization), strings.TrimSpace(project), normalizeRepoSubpath(bucketPathPrefix)) + sum := sha256.Sum256([]byte(body)) + return projectBucketInventoryKeyPrefix + hex.EncodeToString(sum[:]) +} + +func cloneCachedStorageChainAuditResponse(value cachedStorageChainAuditResponse) cachedStorageChainAuditResponse { + return cachedStorageChainAuditResponse{ + CachedAt: value.CachedAt, + RefreshDurationMillis: value.RefreshDurationMillis, + Response: cloneStorageChainAuditResponse(value.Response), + } +} + +func cloneCachedProjectBucketInventory(value cachedProjectBucketInventory) cachedProjectBucketInventory { + return cachedProjectBucketInventory{CachedAt: value.CachedAt, Objects: append([]gintegrationsyfon.ProjectBucketObject(nil), value.Objects...)} +} + +func cloneStorageChainAuditResponse(response GitStorageChainAuditResponse) GitStorageChainAuditResponse { + response.Findings = append([]GitStorageChainFinding(nil), response.Findings...) + response.Groups = append([]GitStorageChainIssueGroup(nil), response.Groups...) + response.Summary.CountsByKind = cloneStringIntMap(response.Summary.CountsByKind) + return response +} + +func cloneCachedExactProjectJoinState(value cachedExactProjectJoinState) cachedExactProjectJoinState { + return cachedExactProjectJoinState{ + CachedAt: value.CachedAt, + RecordsByChecksum: cloneRecordStateMap(value.RecordsByChecksum), + UsageByObjectID: cloneFileUsageMap(value.UsageByObjectID), + } +} + +func cloneFileUsageMap(input map[string]gintegrationsyfon.FileUsage) map[string]gintegrationsyfon.FileUsage { + if input == nil { + return nil + } + out := make(map[string]gintegrationsyfon.FileUsage, len(input)) + for key, value := range input { + out[key] = value + } + return out +} + +func cloneStringIntMap(input map[string]int) map[string]int { + if input == nil { + return nil + } + out := make(map[string]int, len(input)) + for key, value := range input { + out[key] = value + } + return out +} diff --git a/internal/git/storage_chain_audit_cache_test.go b/internal/git/storage_chain_audit_cache_test.go new file mode 100644 index 0000000..dc3c950 --- /dev/null +++ b/internal/git/storage_chain_audit_cache_test.go @@ -0,0 +1,28 @@ +package git + +import "testing" + +func TestProjectBucketInventoryCacheRetention(t *testing.T) { + if projectBucketInventoryRefreshInterval.String() != "10m0s" { + t.Fatalf("projectBucketInventoryRefreshInterval = %s, want 10m", projectBucketInventoryRefreshInterval) + } + if projectBucketInventoryStaleTTL.String() != "24h0m0s" { + t.Fatalf("projectBucketInventoryStaleTTL = %s, want 24h", projectBucketInventoryStaleTTL) + } +} + +func TestRedisURLWithPasswordMatchesFenceStyle(t *testing.T) { + got := redisURLWithPassword("redis://authz-cache-service:6379/0", "secret") + want := "redis://:secret@authz-cache-service:6379/0" + if got != want { + t.Fatalf("redisURLWithPassword() = %q, want %q", got, want) + } +} + +func TestRedisURLWithPasswordKeepsExistingCredential(t *testing.T) { + input := "redis://:existing@authz-cache-service:6379/0" + got := redisURLWithPassword(input, "secret") + if got != input { + t.Fatalf("redisURLWithPassword() = %q, want %q", got, input) + } +} diff --git a/internal/git/storage_children.go b/internal/git/storage_children.go new file mode 100644 index 0000000..6ea65da --- /dev/null +++ b/internal/git/storage_children.go @@ -0,0 +1,252 @@ +package git + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + "github.com/go-git/go-git/v5/plumbing" +) + +type storageChildrenCursor struct { + CommitHash string `json:"commit_hash"` + GitSubpath string `json:"git_subpath"` + SortBy string `json:"sort_by"` + SortOrder string `json:"sort_order"` + Offset int `json:"offset"` +} + +type storageChildrenPage struct { + items []storageAggregate + hasMore bool + nextCursor string +} + +func buildStorageChildrenCursor(hash plumbing.Hash, gitSubpath string, sortBy string, sortOrder string, offset int) string { + cursor := storageChildrenCursor{ + CommitHash: hash.String(), + GitSubpath: normalizeRepoSubpath(gitSubpath), + SortBy: strings.TrimSpace(sortBy), + SortOrder: strings.TrimSpace(sortOrder), + Offset: offset, + } + contentBytes, err := json.Marshal(cursor) + if err != nil { + return "" + } + return base64.RawURLEncoding.EncodeToString(contentBytes) +} + +func decodeStorageChildrenCursor(raw string) (storageChildrenCursor, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return storageChildrenCursor{}, nil + } + contentBytes, err := base64.RawURLEncoding.DecodeString(trimmed) + if err != nil { + return storageChildrenCursor{}, fmt.Errorf("decode storage children cursor: %w", err) + } + cursor := storageChildrenCursor{} + if err := json.Unmarshal(contentBytes, &cursor); err != nil { + return storageChildrenCursor{}, fmt.Errorf("decode storage children cursor JSON: %w", err) + } + if cursor.Offset < 0 { + return storageChildrenCursor{}, fmt.Errorf("storage children cursor offset must be non-negative") + } + return cursor, nil +} + +func storageChildrenPageForRequest(items []storageAggregate, hash plumbing.Hash, gitSubpath string, sortBy string, sortOrder string, limit int, rawCursor string) (storageChildrenPage, error) { + if limit <= 0 { + limit = len(items) + } + offset, err := storageChildrenCursorOffset(hash, gitSubpath, sortBy, sortOrder, rawCursor) + if err != nil { + return storageChildrenPage{}, err + } + if offset > len(items) { + offset = len(items) + } + end := offset + limit + if end > len(items) { + end = len(items) + } + pageItems := append([]storageAggregate(nil), items[offset:end]...) + page := storageChildrenPage{ + items: pageItems, + hasMore: end < len(items), + } + if page.hasMore { + page.nextCursor = buildStorageChildrenCursor(hash, gitSubpath, sortBy, sortOrder, end) + } + return page, nil +} + +func storageChildrenCursorOffset(hash plumbing.Hash, gitSubpath string, sortBy string, sortOrder string, rawCursor string) (int, error) { + cursor, err := decodeStorageChildrenCursor(rawCursor) + if err != nil { + return 0, err + } + if strings.TrimSpace(rawCursor) == "" { + return 0, nil + } + expected := storageChildrenCursor{ + CommitHash: hash.String(), + GitSubpath: normalizeRepoSubpath(gitSubpath), + SortBy: strings.TrimSpace(sortBy), + SortOrder: strings.TrimSpace(sortOrder), + } + if cursor.CommitHash != expected.CommitHash || cursor.GitSubpath != expected.GitSubpath || cursor.SortBy != expected.SortBy || cursor.SortOrder != expected.SortOrder { + return 0, fmt.Errorf("storage children cursor does not match current request") + } + return cursor.Offset, nil +} + +func storageChildrenResponseForServingIndex(directory repoAnalyticsDirectoryServingIndex, hash plumbing.Hash, gitSubpath string, sortBy string, sortOrder string, limit int, rawCursor string) (GitStorageChildrenResponse, error) { + order := sortedOrderForStorageChildrenRequest(directory, sortBy, sortOrder) + if limit <= 0 { + limit = len(order) + } + offset, err := storageChildrenCursorOffset(hash, gitSubpath, sortBy, sortOrder, rawCursor) + if err != nil { + return GitStorageChildrenResponse{}, err + } + if offset > len(order) { + offset = len(order) + } + end := offset + limit + if end > len(order) { + end = len(order) + } + items := make([]GitStorageChildResponseItem, 0, end-offset) + for _, childIndex := range order[offset:end] { + child := directory.directory.Children[childIndex] + items = append(items, GitStorageChildResponseItem{ + Name: child.Name, + Path: child.Path, + Type: child.Type, + FileCount: child.FileCount, + TotalBytes: child.TotalBytes, + }) + } + response := GitStorageChildrenResponse{ + Items: items, + HasMore: end < len(order), + } + if response.HasMore { + response.NextCursor = buildStorageChildrenCursor(hash, gitSubpath, sortBy, sortOrder, end) + } + return response, nil +} + +func sortedOrderForStorageChildrenRequest(directory repoAnalyticsDirectoryServingIndex, sortBy string, sortOrder string) []int { + desc := !strings.EqualFold(strings.TrimSpace(sortOrder), "asc") + switch strings.ToLower(strings.TrimSpace(sortBy)) { + case "name": + if desc { + return directory.nameDesc + } + return directory.nameAsc + default: + if desc { + return directory.bytesDesc + } + return directory.bytesAsc + } +} + +func filterInventoryForStorageChildren(inventory []RepoInventoryFile, children []storageAggregate) []RepoInventoryFile { + if len(children) == 0 || len(inventory) == 0 { + return nil + } + selectedFiles := make(map[string]struct{}) + selectedDirectories := make([]string, 0) + for _, child := range children { + childPath := normalizeRepoSubpath(child.path) + if childPath == "" { + continue + } + if child.rowType == "file" { + selectedFiles[childPath] = struct{}{} + continue + } + selectedDirectories = append(selectedDirectories, childPath+"/") + } + filtered := make([]RepoInventoryFile, 0) + for _, item := range inventory { + itemPath := normalizeRepoSubpath(item.RepoPath) + if _, ok := selectedFiles[itemPath]; ok { + filtered = append(filtered, item) + continue + } + for _, prefix := range selectedDirectories { + if strings.HasPrefix(itemPath, prefix) { + filtered = append(filtered, item) + break + } + } + } + return filtered +} + +func (service *StorageAnalyticsService) enrichStorageChildrenPage(ctx context.Context, authorizationHeader string, organization string, project string, gitSubpath string, inventory []RepoInventoryFile, pageItems []storageAggregate) ([]storageAggregate, error) { + checksums := uniqueInventoryChecksums(inventory) + if len(checksums) == 0 { + return pageItems, nil + } + recordsByChecksumRaw, err := service.storage.BulkGetProjectRecordsByChecksum(ctx, authorizationHeader, organization, project, checksums) + if err != nil { + return nil, fmt.Errorf("lookup syfon project records by checksum: %w", err) + } + recordsByChecksum := make(map[string][]projectRecordState, len(recordsByChecksumRaw)) + for _, records := range recordsByChecksumRaw { + for _, record := range records { + normalizedChecksum := normalizeAnalyticsChecksum(record.Checksum) + if normalizedChecksum == "" { + continue + } + record.Checksum = normalizedChecksum + recordsByChecksum[normalizedChecksum] = append(recordsByChecksum[normalizedChecksum], projectRecordState{ + ProjectRecord: record, + }) + } + } + return aggregateImmediateChildren(gitSubpath, inventory, recordsByChecksum, nil, pageItems), nil +} + +func uniqueInventoryChecksums(inventory []RepoInventoryFile) []string { + checksums := make([]string, 0, len(inventory)) + seen := make(map[string]struct{}, len(inventory)) + for _, item := range inventory { + checksum := normalizeAnalyticsChecksum(item.Checksum) + if checksum == "" { + continue + } + if _, ok := seen[checksum]; ok { + continue + } + seen[checksum] = struct{}{} + checksums = append(checksums, checksum) + } + return checksums +} + +func storageChildrenItemsFromAggregates(aggregates []storageAggregate) []GitStorageChildResponseItem { + items := make([]GitStorageChildResponseItem, 0, len(aggregates)) + for _, agg := range aggregates { + items = append(items, GitStorageChildResponseItem{ + Name: agg.name, + Path: agg.path, + Type: agg.rowType, + FileCount: agg.fileCount, + RecordCount: agg.recordCount, + TotalBytes: agg.totalBytes, + DownloadCount: agg.downloadCount, + LastDownloadTime: formatOptionalTime(agg.lastDownload), + LatestUpdateTime: formatOptionalTime(agg.latestUpdate), + }) + } + return items +} diff --git a/internal/git/storage_index.go b/internal/git/storage_index.go new file mode 100644 index 0000000..76ccd3e --- /dev/null +++ b/internal/git/storage_index.go @@ -0,0 +1,437 @@ +package git + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" +) + +const repoAnalyticsIndexSchemaVersion = 1 + +var repoAnalyticsIndexCache = &repoAnalyticsIndexMemoryCache{ + entries: map[string]*repoAnalyticsIndex{}, + inflight: map[string]*inflightRepoAnalyticsIndex{}, +} + +type repoAnalyticsIndexMemoryCache struct { + mu sync.RWMutex + entries map[string]*repoAnalyticsIndex + inflight map[string]*inflightRepoAnalyticsIndex +} + +type inflightRepoAnalyticsIndex struct { + done chan struct{} + index *repoAnalyticsIndex + err error +} + +type repoAnalyticsIndex struct { + sidecar GitRepoAnalyticsIndexSidecar + directoryLookup map[string]GitRepoAnalyticsDirectory + directoryServing map[string]repoAnalyticsDirectoryServingIndex +} + +type repoAnalyticsDirectoryServingIndex struct { + directory GitRepoAnalyticsDirectory + bytesAsc []int + bytesDesc []int + nameAsc []int + nameDesc []int +} + +func repoAnalyticsCacheKey(mirrorPath string, hash plumbing.Hash) string { + return strings.TrimSpace(mirrorPath) + "::" + hash.String() +} + +func (cache *repoAnalyticsIndexMemoryCache) put(mirrorPath string, hash plumbing.Hash, index *repoAnalyticsIndex) { + if index == nil { + return + } + cache.mu.Lock() + defer cache.mu.Unlock() + prefix := strings.TrimSpace(mirrorPath) + "::" + for key := range cache.entries { + if strings.HasPrefix(key, prefix) && key != repoAnalyticsCacheKey(mirrorPath, hash) { + delete(cache.entries, key) + } + } + cache.entries[repoAnalyticsCacheKey(mirrorPath, hash)] = index +} + +func RepoAnalyticsIndexPath(mirrorPath string) string { + return mirrorPath + ".analytics.index.json" +} + +func PersistRepoAnalyticsIndex(ctx context.Context, mirrorPath string, repo *gogit.Repository, ref string, hash plumbing.Hash) error { + index, err := buildRepoAnalyticsIndex(ref, repo, hash) + if err != nil { + return err + } + if err := writeRepoAnalyticsIndexSidecar(mirrorPath, index.sidecar); err != nil { + return err + } + repoAnalyticsIndexCache.put(mirrorPath, hash, index) + _ = ctx + return nil +} + +func loadOrBuildRepoAnalyticsIndex(ctx context.Context, mirrorPath string, ref string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, error) { + return loadOrBuildRepoAnalyticsIndexWithTimings(ctx, mirrorPath, ref, repo, hash, nil) +} + +func loadOrBuildRepoAnalyticsIndexWithTimings(_ context.Context, mirrorPath string, ref string, repo *gogit.Repository, hash plumbing.Hash, timings *StorageFolderTimings) (*repoAnalyticsIndex, error) { + cacheKey := repoAnalyticsCacheKey(mirrorPath, hash) + repoAnalyticsIndexCache.mu.Lock() + if cached := repoAnalyticsIndexCache.entries[cacheKey]; cached != nil { + repoAnalyticsIndexCache.mu.Unlock() + timings.Record("repo_index_memory_cache_hit", 0) + return cached, nil + } + if inflight := repoAnalyticsIndexCache.inflight[cacheKey]; inflight != nil { + repoAnalyticsIndexCache.mu.Unlock() + waitStart := time.Now() + <-inflight.done + timings.Record("repo_index_inflight_wait", time.Since(waitStart)) + return inflight.index, inflight.err + } + inflight := &inflightRepoAnalyticsIndex{done: make(chan struct{})} + repoAnalyticsIndexCache.inflight[cacheKey] = inflight + repoAnalyticsIndexCache.mu.Unlock() + defer func() { + repoAnalyticsIndexCache.mu.Lock() + delete(repoAnalyticsIndexCache.inflight, cacheKey) + close(inflight.done) + repoAnalyticsIndexCache.mu.Unlock() + }() + + sidecarStart := time.Now() + sidecar, err := readRepoAnalyticsIndexSidecar(mirrorPath) + timings.Record("repo_index_sidecar_read_decode", time.Since(sidecarStart)) + if err == nil && sidecar.SchemaVersion == repoAnalyticsIndexSchemaVersion && sidecar.CommitHash == hash.String() { + servingStart := time.Now() + index := repoAnalyticsIndexFromSidecar(sidecar) + timings.Record("repo_index_sorted_order_build", time.Since(servingStart)) + repoAnalyticsIndexCache.put(mirrorPath, hash, index) + inflight.index = index + return index, nil + } + buildStart := time.Now() + index, buildErr := buildRepoAnalyticsIndex(ref, repo, hash) + timings.Record("repo_index_git_tree_rebuild", time.Since(buildStart)) + if buildErr != nil { + inflight.err = buildErr + return nil, buildErr + } + writeStart := time.Now() + if writeErr := writeRepoAnalyticsIndexSidecar(mirrorPath, index.sidecar); writeErr != nil { + timings.Record("repo_index_sidecar_write", time.Since(writeStart)) + inflight.err = writeErr + return nil, writeErr + } + timings.Record("repo_index_sidecar_write", time.Since(writeStart)) + repoAnalyticsIndexCache.put(mirrorPath, hash, index) + inflight.index = index + return index, nil +} + +func readRepoAnalyticsIndexSidecar(mirrorPath string) (GitRepoAnalyticsIndexSidecar, error) { + sidecarPath := RepoAnalyticsIndexPath(mirrorPath) + contentBytes, err := os.ReadFile(sidecarPath) + if err != nil { + return GitRepoAnalyticsIndexSidecar{}, fmt.Errorf("read repo analytics index sidecar: %w", err) + } + sidecar := GitRepoAnalyticsIndexSidecar{} + if err := json.Unmarshal(contentBytes, &sidecar); err != nil { + return GitRepoAnalyticsIndexSidecar{}, fmt.Errorf("decode repo analytics index sidecar: %w", err) + } + return sidecar, nil +} + +func writeRepoAnalyticsIndexSidecar(mirrorPath string, sidecar GitRepoAnalyticsIndexSidecar) error { + sidecarPath := RepoAnalyticsIndexPath(mirrorPath) + if err := os.MkdirAll(filepath.Dir(sidecarPath), 0o755); err != nil { + return fmt.Errorf("create repo analytics sidecar dir: %w", err) + } + contentBytes, err := json.Marshal(sidecar) + if err != nil { + return fmt.Errorf("encode repo analytics index sidecar: %w", err) + } + tempPath := sidecarPath + ".tmp" + if err := os.WriteFile(tempPath, contentBytes, 0o644); err != nil { + return fmt.Errorf("write repo analytics index temp sidecar: %w", err) + } + if err := os.Rename(tempPath, sidecarPath); err != nil { + return fmt.Errorf("move repo analytics index sidecar into place: %w", err) + } + return nil +} + +func repoAnalyticsIndexFromSidecar(sidecar GitRepoAnalyticsIndexSidecar) *repoAnalyticsIndex { + directoryLookup := make(map[string]GitRepoAnalyticsDirectory, len(sidecar.Directories)) + directoryServing := make(map[string]repoAnalyticsDirectoryServingIndex, len(sidecar.Directories)) + for _, directory := range sidecar.Directories { + directoryLookup[directory.Path] = directory + directoryServing[directory.Path] = buildRepoAnalyticsDirectoryServingIndex(directory) + } + return &repoAnalyticsIndex{ + sidecar: sidecar, + directoryLookup: directoryLookup, + directoryServing: directoryServing, + } +} + +func buildRepoAnalyticsDirectoryServingIndex(directory GitRepoAnalyticsDirectory) repoAnalyticsDirectoryServingIndex { + return repoAnalyticsDirectoryServingIndex{ + directory: directory, + bytesAsc: sortedChildIndexes(directory.Children, "bytes", "asc"), + bytesDesc: sortedChildIndexes(directory.Children, "bytes", "desc"), + nameAsc: sortedChildIndexes(directory.Children, "name", "asc"), + nameDesc: sortedChildIndexes(directory.Children, "name", "desc"), + } +} + +func sortedChildIndexes(children []GitRepoAnalyticsChild, sortBy string, sortOrder string) []int { + indexes := make([]int, len(children)) + for index := range children { + indexes[index] = index + } + desc := !strings.EqualFold(strings.TrimSpace(sortOrder), "asc") + switch strings.ToLower(strings.TrimSpace(sortBy)) { + case "name": + sort.Slice(indexes, func(i, j int) bool { + left := strings.ToLower(children[indexes[i]].Name) + right := strings.ToLower(children[indexes[j]].Name) + if desc { + return left > right + } + return left < right + }) + default: + sort.Slice(indexes, func(i, j int) bool { + left := children[indexes[i]] + right := children[indexes[j]] + if left.TotalBytes != right.TotalBytes { + if desc { + return left.TotalBytes > right.TotalBytes + } + return left.TotalBytes < right.TotalBytes + } + if left.Type != right.Type { + return left.Type == "directory" + } + return strings.ToLower(left.Name) < strings.ToLower(right.Name) + }) + } + return indexes +} + +func buildRepoAnalyticsIndex(ref string, repo *gogit.Repository, hash plumbing.Hash) (*repoAnalyticsIndex, error) { + commit, err := repo.CommitObject(hash) + if err != nil { + return nil, fmt.Errorf("load commit for ref %s: %w", ref, err) + } + tree, err := commit.Tree() + if err != nil { + return nil, fmt.Errorf("load git tree for ref %s: %w", ref, err) + } + files := make([]RepoInventoryFile, 0) + knownDirectories := map[string]struct{}{"": {}} + if err := walkGitRepoInventory("", tree, &files, knownDirectories); err != nil { + return nil, err + } + sort.Slice(files, func(i, j int) bool { + return files[i].RepoPath < files[j].RepoPath + }) + directories := buildRepoAnalyticsDirectories(files, knownDirectories) + sidecar := GitRepoAnalyticsIndexSidecar{ + SchemaVersion: repoAnalyticsIndexSchemaVersion, + CommitHash: hash.String(), + RefName: ref, + GeneratedAt: time.Now().UTC(), + Files: files, + Directories: directories, + } + return repoAnalyticsIndexFromSidecar(sidecar), nil +} + +func buildRepoAnalyticsDirectories(files []RepoInventoryFile, knownDirectories map[string]struct{}) []GitRepoAnalyticsDirectory { + type directoryBuilder struct { + GitRepoAnalyticsDirectory + children map[string]*GitRepoAnalyticsChild + } + builders := make(map[string]*directoryBuilder, len(knownDirectories)) + ensureDirectory := func(path string) *directoryBuilder { + builder := builders[path] + if builder != nil { + return builder + } + builder = &directoryBuilder{ + GitRepoAnalyticsDirectory: GitRepoAnalyticsDirectory{Path: path}, + children: map[string]*GitRepoAnalyticsChild{}, + } + builders[path] = builder + return builder + } + for path := range knownDirectories { + ensureDirectory(path) + } + for _, file := range files { + parentPath := "" + parts := strings.Split(file.RepoPath, "/") + for index, part := range parts { + directory := ensureDirectory(parentPath) + directory.FileCount++ + directory.TotalBytes += file.Size + childPath := part + if parentPath != "" { + childPath = parentPath + "/" + part + } + childType := "file" + if index < len(parts)-1 { + childType = "directory" + } + child := directory.children[childPath] + if child == nil { + child = &GitRepoAnalyticsChild{ + Name: part, + Path: childPath, + Type: childType, + } + directory.children[childPath] = child + } + child.FileCount++ + child.TotalBytes += file.Size + if childType == "directory" { + parentPath = childPath + ensureDirectory(parentPath) + } + } + } + paths := make([]string, 0, len(builders)) + for path := range builders { + paths = append(paths, path) + } + sort.Strings(paths) + directories := make([]GitRepoAnalyticsDirectory, 0, len(paths)) + for _, path := range paths { + builder := builders[path] + childPaths := make([]string, 0, len(builder.children)) + for childPath := range builder.children { + childPaths = append(childPaths, childPath) + } + sort.Strings(childPaths) + builder.Children = make([]GitRepoAnalyticsChild, 0, len(childPaths)) + for _, childPath := range childPaths { + builder.Children = append(builder.Children, *builder.children[childPath]) + } + builder.DirectChildCount = len(builder.Children) + directories = append(directories, builder.GitRepoAnalyticsDirectory) + } + return directories +} + +func filterRepoInventoryFiles(index *repoAnalyticsIndex, gitSubpath string) ([]RepoInventoryFile, error) { + normalizedPath := normalizeRepoSubpath(gitSubpath) + if _, ok := index.directoryLookup[normalizedPath]; !ok { + return nil, fmt.Errorf("load git tree path %s: directory not found", normalizedPath) + } + if normalizedPath == "" { + return append([]RepoInventoryFile(nil), index.sidecar.Files...), nil + } + prefix := normalizedPath + "/" + filtered := make([]RepoInventoryFile, 0) + for _, item := range index.sidecar.Files { + if strings.HasPrefix(item.RepoPath, prefix) { + filtered = append(filtered, item) + } + } + return filtered, nil +} + +func repoDirectoryAggregate(index *repoAnalyticsIndex, gitSubpath string) (GitRepoAnalyticsDirectory, error) { + normalizedPath := normalizeRepoSubpath(gitSubpath) + directory, ok := index.directoryLookup[normalizedPath] + if !ok { + return GitRepoAnalyticsDirectory{}, fmt.Errorf("load git tree path %s: directory not found", normalizedPath) + } + return directory, nil +} + +func repoDirectoryServingIndex(index *repoAnalyticsIndex, gitSubpath string) (repoAnalyticsDirectoryServingIndex, error) { + normalizedPath := normalizeRepoSubpath(gitSubpath) + directory, ok := index.directoryServing[normalizedPath] + if !ok { + return repoAnalyticsDirectoryServingIndex{}, fmt.Errorf("load git tree path %s: directory not found", normalizedPath) + } + return directory, nil +} + +func cloneDirectoryChildren(children []GitRepoAnalyticsChild) []storageAggregate { + out := make([]storageAggregate, 0, len(children)) + for _, child := range children { + out = append(out, storageAggregate{ + name: child.Name, + path: child.Path, + rowType: child.Type, + fileCount: child.FileCount, + totalBytes: child.TotalBytes, + }) + } + return out +} + +func walkGitRepoInventory(prefix string, tree *object.Tree, files *[]RepoInventoryFile, knownDirectories map[string]struct{}) error { + for _, entry := range tree.Entries { + entryPath := entry.Name + if prefix != "" { + entryPath = prefix + "/" + entry.Name + } + if entry.Mode == filemode.Dir { + knownDirectories[entryPath] = struct{}{} + childTree, err := tree.Tree(entry.Name) + if err != nil { + return fmt.Errorf("load nested git tree %s: %w", entryPath, err) + } + if err := walkGitRepoInventory(entryPath, childTree, files, knownDirectories); err != nil { + return err + } + continue + } + file, err := tree.File(entry.Name) + if err != nil { + continue + } + reader, err := file.Reader() + if err != nil { + continue + } + contentBytes, readErr := io.ReadAll(io.LimitReader(reader, 2048)) + _ = reader.Close() + if readErr != nil { + continue + } + pointer := ParseGitLFSPointer(contentBytes) + if pointer == nil { + continue + } + *files = append(*files, RepoInventoryFile{ + RepoPath: entryPath, + Name: entry.Name, + Checksum: pointer.OID, + Size: pointer.Size, + }) + } + return nil +} diff --git a/internal/git/tree_enrichment.go b/internal/git/tree_enrichment.go new file mode 100644 index 0000000..edbf49d --- /dev/null +++ b/internal/git/tree_enrichment.go @@ -0,0 +1,173 @@ +package git + +import ( + "io" + "sort" + "strings" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +func enrichTreeEntries(tree *object.Tree, entries []GitTreeEntry, options GitTreeResponseOptions) { + if !options.IncludeSize && !options.IncludeLFSPointer { + return + } + for index := range entries { + entry := &entries[index] + if entry.Type != "blob" { + continue + } + + file, err := tree.File(entry.Name) + if err != nil { + continue + } + if options.IncludeSize { + entry.Size = file.Size + } + if options.IncludeLFSPointer { + if reader, err := file.Reader(); err == nil { + contentBytes, readErr := io.ReadAll(io.LimitReader(reader, 2048)) + _ = reader.Close() + if readErr == nil { + entry.LFSPointer = ParseGitLFSPointer(contentBytes) + } + } + } + } +} + +func lookupGitPathsLastModified(repo *gogit.Repository, from plumbing.Hash, entries []GitTreeEntry) (map[string]time.Time, error) { + targets := buildGitTreeLastModifiedTargets(entries) + if targets.remaining == 0 { + return map[string]time.Time{}, nil + } + + iter, err := repo.Log(&gogit.LogOptions{ + From: from, + Order: gogit.LogOrderCommitterTime, + }) + if err != nil { + return nil, err + } + defer iter.Close() + + for targets.remaining > 0 { + commit, err := iter.Next() + if err != nil { + if err == io.EOF { + return targets.found, nil + } + return nil, err + } + modifiedAt := commit.Committer.When.UTC() + if commit.NumParents() == 0 { + targets.markAll(modifiedAt) + return targets.found, nil + } + changedPaths, err := commitChangedPaths(commit) + if err != nil { + return nil, err + } + for _, changedPath := range changedPaths { + targets.mark(changedPath, modifiedAt) + if targets.remaining == 0 { + return targets.found, nil + } + } + } + return targets.found, nil +} + +type gitTreeLastModifiedTargets struct { + exact map[string]string + dirs []string + found map[string]time.Time + remaining int +} + +func buildGitTreeLastModifiedTargets(entries []GitTreeEntry) *gitTreeLastModifiedTargets { + targets := &gitTreeLastModifiedTargets{ + exact: make(map[string]string, len(entries)), + found: make(map[string]time.Time, len(entries)), + } + for _, entry := range entries { + normalizedPath := strings.Trim(strings.TrimSpace(entry.Path), "/") + if normalizedPath == "" { + continue + } + targets.exact[normalizedPath] = normalizedPath + if entry.Type == "tree" { + targets.dirs = append(targets.dirs, normalizedPath) + } + targets.remaining++ + } + sort.Slice(targets.dirs, func(i, j int) bool { + return len(targets.dirs[i]) > len(targets.dirs[j]) + }) + return targets +} + +func (targets *gitTreeLastModifiedTargets) mark(changedPath string, modifiedAt time.Time) { + normalizedPath := strings.Trim(strings.TrimSpace(changedPath), "/") + if normalizedPath == "" { + return + } + if targetPath, ok := targets.exact[normalizedPath]; ok { + targets.setFound(targetPath, modifiedAt) + } + for _, dirPath := range targets.dirs { + if normalizedPath == dirPath || strings.HasPrefix(normalizedPath, dirPath+"/") { + targets.setFound(dirPath, modifiedAt) + } + } +} + +func (targets *gitTreeLastModifiedTargets) setFound(path string, modifiedAt time.Time) { + if _, exists := targets.found[path]; exists { + return + } + targets.found[path] = modifiedAt + targets.remaining-- +} + +func (targets *gitTreeLastModifiedTargets) markAll(modifiedAt time.Time) { + for targetPath := range targets.exact { + targets.setFound(targetPath, modifiedAt) + } +} + +func commitChangedPaths(commit *object.Commit) ([]string, error) { + if commit.NumParents() == 0 { + return nil, nil + } + parent, err := commit.Parent(0) + if err != nil { + return nil, err + } + parentTree, err := parent.Tree() + if err != nil { + return nil, err + } + commitTree, err := commit.Tree() + if err != nil { + return nil, err + } + changes, err := parentTree.Diff(commitTree) + if err != nil { + return nil, err + } + paths := make([]string, 0, len(changes)) + for _, change := range changes { + if change.From.Name != "" { + paths = append(paths, change.From.Name) + } + if change.To.Name != "" && change.To.Name != change.From.Name { + paths = append(paths, change.To.Name) + } + } + return paths, nil +} diff --git a/internal/git/tree_response_test.go b/internal/git/tree_response_test.go new file mode 100644 index 0000000..20741f7 --- /dev/null +++ b/internal/git/tree_response_test.go @@ -0,0 +1,316 @@ +package git + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" +) + +func TestBuildGitResponsesDetectLFSPointers(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + pointerContent := strings.Join([]string{ + "version https://git-lfs.github.com/spec/v1", + "oid sha256:0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83", + "size 3780184021", + "", + }, "\n") + if err := os.MkdirAll(filepath.Join(sourcePath, "data"), 0o755); err != nil { + t.Fatalf("create data dir: %v", err) + } + if err := os.WriteFile(filepath.Join(sourcePath, "data", "tcga.tumor.ensembl.tsv"), []byte(pointerContent), 0o644); err != nil { + t.Fatalf("write lfs pointer file: %v", err) + } + if _, err := worktree.Add("data/tcga.tumor.ensembl.tsv"); err != nil { + t.Fatalf("add lfs pointer file: %v", err) + } + if _, err := worktree.Commit("add lfs pointer", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit lfs pointer file: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "data", mirrorRepo, hash, GitTreeResponseOptions{ + IncludeLFSPointer: true, + IncludeSize: true, + }) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if len(treeResponse.Entries) != 1 { + t.Fatalf("expected one tree entry, got %+v", treeResponse.Entries) + } + treePointer := treeResponse.Entries[0].LFSPointer + if treePointer == nil { + t.Fatalf("expected tree entry to be marked as lfs pointer, got %+v", treeResponse.Entries[0]) + } + if treePointer.OID != "0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83" { + t.Fatalf("unexpected lfs oid: %q", treePointer.OID) + } + if treePointer.Size != 3780184021 { + t.Fatalf("unexpected lfs size: %d", treePointer.Size) + } + +} + +func TestBuildGitTreeResponseDefaultsToCheapFields(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + if err := os.WriteFile(filepath.Join(sourcePath, "README.md"), []byte("hello gecko"), 0o644); err != nil { + t.Fatalf("write readme: %v", err) + } + if _, err := worktree.Add("README.md"); err != nil { + t.Fatalf("add readme: %v", err) + } + if _, err := worktree.Commit("initial commit", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: time.Now()}}); err != nil { + t.Fatalf("commit readme: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{}) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if treeResponse.EntryCount != 1 { + t.Fatalf("expected entry count 1, got %d", treeResponse.EntryCount) + } + if treeResponse.Truncated { + t.Fatal("expected non-truncated response by default") + } + if len(treeResponse.Entries) != 1 { + t.Fatalf("expected one tree entry, got %+v", treeResponse.Entries) + } + if treeResponse.Entries[0].Size != 0 { + t.Fatalf("expected default tree response to omit size, got %d", treeResponse.Entries[0].Size) + } + if treeResponse.Entries[0].LFSPointer != nil { + t.Fatalf("expected default tree response to omit lfs pointer, got %+v", treeResponse.Entries[0].LFSPointer) + } + if treeResponse.Entries[0].LastModifiedAt != nil { + t.Fatalf("expected default tree response to omit last modified, got %+v", treeResponse.Entries[0].LastModifiedAt) + } +} + +func TestBuildGitTreeResponseHonorsLimitBeforeEnrichment(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + pointerContent := strings.Join([]string{ + "version https://git-lfs.github.com/spec/v1", + "oid sha256:0bfab2917ce05007ff6297c0ec93ef575209210e4ca998dbd243a270e2f9ca83", + "size 3780184021", + "", + }, "\n") + for name, content := range map[string]string{ + "a.txt": pointerContent, + "b.txt": pointerContent, + "c.txt": "regular file", + } { + if err := os.WriteFile(filepath.Join(sourcePath, name), []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + if _, err := worktree.Add(name); err != nil { + t.Fatalf("add %s: %v", name, err) + } + } + committedAt := time.Date(2026, 7, 6, 12, 0, 0, 0, time.UTC) + if _, err := worktree.Commit("add files", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: committedAt}}); err != nil { + t.Fatalf("commit files: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{ + IncludeLFSPointer: true, + IncludeLastModified: true, + IncludeSize: true, + Limit: 2, + }) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if !treeResponse.Truncated { + t.Fatal("expected truncated response when limit is smaller than entry count") + } + if treeResponse.EntryCount != 3 { + t.Fatalf("expected total entry count 3, got %d", treeResponse.EntryCount) + } + if len(treeResponse.Entries) != 2 { + t.Fatalf("expected two returned entries, got %+v", treeResponse.Entries) + } + if got := []string{treeResponse.Entries[0].Name, treeResponse.Entries[1].Name}; strings.Join(got, ",") != "a.txt,b.txt" { + t.Fatalf("expected limited page to contain only first two entries, got %v", got) + } + for _, entry := range treeResponse.Entries { + if entry.Size == 0 { + t.Fatalf("expected size to be included for limited entry %+v", entry) + } + if entry.LFSPointer == nil { + t.Fatalf("expected lfs pointer to be included for limited entry %+v", entry) + } + if entry.LastModifiedAt == nil || !entry.LastModifiedAt.Equal(committedAt) { + t.Fatalf("expected last modified %s for limited entry, got %+v", committedAt, entry.LastModifiedAt) + } + } +} + +func TestBuildGitTreeResponseBatchesLastModifiedForFilesAndDirectories(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "source") + repo, err := gogit.PlainInit(sourcePath, false) + if err != nil { + t.Fatalf("init source repo: %v", err) + } + worktree, err := repo.Worktree() + if err != nil { + t.Fatalf("load worktree: %v", err) + } + + initialCommitAt := time.Date(2026, 7, 5, 10, 0, 0, 0, time.UTC) + if err := os.MkdirAll(filepath.Join(sourcePath, "data"), 0o755); err != nil { + t.Fatalf("create data dir: %v", err) + } + for path, content := range map[string]string{ + "README.md": "readme", + "data/a.txt": "a", + } { + fullPath := filepath.Join(sourcePath, filepath.FromSlash(path)) + if err := os.WriteFile(fullPath, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + if _, err := worktree.Add(path); err != nil { + t.Fatalf("add %s: %v", path, err) + } + } + if _, err := worktree.Commit("initial files", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: initialCommitAt}}); err != nil { + t.Fatalf("commit initial files: %v", err) + } + + directoryCommitAt := time.Date(2026, 7, 6, 11, 0, 0, 0, time.UTC) + nestedPath := filepath.Join(sourcePath, "data", "nested", "b.txt") + if err := os.MkdirAll(filepath.Dir(nestedPath), 0o755); err != nil { + t.Fatalf("create nested dir: %v", err) + } + if err := os.WriteFile(nestedPath, []byte("b"), 0o644); err != nil { + t.Fatalf("write nested file: %v", err) + } + if _, err := worktree.Add("data/nested/b.txt"); err != nil { + t.Fatalf("add nested file: %v", err) + } + if _, err := worktree.Commit("add nested file", &gogit.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.org", When: directoryCommitAt}}); err != nil { + t.Fatalf("commit nested file: %v", err) + } + + mirrorPath := filepath.Join(tempDir, "mirror.git") + if err := SyncRepositoryMirror(context.Background(), sourcePath, mirrorPath, nil); err != nil { + t.Fatalf("sync mirror: %v", err) + } + mirrorRepo, err := OpenRepository(mirrorPath) + if err != nil { + t.Fatalf("open mirror: %v", err) + } + refName, hash, err := ResolveGitReference(mirrorRepo, "", "") + if err != nil { + t.Fatalf("resolve HEAD: %v", err) + } + + treeResponse, err := BuildGitTreeResponse("org-a/proj-a", refName, "", mirrorRepo, hash, GitTreeResponseOptions{ + IncludeLastModified: true, + }) + if err != nil { + t.Fatalf("build tree response: %v", err) + } + if len(treeResponse.Entries) != 2 { + t.Fatalf("expected data dir and readme entries, got %+v", treeResponse.Entries) + } + + entriesByPath := make(map[string]GitTreeEntry, len(treeResponse.Entries)) + for _, entry := range treeResponse.Entries { + entriesByPath[entry.Path] = entry + } + assertTreeLastModified(t, entriesByPath["data"], directoryCommitAt) + assertTreeLastModified(t, entriesByPath["README.md"], initialCommitAt) + + lastModifiedByPath, err := lookupGitPathsLastModified(mirrorRepo, hash, treeResponse.Entries) + if err != nil { + t.Fatalf("lookup batched last modified: %v", err) + } + if got := lastModifiedByPath["data"]; !got.Equal(directoryCommitAt) { + t.Fatalf("expected batched data timestamp %s, got %s", directoryCommitAt, got) + } + if got := lastModifiedByPath["README.md"]; !got.Equal(initialCommitAt) { + t.Fatalf("expected batched readme timestamp %s, got %s", initialCommitAt, got) + } +} + +func assertTreeLastModified(t *testing.T, entry GitTreeEntry, expected time.Time) { + t.Helper() + if entry.LastModifiedAt == nil { + t.Fatalf("expected %s to have last modified timestamp", entry.Path) + } + if !entry.LastModifiedAt.Equal(expected) { + t.Fatalf("expected %s last modified %s, got %s", entry.Path, expected, *entry.LastModifiedAt) + } +} diff --git a/internal/git/types.go b/internal/git/types.go index 9f8b4d4..93685dc 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -9,8 +9,10 @@ import ( appconfig "github.com/calypr/gecko/config" geckodb "github.com/calypr/gecko/internal/db" "github.com/calypr/gecko/internal/git/domain" + "github.com/calypr/gecko/internal/httpclient" "github.com/calypr/gecko/internal/integrations/fence" gitapi "github.com/calypr/gecko/internal/integrations/github" + "github.com/calypr/gecko/internal/storageaudit" "github.com/jmoiron/sqlx" ) @@ -22,6 +24,9 @@ const ( GitInstallationNotConnected = "not_connected" GitInstallationConnected = "connected" + + GitWorkflowStageAwaitingGitHubConnect = "awaiting_github_connect" + GitWorkflowStageGitHubConnected = "github_connected" ) type GitServiceConfig struct { @@ -53,6 +58,7 @@ type GitProjectStatusResponse struct { RequestAccessResourcePath string `json:"request_access_resource_path,omitempty"` Config appconfig.ProjectConfig `json:"config"` Repository GitRepositoryIdentity `json:"repository"` + WorkflowStage string `json:"workflow_stage,omitempty"` InstallationState string `json:"installation_state"` InstallationID *int64 `json:"installation_id,omitempty"` InstallationTarget string `json:"installation_target,omitempty"` @@ -77,7 +83,6 @@ type GitOrganizationConnectResponse struct { // GitRepositoryInstallationStatus is an alias for domain.GitRepositoryInstallationStatus. type GitRepositoryInstallationStatus = domain.GitRepositoryInstallationStatus - type ProjectIntegrationCheck struct { Pass bool `json:"pass"` Reason string `json:"reason,omitempty"` @@ -94,6 +99,7 @@ type GitOrganizationProjectStatus struct { Project string `json:"project"` ResourcePath string `json:"resource_path"` Repository GitRepositoryIdentity `json:"repository"` + WorkflowStage string `json:"workflow_stage,omitempty"` Configured bool `json:"configured"` Readiness *CalyprProjectReadiness `json:"readiness,omitempty"` Integrations ProjectIntegrationStatus `json:"integrations"` @@ -163,8 +169,6 @@ type CalyprProjectSetupResponse struct { // GitHubInstallationRepository is an alias for domain.GitHubInstallationRepository. type GitHubInstallationRepository = domain.GitHubInstallationRepository - - type GitOrganizationStatusResponse struct { Organization string `json:"organization"` Connected bool `json:"connected"` @@ -228,11 +232,36 @@ type GitTreeEntry struct { LFSPointer *GitLFSPointerInfo `json:"lfs_pointer,omitempty"` } +type GitTreeResponseOptions struct { + IncludeSize bool + IncludeLastModified bool + IncludeLFSPointer bool + Limit int +} + type GitProjectTreeResponse struct { - ProjectID string `json:"project_id"` - Ref string `json:"ref"` - Path string `json:"path"` - Entries []GitTreeEntry `json:"entries"` + ProjectID string `json:"project_id"` + Ref string `json:"ref"` + Path string `json:"path"` + EntryCount int `json:"entry_count"` + Truncated bool `json:"truncated,omitempty"` + Entries []GitTreeEntry `json:"entries"` +} + +type GitManifestResponseOptions struct { + Limit int + Cursor string + FilesOnly bool +} + +type GitProjectManifestResponse struct { + ProjectID string `json:"project_id"` + Ref string `json:"ref"` + Path string `json:"path"` + EntryCount int `json:"entry_count"` + HasMore bool `json:"has_more"` + NextCursor string `json:"next_cursor,omitempty"` + Entries []GitTreeEntry `json:"entries"` } type GitProjectFileResponse struct { @@ -253,6 +282,94 @@ type GitLFSPointerInfo struct { Size int64 `json:"size"` } +type GitRepoAnalyticsChild struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + FileCount int `json:"file_count"` + TotalBytes int64 `json:"total_bytes"` +} + +type GitRepoAnalyticsDirectory struct { + Path string `json:"path"` + DirectChildCount int `json:"direct_child_count"` + FileCount int `json:"file_count"` + TotalBytes int64 `json:"total_bytes"` + Children []GitRepoAnalyticsChild `json:"children"` +} + +type GitRepoAnalyticsIndexSidecar struct { + SchemaVersion int `json:"schema_version"` + CommitHash string `json:"commit_hash"` + RefName string `json:"ref_name"` + GeneratedAt time.Time `json:"generated_at"` + Files []RepoInventoryFile `json:"files"` + Directories []GitRepoAnalyticsDirectory `json:"directories"` +} + +type GitStorageSummaryResponse struct { + Path string `json:"path"` + Source string `json:"source,omitempty"` + FileCount int `json:"file_count"` + RecordCount int `json:"record_count"` + DirectChildCount int `json:"direct_child_count"` + TotalBytes int64 `json:"total_bytes"` + DownloadCount int64 `json:"download_count"` + LastDownloadTime string `json:"last_download_time,omitempty"` + LatestUpdateTime string `json:"latest_update_time,omitempty"` + DuplicatePathCount int `json:"duplicate_path_count"` +} + +type GitStorageChildResponseItem struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + FileCount int `json:"file_count"` + RecordCount int `json:"record_count"` + TotalBytes int64 `json:"total_bytes"` + DownloadCount int64 `json:"download_count"` + LastDownloadTime string `json:"last_download_time,omitempty"` + LatestUpdateTime string `json:"latest_update_time,omitempty"` +} + +type GitStorageChildrenResponse struct { + Items []GitStorageChildResponseItem `json:"items"` + HasMore bool `json:"has_more"` + NextCursor string `json:"next_cursor,omitempty"` +} + +type GitStorageFolderResponse struct { + Summary GitStorageSummaryResponse `json:"summary"` + Children GitStorageChildrenResponse `json:"children"` +} + +type GitProjectDiffAuditRequest = storageaudit.ProjectDiffAuditRequest +type GitAuditEvidence = storageaudit.Evidence +type GitProjectDiffFinding = storageaudit.ProjectDiffFinding +type GitProjectDiffSummary = storageaudit.ProjectDiffSummary +type GitProjectDiffAuditResponse = storageaudit.ProjectDiffAuditResponse + +type GitStorageCleanupAuditRequest = storageaudit.CleanupAuditRequest +type GitStorageChainAuditRequest = storageaudit.ChainAuditRequest +type GitStorageChainFinding = storageaudit.ChainFinding +type GitStorageChainAuditSummary = storageaudit.ChainSummary +type GitStorageChainIssueGroup = storageaudit.ChainIssueGroup +type GitStorageChainAuditResponse = storageaudit.ChainAuditResponse +type GitOnlySyfonRegistrationRequest = storageaudit.GitOnlySyfonRegistrationRequest +type GitOnlySyfonRegistrationResult = storageaudit.GitOnlySyfonRegistrationResult +type GitOnlySyfonRegistrationResponse = storageaudit.GitOnlySyfonRegistrationResponse +type GitStorageCleanupAccessProbe = storageaudit.CleanupAccessProbe +type GitStorageCleanupAccessMethod = storageaudit.CleanupAccessMethod +type GitStorageCleanupRecordAudit = storageaudit.CleanupRecordAudit +type GitStorageCleanupFinding = storageaudit.CleanupFinding +type GitStorageCleanupApplyAction = storageaudit.CleanupApplyAction +type GitStorageCleanupApplyFinding = storageaudit.CleanupApplyFinding +type GitStorageCleanupAuditSummary = storageaudit.CleanupAuditSummary +type GitStorageCleanupAuditResponse = storageaudit.CleanupAuditResponse +type GitStorageCleanupApplyRequest = storageaudit.CleanupApplyRequest +type GitStorageCleanupPurgeResult = storageaudit.CleanupPurgeResult +type GitStorageCleanupApplyResponse = storageaudit.CleanupApplyResponse + type GitUploadSessionFileManifest struct { Name string `json:"name"` Size int64 `json:"size"` @@ -310,18 +427,16 @@ type GitUploadSessionResponse struct { // GitHubRepositoryMetadata is an alias for domain.GitHubRepositoryMetadata. type GitHubRepositoryMetadata = domain.GitHubRepositoryMetadata - // HTTPStatusError is an alias for domain.HTTPStatusError. type HTTPStatusError = domain.HTTPStatusError - func NewGitService(config GitServiceConfig) *GitService { if config.GitHubAPIBase == "" { config.GitHubAPIBase = "https://api.github.com" } client := config.HTTPClient if client == nil { - client = &http.Client{Timeout: 20 * time.Second} + client = httpclient.NewServiceClient(20 * time.Second) } return &GitService{ config: config, @@ -372,8 +487,6 @@ type StorageBucket = domain.StorageBucket // StorageConfig is an alias for domain.StorageConfig. type StorageConfig = domain.StorageConfig - func ProgramProjectResourcePath(organization, project string) string { return fmt.Sprintf("/programs/%s/projects/%s", organization, project) } - diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go new file mode 100644 index 0000000..0b5f954 --- /dev/null +++ b/internal/httpclient/client.go @@ -0,0 +1,22 @@ +package httpclient + +import ( + "net" + "net/http" + "time" +) + +// NewServiceClient returns a default outbound service client with normal Go +// transport behavior, including HTTP/2 when the peer supports it. +func NewServiceClient(timeout time.Duration) *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DialContext = (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext + + return &http.Client{ + Timeout: timeout, + Transport: transport, + } +} diff --git a/internal/integrations/fence/client.go b/internal/integrations/fence/client.go index 2de6dba..36e3437 100644 --- a/internal/integrations/fence/client.go +++ b/internal/integrations/fence/client.go @@ -8,8 +8,10 @@ import ( "io" "net/http" "strings" + "time" "github.com/calypr/gecko/internal/git/domain" + "github.com/calypr/gecko/internal/httpclient" servermw "github.com/calypr/gecko/internal/server/middleware" ) @@ -24,7 +26,7 @@ type Client struct { func NewClient(client *http.Client, config Config) *Client { if client == nil { - client = http.DefaultClient + client = httpclient.NewServiceClient(30 * time.Second) } return &Client{client: client, config: config} } @@ -166,10 +168,12 @@ func (c *Client) RequestOrganizationInstallationStatus(ctx context.Context, auth }, nil } -func (c *Client) ListInstallationRepositories(ctx context.Context, authorizationHeader string, installationID int64) ([]domain.GitHubInstallationRepository, error) { +func (c *Client) ListInstallationRepositories(ctx context.Context, authorizationHeader string, organization string, owner string, installationID int64) ([]domain.GitHubInstallationRepository, error) { var payload fenceGitHubInstallationRepositoriesResponse if err := c.requestFenceGitHubBroker(ctx, authorizationHeader, map[string]any{ "action": "installation_repositories", + "organization": organization, + "owner": owner, "installation_id": installationID, }, &payload); err != nil { return nil, err @@ -177,26 +181,6 @@ func (c *Client) ListInstallationRepositories(ctx context.Context, authorization return payload.Repositories, nil } -func (c *Client) RequestInstallationStatus(ctx context.Context, authorizationHeader string, organization string, identity domain.GitRepositoryIdentity) (domain.GitRepositoryInstallationStatus, error) { - var payload fenceGitHubInstallationStatusResponse - if err := c.requestFenceGitHubBroker(ctx, authorizationHeader, map[string]any{ - "action": "repository_installation", - "owner": identity.Owner, - "repo": identity.Repo, - "organization": organization, - }, &payload); err != nil { - return domain.GitRepositoryInstallationStatus{}, err - } - return domain.GitRepositoryInstallationStatus{ - Installed: payload.Installed, - InstallationID: payload.InstallationID, - Target: strings.TrimSpace(payload.Target), - TargetType: strings.TrimSpace(payload.TargetType), - HTMLURL: strings.TrimSpace(payload.HTMLURL), - RepositorySelection: strings.TrimSpace(payload.RepositorySelection), - }, nil -} - func (c *Client) RequestInstallationToken(ctx context.Context, authorizationHeader string, organization string, project string, identity domain.GitRepositoryIdentity, access string) (string, error) { requestedAccess := strings.TrimSpace(access) if requestedAccess == "" { diff --git a/internal/integrations/fence/client_test.go b/internal/integrations/fence/client_test.go index 0387314..1891689 100644 --- a/internal/integrations/fence/client_test.go +++ b/internal/integrations/fence/client_test.go @@ -65,49 +65,6 @@ func TestRequestOrganizationInstallationStatusForwardsAuthorizationAndParsesStat } } -func TestRequestInstallationStatusForwardsAuthorizationAndParsesStatus(t *testing.T) { - var receivedAuth string - server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - receivedAuth = request.Header.Get("Authorization") - if request.URL.Path != "/credentials/github" { - t.Fatalf("unexpected request path: %s", request.URL.Path) - } - if request.Method != http.MethodPost { - t.Fatalf("unexpected request method: %s", request.Method) - } - writer.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(writer).Encode(map[string]any{ - "installed": true, - "installation_id": 42, - "target": "HTAN_INT", - "target_type": "Organization", - "html_url": "https://github.com/organizations/HTAN_INT/settings/installations/42", - }) - })) - defer server.Close() - - client := NewClient(server.Client(), Config{BaseURL: server.URL}) - status, err := client.RequestInstallationStatus(context.Background(), "Bearer user-token", "HTAN_INT", domain.GitRepositoryIdentity{ - Owner: "HTAN_INT", - Repo: "BForePC", - }) - if err != nil { - t.Fatalf("request installation status: %v", err) - } - if !status.Installed { - t.Fatal("expected installed status") - } - if status.InstallationID == nil || *status.InstallationID != 42 { - t.Fatalf("unexpected installation id: %+v", status.InstallationID) - } - if status.Target != "HTAN_INT" { - t.Fatalf("unexpected target: %q", status.Target) - } - if receivedAuth != "Bearer user-token" { - t.Fatalf("expected forwarded authorization header, got %q", receivedAuth) - } -} - func TestListInstallationRepositoriesForwardsAuthorizationAndParsesRepositories(t *testing.T) { var receivedAuth string var receivedBody map[string]any @@ -139,7 +96,7 @@ func TestListInstallationRepositoriesForwardsAuthorizationAndParsesRepositories( defer server.Close() client := NewClient(server.Client(), Config{BaseURL: server.URL}) - repositories, err := client.ListInstallationRepositories(context.Background(), "Bearer user-token", 42) + repositories, err := client.ListInstallationRepositories(context.Background(), "Bearer user-token", "Ellrott_Lab", "EllrottLab", 42) if err != nil { t.Fatalf("list installation repositories: %v", err) } @@ -152,6 +109,12 @@ func TestListInstallationRepositoriesForwardsAuthorizationAndParsesRepositories( if receivedBody["installation_id"] != float64(42) { t.Fatalf("expected installation id in request body, got %#v", receivedBody) } + if receivedBody["organization"] != "Ellrott_Lab" { + t.Fatalf("expected organization in request body, got %#v", receivedBody) + } + if receivedBody["owner"] != "EllrottLab" { + t.Fatalf("expected owner in request body, got %#v", receivedBody) + } if len(repositories) != 1 { t.Fatalf("expected one repository, got %+v", repositories) } diff --git a/internal/integrations/github/client.go b/internal/integrations/github/client.go index e434851..47f6c62 100644 --- a/internal/integrations/github/client.go +++ b/internal/integrations/github/client.go @@ -2,16 +2,26 @@ package github import ( "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" "fmt" + "io" + "log" + "net" "net/http" + "net/url" "strings" + "time" "github.com/calypr/gecko/internal/git/domain" + "github.com/calypr/gecko/internal/httpclient" google_github "github.com/google/go-github/v87/github" ) type Config struct { - APIBase string + APIBase string + DisableTransportDiagnostics bool } type Client struct { @@ -26,7 +36,7 @@ type GitHubRepositoryMetadata struct { func NewClient(client *http.Client, config Config) *Client { if client == nil { - client = http.DefaultClient + client = httpclient.NewServiceClient(30 * time.Second) } if config.APIBase == "" { config.APIBase = "https://api.github.com" @@ -39,7 +49,7 @@ func (c *Client) FetchRepositoryMetadata(ctx context.Context, accessToken string if err != nil { return nil, err } - repo, _, err := githubClient.Repositories.Get(ctx, identity.Owner, identity.Repo) + repo, err := c.fetchRepositoryMetadata(ctx, githubClient, identity, accessToken) if err != nil { return nil, fmt.Errorf("github repository metadata lookup failed for %s/%s: %w", identity.Owner, identity.Repo, err) } @@ -51,6 +61,151 @@ func (c *Client) FetchRepositoryMetadata(ctx context.Context, accessToken string return &domain.GitHubRepositoryMetadata{DefaultBranch: defaultBranch, HTMLURL: htmlURL}, nil } +func (c *Client) fetchRepositoryMetadata(ctx context.Context, client *google_github.Client, identity domain.GitRepositoryIdentity, accessToken string) (*google_github.Repository, error) { + started := time.Now() + requestURL := c.repositoryMetadataRequestURL(identity) + authConfigured := strings.TrimSpace(accessToken) != "" + tokenFingerprint := githubAccessTokenFingerprint(accessToken) + tokenLength := githubAccessTokenLength(accessToken) + log.Printf("INFO: github_repository_metadata_request_start owner=%s repo=%s request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d", identity.Owner, identity.Repo, requestURL, authConfigured, tokenFingerprint, tokenLength) + repo, response, err := client.Repositories.Get(ctx, identity.Owner, identity.Repo) + statusCode := 0 + rateLimitRemaining := -1 + rateLimitReset := "" + if response != nil { + rateLimitRemaining = response.Rate.Remaining + if !response.Rate.Reset.Time.IsZero() { + rateLimitReset = response.Rate.Reset.Time.UTC().Format(time.RFC3339) + } + if response.Response != nil { + statusCode = response.Response.StatusCode + } + } + if err == nil { + log.Printf("INFO: github_repository_metadata_request_done owner=%s repo=%s request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%d rate_limit_reset=%q duration_ms=%d", identity.Owner, identity.Repo, requestURL, authConfigured, tokenFingerprint, tokenLength, statusCode, rateLimitRemaining, rateLimitReset, time.Since(started).Milliseconds()) + return repo, nil + } + log.Printf("INFO: github_repository_metadata_request_done owner=%s repo=%s request_url=%q auth_configured=%t auth_scheme=Bearer token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%d rate_limit_reset=%q duration_ms=%d error_type=%T error=%q", identity.Owner, identity.Repo, requestURL, authConfigured, tokenFingerprint, tokenLength, statusCode, rateLimitRemaining, rateLimitReset, time.Since(started).Milliseconds(), err, err.Error()) + if response == nil { + c.logTransportDiagnostic(ctx, accessToken, err) + } + return nil, err +} + +func (c *Client) repositoryMetadataRequestURL(identity domain.GitRepositoryIdentity) string { + apiBase := strings.TrimRight(c.config.APIBase, "/") + if apiBase == "" { + apiBase = "https://api.github.com" + } + return fmt.Sprintf("%s/repos/%s/%s", apiBase, identity.Owner, identity.Repo) +} + +func githubAccessTokenFingerprint(accessToken string) string { + accessToken = strings.TrimSpace(accessToken) + if accessToken == "" { + return "" + } + sum := sha256.Sum256([]byte(accessToken)) + return hex.EncodeToString(sum[:])[:16] +} + +func githubAccessTokenLength(accessToken string) int { + return len(strings.TrimSpace(accessToken)) +} + +func (c *Client) logTransportDiagnostic(parent context.Context, accessToken string, originalErr error) { + if c.config.DisableTransportDiagnostics { + return + } + apiBase := strings.TrimRight(c.config.APIBase, "/") + if apiBase == "" { + apiBase = "https://api.github.com" + } + parsed, err := url.Parse(apiBase) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_done api_base=%q token_fingerprint=%s token_length=%d original_error_type=%T original_error=%q diagnostic_error=%q", apiBase, githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), originalErr, originalErr.Error(), err.Error()) + return + } + host := parsed.Hostname() + port := parsed.Port() + if port == "" { + if parsed.Scheme == "http" { + port = "80" + } else { + port = "443" + } + } + address := net.JoinHostPort(host, port) + log.Printf("INFO: github_transport_diagnostic_start api_base=%q host=%s port=%s scheme=%s token_fingerprint=%s token_length=%d original_error_type=%T original_error=%q", apiBase, host, port, parsed.Scheme, githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), originalErr, originalErr.Error()) + + diagCtx, cancel := context.WithTimeout(parent, 15*time.Second) + defer cancel() + c.logDNSDiagnostic(diagCtx, host) + c.logTCPDiagnostic(diagCtx, address) + if parsed.Scheme == "https" { + c.logTLSDiagnostic(host, address) + } + c.logHTTPDiagnostic(diagCtx, apiBase, accessToken) +} + +func (c *Client) logDNSDiagnostic(ctx context.Context, host string) { + started := time.Now() + ips, err := net.DefaultResolver.LookupHost(ctx, host) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=dns host=%s duration_ms=%d error_type=%T error=%q", host, time.Since(started).Milliseconds(), err, err.Error()) + return + } + log.Printf("INFO: github_transport_diagnostic_stage stage=dns host=%s duration_ms=%d ips=%q", host, time.Since(started).Milliseconds(), strings.Join(ips, ",")) +} + +func (c *Client) logTCPDiagnostic(ctx context.Context, address string) { + started := time.Now() + conn, err := (&net.Dialer{Timeout: 5 * time.Second}).DialContext(ctx, "tcp", address) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=tcp address=%s duration_ms=%d error_type=%T error=%q", address, time.Since(started).Milliseconds(), err, err.Error()) + return + } + remote := conn.RemoteAddr().String() + local := conn.LocalAddr().String() + _ = conn.Close() + log.Printf("INFO: github_transport_diagnostic_stage stage=tcp address=%s local_addr=%s remote_addr=%s duration_ms=%d", address, local, remote, time.Since(started).Milliseconds()) +} + +func (c *Client) logTLSDiagnostic(host string, address string) { + started := time.Now() + dialer := &net.Dialer{Timeout: 5 * time.Second} + conn, err := tls.DialWithDialer(dialer, "tcp", address, &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12}) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=tls host=%s address=%s duration_ms=%d error_type=%T error=%q", host, address, time.Since(started).Milliseconds(), err, err.Error()) + return + } + state := conn.ConnectionState() + _ = conn.Close() + log.Printf("INFO: github_transport_diagnostic_stage stage=tls host=%s address=%s version=%x cipher_suite=%x server_name=%s duration_ms=%d", host, address, state.Version, state.CipherSuite, state.ServerName, time.Since(started).Milliseconds()) +} + +func (c *Client) logHTTPDiagnostic(ctx context.Context, apiBase string, accessToken string) { + started := time.Now() + requestURL := strings.TrimRight(apiBase, "/") + "/rate_limit" + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=http request_url=%q token_fingerprint=%s token_length=%d duration_ms=%d error_type=%T error=%q", requestURL, githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), time.Since(started).Milliseconds(), err, err.Error()) + return + } + if strings.TrimSpace(accessToken) != "" { + request.Header.Set("Authorization", "Bearer "+strings.TrimSpace(accessToken)) + } + request.Header.Set("Accept", "application/vnd.github+json") + response, err := c.client.Do(request) + if err != nil { + log.Printf("INFO: github_transport_diagnostic_stage stage=http request_url=%q auth_configured=%t token_fingerprint=%s token_length=%d duration_ms=%d error_type=%T error=%q", requestURL, strings.TrimSpace(accessToken) != "", githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), time.Since(started).Milliseconds(), err, err.Error()) + return + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1024)) + log.Printf("INFO: github_transport_diagnostic_stage stage=http request_url=%q auth_configured=%t token_fingerprint=%s token_length=%d status=%d rate_limit_remaining=%q rate_limit_reset=%q duration_ms=%d", requestURL, strings.TrimSpace(accessToken) != "", githubAccessTokenFingerprint(accessToken), githubAccessTokenLength(accessToken), response.StatusCode, response.Header.Get("X-RateLimit-Remaining"), response.Header.Get("X-RateLimit-Reset"), time.Since(started).Milliseconds()) +} + func (c *Client) githubClient(accessToken string) (*google_github.Client, error) { options := []google_github.ClientOptionsFunc{ google_github.WithAuthToken(accessToken), diff --git a/internal/integrations/github/client_test.go b/internal/integrations/github/client_test.go new file mode 100644 index 0000000..0812ab0 --- /dev/null +++ b/internal/integrations/github/client_test.go @@ -0,0 +1,96 @@ +package github + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/calypr/gecko/internal/git/domain" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + +func TestFetchRepositoryMetadataUsesInstallationToken(t *testing.T) { + attempts := 0 + client := NewClient(&http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + if req.Header.Get("Authorization") != "Bearer install-token" { + t.Fatalf("expected installation token auth header, got %q", req.Header.Get("Authorization")) + } + return githubJSONResponse(http.StatusOK, `{"default_branch":"main","html_url":"https://github.com/BForePC/BForePC"}`), nil + }), + }, Config{}) + + metadata, err := client.FetchRepositoryMetadata(context.Background(), "install-token", domain.GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }) + if err != nil { + t.Fatalf("fetch repository metadata: %v", err) + } + if attempts != 1 { + t.Fatalf("expected one metadata request, got %d attempts", attempts) + } + if metadata.DefaultBranch != "main" || metadata.HTMLURL != "https://github.com/BForePC/BForePC" { + t.Fatalf("unexpected metadata: %+v", metadata) + } +} + +func TestFetchRepositoryMetadataDoesNotRetryEOF(t *testing.T) { + attempts := 0 + client := NewClient(&http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + return nil, io.EOF + }), + }, Config{DisableTransportDiagnostics: true}) + + _, err := client.FetchRepositoryMetadata(context.Background(), "install-token", domain.GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }) + if err == nil { + t.Fatal("expected EOF error") + } + if attempts != 1 { + t.Fatalf("expected no retry after EOF, got %d attempts", attempts) + } +} + +func TestFetchRepositoryMetadataDoesNotRetryUnauthorized(t *testing.T) { + attempts := 0 + client := NewClient(&http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + return githubJSONResponse(http.StatusUnauthorized, `{"message":"Bad credentials"}`), nil + }), + }, Config{}) + + _, err := client.FetchRepositoryMetadata(context.Background(), "bad-token", domain.GitRepositoryIdentity{ + Owner: "BForePC", + Repo: "BForePC", + }) + if err == nil { + t.Fatal("expected unauthorized error") + } + if attempts != 1 { + t.Fatalf("expected no retry for unauthorized response, got %d attempts", attempts) + } +} + +func githubJSONResponse(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(body)), + } +} diff --git a/internal/integrations/syfon/adapter.go b/internal/integrations/syfon/adapter.go index 453137a..98b9d42 100644 --- a/internal/integrations/syfon/adapter.go +++ b/internal/integrations/syfon/adapter.go @@ -2,28 +2,178 @@ package syfon import ( "context" + "encoding/json" + "errors" "fmt" + "io" + "log" "net/http" "net/url" "path" + "sort" + "strconv" "strings" + "sync" + "time" "github.com/calypr/gecko/internal/git/domain" + "github.com/calypr/gecko/internal/httpclient" "github.com/calypr/syfon/apigen/client/bucketapi" + drsapi "github.com/calypr/syfon/apigen/client/drs" + internalapi "github.com/calypr/syfon/apigen/client/internalapi" + metricsapi "github.com/calypr/syfon/apigen/client/metricsapi" syfonservices "github.com/calypr/syfon/client/services" ) const refreshAuthzHeader = "X-Syfon-Refresh-Authz" +const bulkStorageProbeBatchSize = 200 +const bulkStorageProbeConcurrency = 4 + +type HTTPError struct { + Method string + Path string + Status int + Body string +} + +func (err HTTPError) Error() string { + return fmt.Sprintf("syfon %s %s failed with status %d: %s", err.Method, err.Path, err.Status, strings.TrimSpace(err.Body)) +} + +func IsHTTPStatus(err error, statuses ...int) bool { + var httpErr *HTTPError + if !errors.As(err, &httpErr) { + return false + } + for _, status := range statuses { + if httpErr.Status == status { + return true + } + } + return false +} type Manager struct { baseURL string client *http.Client } +type ProjectRecord struct { + ObjectID string + Name string + Checksum string + Organization string + Project string + Size int64 + UpdatedAt *time.Time + CreatedAt *time.Time + AccessURLs []string + AccessMethods []ProjectAccessMethod +} + +type ProjectAccessMethod struct { + AccessID string + Type string + URL string + Headers []string +} + +type ProjectMetricsSummary struct { + RecordCount int + RecordLatestUpdatedTime string + RecordRevision string +} + +type BulkStorageProbeItem struct { + ID string + ObjectURL string + ExpectedSizeBytes *int64 + ExpectedSHA256 string + ExpectedName string +} + +type BulkStorageProbeResult struct { + ID string + Operation string + ObjectURL string + Provider string + Bucket string + Key string + Path string + Exists bool + Status string + Error string + ErrorKind string + SizeBytes *int64 + MetaSHA256 string + ETag string + LastModified string + ValidationStatus string + SizeMatch *bool + NameMatch *bool + SHA256Match *bool + ValidationMismatches []string +} + +type ProjectBucketSummary struct { + ObjectURL string + Provider string + Bucket string + Prefix string + Exists bool + ObjectCount int + TotalBytes int64 + ComputedAt string + Mode string +} + +type ProjectBucketObject struct { + ObjectURL string + Provider string + Bucket string + Key string + Path string + SizeBytes int64 + MetaSHA256 string + ETag string + LastModified string +} + +type FileUsage struct { + ObjectID string + Name string + Size int64 + DownloadCount int64 + UploadCount int64 + LastAccessTime *time.Time + LastDownloadTime *time.Time + LastUploadTime *time.Time +} + +type ProjectBucketDeleteResult struct { + ObjectURL string + Status string + Error string +} + +// ProjectObjectRegistration is the narrow subset of DRS registration Gecko +// needs when it restores a Syfon record for an existing Git LFS pointer. +type ProjectObjectRegistration struct { + Name string + Checksum string + Size int64 + ControlledAccess []string + AccessURLs []string +} + +type ProjectObjectRegistrationResult struct { + ObjectID string +} + func NewManager(baseURL string, client *http.Client) *Manager { httpClient := client if httpClient == nil { - httpClient = http.DefaultClient + httpClient = httpclient.NewServiceClient(5 * time.Minute) } return &Manager{ baseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"), @@ -114,6 +264,45 @@ func (manager *Manager) AddScope(ctx context.Context, authorizationHeader string return nil } +func (manager *Manager) ListBucketScopes(ctx context.Context, authorizationHeader string, bucket string) ([]domain.StorageBucketScope, error) { + requestPath := "/data/buckets/" + url.PathEscape(strings.TrimSpace(bucket)) + "/scopes" + var scopes []struct { + Organization string `json:"organization"` + ProjectId string `json:"project_id"` + Path *string `json:"path"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, requestPath, nil, nil, &scopes); err != nil { + return nil, fmt.Errorf("request syfon bucket scopes: %w", err) + } + out := make([]domain.StorageBucketScope, 0, len(scopes)) + for _, scope := range scopes { + out = append(out, domain.StorageBucketScope{ + Bucket: strings.TrimSpace(bucket), + Organization: strings.TrimSpace(scope.Organization), + ProjectID: strings.TrimSpace(scope.ProjectId), + Path: stringValue(scope.Path), + }) + } + return out, nil +} + +func uniqueNonEmptyStrings(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + func (manager *Manager) CleanupProject(ctx context.Context, authorizationHeader string, organization string, project string) error { dataBaseURL, err := manager.dataAPIBaseURL() if err != nil { @@ -141,6 +330,774 @@ func (manager *Manager) CleanupProject(ctx context.Context, authorizationHeader return nil } +func (manager *Manager) ListProjectRecords(ctx context.Context, authorizationHeader string, organization string, project string) ([]ProjectRecord, error) { + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) + page := 1 + out := make([]ProjectRecord, 0) + for { + params.Set("limit", "1000") + params.Set("page", strconv.Itoa(page)) + var response internalapi.ListRecordsResponse + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, "/index", params, nil, &response); err != nil { + return nil, fmt.Errorf("list syfon project records page %d: %w", page, err) + } + records := []internalapi.InternalRecord{} + if response.Records != nil { + records = *response.Records + } + if len(records) == 0 { + break + } + for _, record := range records { + projectRecord, ok := projectRecordFromInternal(record) + if ok { + out = append(out, projectRecord) + } + } + if len(records) < 1000 { + break + } + page++ + } + return out, nil +} + +func (manager *Manager) ListProjectAuditRecords(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]ProjectRecord, error) { + requestBody := struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + PathPrefix string `json:"path_prefix,omitempty"` + }{ + Organization: strings.TrimSpace(organization), + Project: strings.TrimSpace(project), + PathPrefix: strings.Trim(strings.TrimSpace(pathPrefix), "/"), + } + var response struct { + Items []struct { + ObjectID string `json:"object_id"` + Name string `json:"name"` + Checksum string `json:"checksum"` + Organization string `json:"organization"` + Project string `json:"project"` + Size int64 `json:"size"` + CreatedTime string `json:"created_time"` + UpdatedTime string `json:"updated_time"` + AccessURLs []string `json:"access_urls"` + AccessMethods []struct { + AccessID string `json:"access_id"` + Type string `json:"type"` + URL string `json:"url"` + Headers []string `json:"headers"` + } `json:"access_methods"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-records", nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("list syfon project audit records: %w", err) + } + out := make([]ProjectRecord, 0, len(response.Items)) + for _, item := range response.Items { + checksum := normalizeSHA256(item.Checksum) + if checksum == "" { + continue + } + accessURLs := make([]string, 0, len(item.AccessURLs)) + for _, raw := range item.AccessURLs { + if trimmed := strings.TrimSpace(raw); trimmed != "" { + accessURLs = append(accessURLs, trimmed) + } + } + accessMethods := make([]ProjectAccessMethod, 0, len(item.AccessMethods)) + for _, method := range item.AccessMethods { + methodURL := strings.TrimSpace(method.URL) + accessMethods = append(accessMethods, ProjectAccessMethod{ + AccessID: strings.TrimSpace(method.AccessID), + Type: strings.TrimSpace(method.Type), + URL: methodURL, + Headers: append([]string(nil), method.Headers...), + }) + if methodURL != "" { + accessURLs = append(accessURLs, methodURL) + } + } + out = append(out, ProjectRecord{ + ObjectID: strings.TrimSpace(item.ObjectID), + Name: strings.TrimSpace(item.Name), + Checksum: checksum, + Organization: strings.TrimSpace(item.Organization), + Project: strings.TrimSpace(item.Project), + Size: item.Size, + CreatedAt: parseOptionalTime(optionalString(item.CreatedTime)), + UpdatedAt: parseOptionalTime(optionalString(item.UpdatedTime)), + AccessURLs: uniqueNonEmptyStrings(accessURLs), + AccessMethods: accessMethods, + }) + } + return out, nil +} + +func (manager *Manager) GetProjectMetricsSummary(ctx context.Context, authorizationHeader string, organization string, project string) (*ProjectMetricsSummary, error) { + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) + var response struct { + RecordCount *int `json:"record_count"` + RecordLatestUpdatedTime string `json:"record_latest_updated_time"` + RecordRevision string `json:"record_revision"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, "/index/v1/metrics/summary", params, nil, &response); err != nil { + return nil, fmt.Errorf("get syfon metrics summary: %w", err) + } + if response.RecordCount == nil { + return nil, nil + } + return &ProjectMetricsSummary{ + RecordCount: *response.RecordCount, + RecordLatestUpdatedTime: strings.TrimSpace(response.RecordLatestUpdatedTime), + RecordRevision: strings.TrimSpace(response.RecordRevision), + }, nil +} + +func (manager *Manager) ListProjectScopes(ctx context.Context, authorizationHeader string, organization string, project string) ([]domain.StorageBucketScope, error) { + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) + var response struct { + Items []struct { + Bucket string `json:"bucket"` + Organization string `json:"organization"` + ProjectID string `json:"project_id"` + Path string `json:"path"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, "/data/inspect/project-scopes", params, nil, &response); err != nil { + return nil, fmt.Errorf("list syfon project scopes: %w", err) + } + out := make([]domain.StorageBucketScope, 0, len(response.Items)) + for _, item := range response.Items { + out = append(out, domain.StorageBucketScope{ + Bucket: strings.TrimSpace(item.Bucket), + Organization: strings.TrimSpace(item.Organization), + ProjectID: strings.TrimSpace(item.ProjectID), + Path: strings.TrimSpace(item.Path), + }) + } + return out, nil +} + +func (manager *Manager) BulkGetProjectRecordsByChecksum(ctx context.Context, authorizationHeader string, organization string, project string, checksums []string) (map[string][]ProjectRecord, error) { + normalized := dedupeChecksums(checksums) + out := make(map[string][]ProjectRecord, len(normalized)) + if len(normalized) == 0 { + return out, nil + } + const batchSize = 200 + for start := 0; start < len(normalized); start += batchSize { + end := start + batchSize + if end > len(normalized) { + end = len(normalized) + } + requestBody := internalapi.BulkHashesRequest{Hashes: normalized[start:end]} + var response struct { + Results map[string][]internalapi.InternalRecord `json:"results"` + Records *[]internalapi.InternalRecord `json:"records"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/index/bulk/hashes", nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("bulk lookup syfon checksums: %w", err) + } + if len(response.Results) > 0 { + for _, records := range response.Results { + for _, record := range records { + projectRecord, ok := projectRecordFromInternal(record) + if !ok { + continue + } + if !recordMatchesScope(projectRecord, organization, project) { + continue + } + out[projectRecord.Checksum] = append(out[projectRecord.Checksum], projectRecord) + } + } + continue + } + if response.Records != nil { + for _, record := range *response.Records { + projectRecord, ok := projectRecordFromInternal(record) + if !ok { + continue + } + if !recordMatchesScope(projectRecord, organization, project) { + continue + } + out[projectRecord.Checksum] = append(out[projectRecord.Checksum], projectRecord) + } + } + } + return out, nil +} + +func (manager *Manager) ListProjectFileUsage(ctx context.Context, authorizationHeader string, organization string, project string, inactiveDays int) (map[string]FileUsage, error) { + out := make(map[string]FileUsage) + offset := 0 + for { + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) + params.Set("limit", "1000") + params.Set("offset", strconv.Itoa(offset)) + if inactiveDays > 0 { + params.Set("inactive_days", strconv.Itoa(inactiveDays)) + } + var response metricsapi.MetricsListResponse + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodGet, "/index/v1/metrics/files", params, nil, &response); err != nil { + return nil, fmt.Errorf("list syfon metrics files offset %d: %w", offset, err) + } + items := []metricsapi.FileUsage{} + if response.Data != nil { + items = *response.Data + } + if len(items) == 0 { + break + } + for _, item := range items { + objectID := strings.TrimSpace(stringValue(item.ObjectId)) + if objectID == "" { + continue + } + out[objectID] = FileUsage{ + ObjectID: objectID, + Name: stringValue(item.Name), + Size: int64Value(item.Size), + DownloadCount: int64Value(item.DownloadCount), + UploadCount: int64Value(item.UploadCount), + LastAccessTime: item.LastAccessTime, + LastDownloadTime: item.LastDownloadTime, + LastUploadTime: item.LastUploadTime, + } + } + if len(items) < 1000 { + break + } + offset += len(items) + } + return out, nil +} + +func (manager *Manager) ListProjectFileUsageByObjectIDs(ctx context.Context, authorizationHeader string, organization string, project string, objectIDs []string, inactiveDays int) (map[string]FileUsage, error) { + out := make(map[string]FileUsage) + ids := uniqueNonEmptyStrings(objectIDs) + if len(ids) == 0 { + return out, nil + } + params := url.Values{} + params.Set("organization", strings.TrimSpace(organization)) + params.Set("project", strings.TrimSpace(project)) + request := struct { + ObjectIDs []string `json:"object_ids"` + InactiveDays *int `json:"inactive_days,omitempty"` + }{ + ObjectIDs: ids, + } + if inactiveDays > 0 { + request.InactiveDays = &inactiveDays + } + var response metricsapi.MetricsListResponse + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/index/v1/metrics/files/bulk", params, request, &response); err != nil { + return nil, fmt.Errorf("bulk list syfon metrics files: %w", err) + } + if response.Data == nil { + return out, nil + } + for _, item := range *response.Data { + objectID := strings.TrimSpace(stringValue(item.ObjectId)) + if objectID == "" { + continue + } + out[objectID] = FileUsage{ + ObjectID: objectID, + Name: stringValue(item.Name), + Size: int64Value(item.Size), + DownloadCount: int64Value(item.DownloadCount), + UploadCount: int64Value(item.UploadCount), + LastAccessTime: item.LastAccessTime, + LastDownloadTime: item.LastDownloadTime, + LastUploadTime: item.LastUploadTime, + } + } + return out, nil +} + +func (manager *Manager) BulkDeleteObjects(ctx context.Context, authorizationHeader string, objectIDs []string, deleteStorageData bool) error { + normalized := dedupeStrings(objectIDs) + if len(normalized) == 0 { + return nil + } + resp, err := manager.drsClient(authorizationHeader) + if err != nil { + return err + } + deleteMetadata := true + requestBody := drsapi.BulkDeleteObjectsJSONRequestBody{ + BulkObjectIds: normalized, + DeleteObjectMetadata: &deleteMetadata, + DeleteStorageData: &deleteStorageData, + } + response, err := resp.BulkDeleteObjectsWithResponse(ctx, requestBody) + if err != nil { + return fmt.Errorf("bulk delete syfon objects: %w", err) + } + if response.StatusCode() != http.StatusOK && response.StatusCode() != http.StatusNoContent { + return fmt.Errorf("bulk delete syfon objects failed with status %d", response.StatusCode()) + } + return nil +} + +func (manager *Manager) RegisterProjectObjects(ctx context.Context, authorizationHeader string, candidates []ProjectObjectRegistration) ([]ProjectObjectRegistrationResult, error) { + if len(candidates) == 0 { + return []ProjectObjectRegistrationResult{}, nil + } + client, err := manager.drsClient(authorizationHeader) + if err != nil { + return nil, err + } + request := drsapi.RegisterObjectsJSONRequestBody{Candidates: make([]drsapi.DrsObjectCandidate, 0, len(candidates))} + for _, candidate := range candidates { + name := strings.TrimSpace(candidate.Name) + checksum := normalizeSHA256(candidate.Checksum) + accessURLs := uniqueNonEmptyStrings(candidate.AccessURLs) + if name == "" || checksum == "" || candidate.Size < 0 || len(accessURLs) == 0 { + return nil, fmt.Errorf("register Syfon object: candidate is missing a name, SHA-256, size, or access URL") + } + accessMethods := make([]drsapi.AccessMethod, 0, len(accessURLs)) + for index, accessURL := range accessURLs { + accessID := fmt.Sprintf("s3-%d", index+1) + accessURLCopy := accessURL + accessMethods = append(accessMethods, drsapi.AccessMethod{ + AccessId: &accessID, + AccessUrl: &struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: accessURLCopy}, + Type: drsapi.AccessMethodType("s3"), + }) + } + controlledAccess := uniqueNonEmptyStrings(candidate.ControlledAccess) + request.Candidates = append(request.Candidates, drsapi.DrsObjectCandidate{ + Name: &name, + Size: candidate.Size, + Checksums: []drsapi.Checksum{{Type: "sha256", Checksum: checksum}}, + ControlledAccess: &controlledAccess, + AccessMethods: &accessMethods, + }) + } + response, err := client.RegisterObjectsWithResponse(ctx, request) + if err != nil { + return nil, fmt.Errorf("register Syfon objects: %w", err) + } + if response.StatusCode() != http.StatusCreated || response.JSON201 == nil { + return nil, fmt.Errorf("register Syfon objects failed with status %d: %s", response.StatusCode(), strings.TrimSpace(string(response.Body))) + } + if len(response.JSON201.Objects) != len(candidates) { + return nil, fmt.Errorf("register Syfon objects: expected %d objects, received %d", len(candidates), len(response.JSON201.Objects)) + } + results := make([]ProjectObjectRegistrationResult, len(response.JSON201.Objects)) + for index, object := range response.JSON201.Objects { + results[index] = ProjectObjectRegistrationResult{ObjectID: strings.TrimSpace(object.Id)} + } + return results, nil +} + +func (manager *Manager) DeleteProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, objectURLs []string) ([]ProjectBucketDeleteResult, error) { + requestBody := struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + ObjectURLs []string `json:"object_urls"` + }{ + Organization: strings.TrimSpace(organization), + Project: strings.TrimSpace(project), + ObjectURLs: dedupeStrings(objectURLs), + } + if len(requestBody.ObjectURLs) == 0 { + return []ProjectBucketDeleteResult{}, nil + } + var response struct { + Items []struct { + ObjectURL string `json:"object_url"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-bucket/delete", nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("delete syfon project bucket objects: %w", err) + } + out := make([]ProjectBucketDeleteResult, 0, len(response.Items)) + for _, item := range response.Items { + out = append(out, ProjectBucketDeleteResult{ + ObjectURL: strings.TrimSpace(item.ObjectURL), + Status: strings.TrimSpace(item.Status), + Error: strings.TrimSpace(item.Error), + }) + } + return out, nil +} + +func (manager *Manager) BulkUpdateAccessMethods(ctx context.Context, authorizationHeader string, updates map[string][]ProjectAccessMethod) error { + if len(updates) == 0 { + return nil + } + resp, err := manager.drsClient(authorizationHeader) + if err != nil { + return err + } + const batchSize = 200 + objectIDs := make([]string, 0, len(updates)) + for objectID := range updates { + if trimmed := strings.TrimSpace(objectID); trimmed != "" { + objectIDs = append(objectIDs, trimmed) + } + } + sort.Strings(objectIDs) + for start := 0; start < len(objectIDs); start += batchSize { + end := start + batchSize + if end > len(objectIDs) { + end = len(objectIDs) + } + body := drsapi.BulkUpdateAccessMethodsJSONRequestBody{ + Updates: make([]struct { + AccessMethods []drsapi.AccessMethod `json:"access_methods"` + ObjectId string `json:"object_id"` + }, 0, end-start), + } + for _, objectID := range objectIDs[start:end] { + methods := projectAccessMethodsToDRS(updates[objectID]) + if len(methods) == 0 { + continue + } + body.Updates = append(body.Updates, struct { + AccessMethods []drsapi.AccessMethod `json:"access_methods"` + ObjectId string `json:"object_id"` + }{ + ObjectId: objectID, + AccessMethods: methods, + }) + } + if len(body.Updates) == 0 { + continue + } + response, err := resp.BulkUpdateAccessMethodsWithResponse(ctx, body) + if err != nil { + return fmt.Errorf("bulk update syfon access methods: %w", err) + } + if response.StatusCode() != http.StatusOK { + return fmt.Errorf("bulk update syfon access methods failed with status %d", response.StatusCode()) + } + } + return nil +} + +func (manager *Manager) BulkProbeStorageObjects(ctx context.Context, authorizationHeader string, items []BulkStorageProbeItem) ([]BulkStorageProbeResult, error) { + return manager.bulkStorageObjectRequest(ctx, authorizationHeader, "/data/inspect/bulk", items, false) +} + +func (manager *Manager) bulkStorageObjectRequest(ctx context.Context, authorizationHeader string, requestPath string, items []BulkStorageProbeItem, includeExpectedName bool) ([]BulkStorageProbeResult, error) { + if len(items) == 0 { + return []BulkStorageProbeResult{}, nil + } + started := time.Now() + requestItems := append([]BulkStorageProbeItem(nil), items...) + duplicateCount := 0 + resultKeyByOriginalID := make(map[string]string, len(requestItems)) + for _, item := range requestItems { + resultKeyByOriginalID[strings.TrimSpace(item.ID)] = strings.TrimSpace(item.ID) + } + if includeExpectedName { + var deduped map[string]string + items, deduped = dedupeBulkStorageProbeItems(items, includeExpectedName) + duplicateCount = len(requestItems) - len(items) + resultKeyByOriginalID = deduped + } + batchCount := (len(items) + bulkStorageProbeBatchSize - 1) / bulkStorageProbeBatchSize + log.Printf("INFO: syfon_bulk_storage_request_start path=%s items=%d unique_items=%d duplicate_items=%d batch_size=%d batches=%d concurrency=%d include_expected_name=%t", requestPath, len(requestItems), len(items), duplicateCount, bulkStorageProbeBatchSize, batchCount, bulkStorageProbeConcurrency, includeExpectedName) + resultsByID := make(map[string]BulkStorageProbeResult, len(items)) + var mu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, bulkStorageProbeConcurrency) + var firstErr error + for start := 0; start < len(items); start += bulkStorageProbeBatchSize { + end := start + bulkStorageProbeBatchSize + if end > len(items) { + end = len(items) + } + batchStart := start + batch := append([]BulkStorageProbeItem(nil), items[start:end]...) + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + batchStarted := time.Now() + results, err := manager.probeStorageObjectBatch(ctx, authorizationHeader, requestPath, batch, includeExpectedName) + batchMs := time.Since(batchStarted).Milliseconds() + mu.Lock() + defer mu.Unlock() + if err != nil { + log.Printf("INFO: syfon_bulk_storage_request_batch path=%s batch_start=%d batch_items=%d duration_ms=%d error=%q", requestPath, batchStart, len(batch), batchMs, err.Error()) + if firstErr == nil { + firstErr = fmt.Errorf("bulk syfon storage object request %s batch starting at %d: %w", requestPath, batchStart, err) + } + return + } + for _, result := range results { + resultsByID[strings.TrimSpace(result.ID)] = result + } + log.Printf("INFO: syfon_bulk_storage_request_batch path=%s batch_start=%d batch_items=%d result_items=%d duration_ms=%d", requestPath, batchStart, len(batch), len(results), batchMs) + }() + } + wg.Wait() + if firstErr != nil { + log.Printf("INFO: syfon_bulk_storage_request_done path=%s items=%d results=%d batches=%d duration_ms=%d error=%q", requestPath, len(items), len(resultsByID), batchCount, time.Since(started).Milliseconds(), firstErr.Error()) + return nil, firstErr + } + out := make([]BulkStorageProbeResult, 0, len(requestItems)) + for _, item := range requestItems { + resultKey := resultKeyByOriginalID[strings.TrimSpace(item.ID)] + if result, ok := resultsByID[resultKey]; ok { + result.ID = strings.TrimSpace(item.ID) + out = append(out, result) + } + } + log.Printf("INFO: syfon_bulk_storage_request_done path=%s items=%d unique_items=%d results=%d batches=%d duration_ms=%d", requestPath, len(requestItems), len(items), len(out), batchCount, time.Since(started).Milliseconds()) + return out, nil +} + +func dedupeBulkStorageProbeItems(items []BulkStorageProbeItem, includeExpectedName bool) ([]BulkStorageProbeItem, map[string]string) { + seen := make(map[string]BulkStorageProbeItem, len(items)) + keyByOriginalID := make(map[string]string, len(items)) + out := make([]BulkStorageProbeItem, 0, len(items)) + for _, item := range items { + key := bulkStorageProbeDedupKey(item, includeExpectedName) + originalID := strings.TrimSpace(item.ID) + if existing, ok := seen[key]; ok { + keyByOriginalID[originalID] = strings.TrimSpace(existing.ID) + continue + } + seen[key] = item + keyByOriginalID[originalID] = originalID + out = append(out, item) + } + return out, keyByOriginalID +} + +func bulkStorageProbeDedupKey(item BulkStorageProbeItem, includeExpectedName bool) string { + parts := []string{strings.TrimSpace(item.ObjectURL)} + if item.ExpectedSizeBytes == nil { + parts = append(parts, "") + } else { + parts = append(parts, strconv.FormatInt(*item.ExpectedSizeBytes, 10)) + } + if includeExpectedName { + parts = append(parts, strings.TrimSpace(item.ExpectedName)) + } else { + parts = append(parts, strings.TrimSpace(item.ExpectedSHA256)) + } + return strings.Join(parts, "\x00") +} + +func (manager *Manager) probeStorageObjectBatch(ctx context.Context, authorizationHeader string, requestPath string, items []BulkStorageProbeItem, includeExpectedName bool) ([]BulkStorageProbeResult, error) { + requestBody := struct { + Items []struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` + ExpectedName string `json:"expected_name,omitempty"` + } `json:"items"` + }{Items: make([]struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` + ExpectedName string `json:"expected_name,omitempty"` + }, 0, len(items))} + for _, item := range items { + row := struct { + ID string `json:"id,omitempty"` + ObjectURL string `json:"object_url,omitempty"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes,omitempty"` + ExpectedSHA256 string `json:"expected_sha256,omitempty"` + ExpectedName string `json:"expected_name,omitempty"` + }{ + ID: strings.TrimSpace(item.ID), + ObjectURL: strings.TrimSpace(item.ObjectURL), + ExpectedSizeBytes: item.ExpectedSizeBytes, + ExpectedSHA256: strings.TrimSpace(item.ExpectedSHA256), + } + if includeExpectedName { + row.ExpectedName = strings.TrimSpace(item.ExpectedName) + } + requestBody.Items = append(requestBody.Items, row) + } + var response struct { + Items []struct { + ID string `json:"id"` + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Path string `json:"path"` + Exists bool `json:"exists"` + Status string `json:"status"` + Error string `json:"error"` + ErrorKind string `json:"error_kind"` + SizeBytes *int64 `json:"size_bytes"` + MetaSHA256 string `json:"meta_sha256"` + ETag string `json:"etag"` + LastModified string `json:"last_modified"` + ValidationStatus string `json:"validation_status"` + SizeMatch *bool `json:"size_match"` + NameMatch *bool `json:"name_match"` + SHA256Match *bool `json:"sha256_match"` + ValidationMismatches []string `json:"validation_mismatches"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, requestPath, nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("bulk syfon storage object request %s: %w", requestPath, err) + } + out := make([]BulkStorageProbeResult, 0, len(response.Items)) + for _, item := range response.Items { + out = append(out, BulkStorageProbeResult{ + ID: strings.TrimSpace(item.ID), + ObjectURL: strings.TrimSpace(item.ObjectURL), + Provider: strings.TrimSpace(item.Provider), + Bucket: strings.TrimSpace(item.Bucket), + Key: strings.TrimSpace(item.Key), + Path: strings.TrimSpace(item.Path), + Exists: item.Exists, + Status: strings.TrimSpace(item.Status), + Error: strings.TrimSpace(item.Error), + ErrorKind: strings.TrimSpace(item.ErrorKind), + SizeBytes: item.SizeBytes, + MetaSHA256: strings.TrimSpace(item.MetaSHA256), + ETag: strings.TrimSpace(item.ETag), + LastModified: strings.TrimSpace(item.LastModified), + ValidationStatus: strings.TrimSpace(item.ValidationStatus), + SizeMatch: item.SizeMatch, + NameMatch: item.NameMatch, + SHA256Match: item.SHA256Match, + ValidationMismatches: append([]string(nil), item.ValidationMismatches...), + }) + } + return out, nil +} + +func (manager *Manager) BulkListStorageObjects(ctx context.Context, authorizationHeader string, items []BulkStorageProbeItem) ([]BulkStorageProbeResult, error) { + return manager.bulkStorageObjectRequest(ctx, authorizationHeader, "/data/inspect/bulk-list", items, true) +} + +func (manager *Manager) ListProjectBucketSummary(ctx context.Context, authorizationHeader string, organization string, project string, mode string) (*ProjectBucketSummary, error) { + started := time.Now() + trimmedOrg := strings.TrimSpace(organization) + trimmedProject := strings.TrimSpace(project) + trimmedMode := strings.TrimSpace(mode) + log.Printf("INFO: syfon_project_bucket_summary_request_start organization=%s project=%s mode=%s", trimmedOrg, trimmedProject, trimmedMode) + requestBody := struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + Mode string `json:"mode,omitempty"` + }{ + Organization: trimmedOrg, + Project: trimmedProject, + Mode: trimmedMode, + } + var response struct { + Summary *struct { + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + Exists bool `json:"exists"` + ObjectCount int `json:"object_count"` + TotalBytes int64 `json:"total_bytes"` + ComputedAt string `json:"computed_at"` + Mode string `json:"mode"` + } `json:"summary"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, "/data/inspect/project-bucket", nil, requestBody, &response); err != nil { + log.Printf("INFO: syfon_project_bucket_summary_request_done organization=%s project=%s mode=%s duration_ms=%d error=%q", trimmedOrg, trimmedProject, trimmedMode, time.Since(started).Milliseconds(), err.Error()) + return nil, fmt.Errorf("inspect syfon project bucket summary: %w", err) + } + if response.Summary == nil { + log.Printf("INFO: syfon_project_bucket_summary_request_done organization=%s project=%s mode=%s duration_ms=%d error=%q", trimmedOrg, trimmedProject, trimmedMode, time.Since(started).Milliseconds(), "response summary is missing") + return nil, fmt.Errorf("inspect syfon project bucket summary: response summary is missing") + } + log.Printf("INFO: syfon_project_bucket_summary_request_done organization=%s project=%s mode=%s exists=%t object_count=%d total_bytes=%d duration_ms=%d", trimmedOrg, trimmedProject, trimmedMode, response.Summary.Exists, response.Summary.ObjectCount, response.Summary.TotalBytes, time.Since(started).Milliseconds()) + return &ProjectBucketSummary{ + ObjectURL: strings.TrimSpace(response.Summary.ObjectURL), + Provider: strings.TrimSpace(response.Summary.Provider), + Bucket: strings.TrimSpace(response.Summary.Bucket), + Prefix: strings.TrimSpace(response.Summary.Prefix), + Exists: response.Summary.Exists, + ObjectCount: response.Summary.ObjectCount, + TotalBytes: response.Summary.TotalBytes, + ComputedAt: strings.TrimSpace(response.Summary.ComputedAt), + Mode: strings.TrimSpace(response.Summary.Mode), + }, nil +} + +func (manager *Manager) ListProjectBucketObjects(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]ProjectBucketObject, error) { + return manager.listProjectBucketObjects(ctx, authorizationHeader, "/data/inspect/project-bucket", organization, project, pathPrefix) +} + +func (manager *Manager) ListProjectBucketInventory(ctx context.Context, authorizationHeader string, organization string, project string, pathPrefix string) ([]ProjectBucketObject, error) { + return manager.listProjectBucketObjects(ctx, authorizationHeader, "/data/inspect/project-bucket/inventory", organization, project, pathPrefix) +} + +func (manager *Manager) listProjectBucketObjects(ctx context.Context, authorizationHeader string, requestPath string, organization string, project string, pathPrefix string) ([]ProjectBucketObject, error) { + requestBody := struct { + Organization string `json:"organization,omitempty"` + Project string `json:"project,omitempty"` + Mode string `json:"mode,omitempty"` + PathPrefix string `json:"path_prefix,omitempty"` + }{ + Organization: strings.TrimSpace(organization), + Project: strings.TrimSpace(project), + Mode: "items", + PathPrefix: strings.Trim(strings.TrimSpace(pathPrefix), "/"), + } + var response struct { + Items []struct { + ObjectURL string `json:"object_url"` + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + MetaSHA256 string `json:"meta_sha256"` + ETag string `json:"etag"` + LastModified string `json:"last_modified"` + } `json:"items"` + } + if err := manager.requestJSON(ctx, authorizationHeader, http.MethodPost, requestPath, nil, requestBody, &response); err != nil { + return nil, fmt.Errorf("list syfon project bucket objects: %w", err) + } + out := make([]ProjectBucketObject, 0, len(response.Items)) + for _, item := range response.Items { + out = append(out, ProjectBucketObject{ + ObjectURL: strings.TrimSpace(item.ObjectURL), + Provider: strings.TrimSpace(item.Provider), + Bucket: strings.TrimSpace(item.Bucket), + Key: strings.TrimSpace(item.Key), + Path: strings.TrimSpace(item.Path), + SizeBytes: item.SizeBytes, + MetaSHA256: strings.TrimSpace(item.MetaSHA256), + ETag: strings.TrimSpace(item.ETag), + LastModified: strings.TrimSpace(item.LastModified), + }) + } + return out, nil +} + func (manager *Manager) bucketsService(authorizationHeader string) (*syfonservices.BucketsService, error) { clientBaseURL, err := manager.clientBaseURL() if err != nil { @@ -160,6 +1117,36 @@ func (manager *Manager) bucketsService(authorizationHeader string) (*syfonservic return syfonservices.NewBucketsService(client), nil } +func (manager *Manager) drsClient(authorizationHeader string) (*drsapi.ClientWithResponses, error) { + clientBaseURL, err := manager.drsAPIBaseURL() + if err != nil { + return nil, err + } + client, err := drsapi.NewClientWithResponses(clientBaseURL, + drsapi.WithHTTPClient(manager.client), + drsapi.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error { + req.Header.Set("Authorization", authorizationHeader) + req.Header.Set(refreshAuthzHeader, "true") + return nil + }), + ) + if err != nil { + return nil, fmt.Errorf("create syfon drs client: %w", err) + } + return client, nil +} + +func (manager *Manager) drsAPIBaseURL() (string, error) { + clientBaseURL, err := manager.clientBaseURL() + if err != nil { + return "", err + } + if strings.HasSuffix(clientBaseURL, "/ga4gh/drs/v1") { + return clientBaseURL, nil + } + return strings.TrimRight(clientBaseURL+"/ga4gh/drs/v1", "/"), nil +} + func (manager *Manager) clientBaseURL() (string, error) { if manager.baseURL == "" { return "", fmt.Errorf("SYFON_DATA_API_BASE_URL is not configured") @@ -223,3 +1210,208 @@ func bucketPath(provider string, bucket string, prefix string) string { return "s3://" + cleanBucket + "/" + cleanPrefix } } + +func (manager *Manager) requestJSON(ctx context.Context, authorizationHeader string, method string, requestPath string, query url.Values, requestBody any, out any) error { + baseURL, err := manager.clientBaseURL() + if err != nil { + return err + } + queryURL, err := url.Parse(strings.TrimRight(baseURL, "/") + requestPath) + if err != nil { + return fmt.Errorf("parse syfon request url: %w", err) + } + if len(query) > 0 { + queryURL.RawQuery = query.Encode() + } + var body io.Reader + if requestBody != nil { + bodyBytes, marshalErr := json.Marshal(requestBody) + if marshalErr != nil { + return fmt.Errorf("marshal syfon request body: %w", marshalErr) + } + body = strings.NewReader(string(bodyBytes)) + } + req, err := http.NewRequestWithContext(ctx, method, queryURL.String(), body) + if err != nil { + return fmt.Errorf("build syfon request: %w", err) + } + req.Header.Set("Authorization", authorizationHeader) + req.Header.Set(refreshAuthzHeader, "true") + if requestBody != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := manager.client.Do(req) + if err != nil { + return fmt.Errorf("request syfon %s %s: %w", method, requestPath, err) + } + defer resp.Body.Close() + if resp.StatusCode >= http.StatusBadRequest { + bodyBytes, _ := io.ReadAll(resp.Body) + return &HTTPError{ + Method: method, + Path: requestPath, + Status: resp.StatusCode, + Body: strings.TrimSpace(string(bodyBytes)), + } + } + if out == nil { + return nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("decode syfon %s %s response: %w", method, requestPath, err) + } + return nil +} + +func projectRecordFromInternal(record internalapi.InternalRecord) (ProjectRecord, bool) { + checksum := "" + if record.Hashes != nil { + checksum = normalizeSHA256((*record.Hashes)["sha256"]) + } + if checksum == "" { + return ProjectRecord{}, false + } + accessURLs := make([]string, 0) + accessMethods := make([]ProjectAccessMethod, 0) + if record.AccessMethods != nil { + for _, method := range *record.AccessMethods { + projectMethod := ProjectAccessMethod{ + Type: strings.TrimSpace(string(method.Type)), + Headers: []string{}, + } + if method.AccessId != nil { + projectMethod.AccessID = strings.TrimSpace(*method.AccessId) + } + if method.AccessUrl != nil { + projectMethod.URL = strings.TrimSpace(method.AccessUrl.Url) + accessURLs = append(accessURLs, projectMethod.URL) + if method.AccessUrl.Headers != nil { + projectMethod.Headers = append([]string(nil), (*method.AccessUrl.Headers)...) + } + } + accessMethods = append(accessMethods, projectMethod) + } + } + return ProjectRecord{ + ObjectID: strings.TrimSpace(record.Did), + Checksum: checksum, + Organization: stringValue(record.Organization), + Project: stringValue(record.Project), + Size: int64Value(record.Size), + UpdatedAt: parseOptionalTime(record.UpdatedTime), + CreatedAt: parseOptionalTime(record.CreatedTime), + AccessURLs: accessURLs, + AccessMethods: accessMethods, + }, true +} + +func projectAccessMethodsToDRS(methods []ProjectAccessMethod) []drsapi.AccessMethod { + out := make([]drsapi.AccessMethod, 0, len(methods)) + for _, method := range methods { + accessMethod := drsapi.AccessMethod{} + if trimmed := strings.TrimSpace(method.AccessID); trimmed != "" { + accessMethod.AccessId = &trimmed + } + if trimmed := strings.TrimSpace(method.Type); trimmed != "" { + accessMethod.Type = drsapi.AccessMethodType(trimmed) + } + if trimmed := strings.TrimSpace(method.URL); trimmed != "" { + accessURL := struct { + Headers *[]string `json:"headers,omitempty"` + Url string `json:"url"` + }{Url: trimmed} + if len(method.Headers) > 0 { + headers := append([]string(nil), method.Headers...) + accessURL.Headers = &headers + } + accessMethod.AccessUrl = &accessURL + } + out = append(out, accessMethod) + } + return out +} + +func stringValue(value *string) string { + if value == nil { + return "" + } + return strings.TrimSpace(*value) +} + +func int64Value(value *int64) int64 { + if value == nil { + return 0 + } + return *value +} + +func parseOptionalTime(value *string) *time.Time { + if value == nil || strings.TrimSpace(*value) == "" { + return nil + } + parsed, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(*value)) + if err != nil { + return nil + } + parsed = parsed.UTC() + return &parsed +} + +func optionalString(value string) *string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return nil + } + return &trimmed +} + +func dedupeChecksums(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + normalized := normalizeSHA256(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + +func recordMatchesScope(record ProjectRecord, organization string, project string) bool { + recordOrg := strings.TrimSpace(record.Organization) + recordProject := strings.TrimSpace(record.Project) + if recordOrg == "" && recordProject == "" { + return true + } + return strings.EqualFold(recordOrg, organization) && strings.EqualFold(recordProject, project) +} + +func dedupeStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(strings.TrimPrefix(value, "sha256:")) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func normalizeSHA256(value string) string { + trimmed := strings.TrimSpace(strings.TrimPrefix(value, "sha256:")) + if trimmed == "" { + return "" + } + return strings.ToLower(trimmed) +} diff --git a/internal/integrations/syfon/adapter_test.go b/internal/integrations/syfon/adapter_test.go new file mode 100644 index 0000000..bfe8ce3 --- /dev/null +++ b/internal/integrations/syfon/adapter_test.go @@ -0,0 +1,645 @@ +package syfon + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "reflect" + "strconv" + "strings" + "testing" +) + +func TestGetProjectMetricsSummaryReadsRecordValidator(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodGet || r.URL.Path != "/index/v1/metrics/summary" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + if got := r.URL.Query().Get("organization"); got != "org" { + t.Fatalf("expected organization query, got %q", got) + } + if got := r.URL.Query().Get("project"); got != "proj" { + t.Fatalf("expected project query, got %q", got) + } + body := []byte(`{"total_files":99,"record_count":42,"record_latest_updated_time":"2026-07-02T00:00:00Z","record_revision":"rev-1"}`) + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + summary, err := manager.GetProjectMetricsSummary(context.Background(), "Bearer token", "org", "proj") + if err != nil { + t.Fatalf("GetProjectMetricsSummary returned error: %v", err) + } + if summary == nil { + t.Fatal("expected metrics summary") + } + if summary.RecordCount != 42 || summary.RecordLatestUpdatedTime != "2026-07-02T00:00:00Z" || summary.RecordRevision != "rev-1" { + t.Fatalf("unexpected metrics summary: %+v", summary) + } +} + +func TestListProjectFileUsageByObjectIDsUsesBulkEndpoint(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/index/v1/metrics/files/bulk" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + if got := r.URL.Query().Get("organization"); got != "org" { + t.Fatalf("expected organization query, got %q", got) + } + if got := r.URL.Query().Get("project"); got != "proj" { + t.Fatalf("expected project query, got %q", got) + } + var req struct { + ObjectIDs []string `json:"object_ids"` + InactiveDays *int `json:"inactive_days"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if !reflect.DeepEqual(req.ObjectIDs, []string{"obj-a", "obj-b"}) { + t.Fatalf("unexpected object ids: %+v", req.ObjectIDs) + } + if req.InactiveDays == nil || *req.InactiveDays != 30 { + t.Fatalf("unexpected inactive days: %+v", req.InactiveDays) + } + body := []byte(`{"data":[{"object_id":"obj-a","name":"a.txt","size":100,"download_count":3,"upload_count":1}]}`) + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + usage, err := manager.ListProjectFileUsageByObjectIDs(context.Background(), "Bearer token", "org", "proj", []string{"obj-a", "obj-b", "obj-a", ""}, 30) + if err != nil { + t.Fatalf("ListProjectFileUsageByObjectIDs returned error: %v", err) + } + if len(usage) != 1 || usage["obj-a"].DownloadCount != 3 || usage["obj-a"].UploadCount != 1 { + t.Fatalf("unexpected usage response: %+v", usage) + } +} + +func TestListProjectBucketInventoryUsesInventoryEndpoint(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/project-bucket/inventory" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Organization string `json:"organization"` + Project string `json:"project"` + Mode string `json:"mode"` + PathPrefix string `json:"path_prefix"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if req.Organization != "org" || req.Project != "proj" || req.Mode != "items" || req.PathPrefix != "CONFIG" { + t.Fatalf("unexpected request body: %+v", req) + } + body, err := json.Marshal(map[string]any{"items": []map[string]any{{ + "object_url": "s3://bucket/root/CONFIG/a.bin", + "provider": "s3", + "bucket": "bucket", + "key": "root/CONFIG/a.bin", + "path": "CONFIG/a.bin", + "size_bytes": 123, + "etag": "etag-1", + "last_modified": "2026-07-07T18:00:00Z", + }}}) + if err != nil { + t.Fatalf("marshal response: %v", err) + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: make(http.Header)}, nil + })} + manager := NewManager("http://syfon", client) + + items, err := manager.ListProjectBucketInventory(context.Background(), "Bearer token", "org", "proj", "CONFIG") + if err != nil { + t.Fatalf("ListProjectBucketInventory returned error: %v", err) + } + if len(items) != 1 || items[0].ObjectURL != "s3://bucket/root/CONFIG/a.bin" || items[0].SizeBytes != 123 { + t.Fatalf("unexpected inventory items: %+v", items) + } +} + +func TestBulkGetProjectRecordsByChecksumReadsResultsMap(t *testing.T) { + t.Helper() + + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/index/bulk/hashes" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Hashes []string `json:"hashes"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if !reflect.DeepEqual(req.Hashes, []string{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }) { + t.Fatalf("unexpected hashes payload: %#v", req.Hashes) + } + + org := "org" + project := "proj" + sizeA := int64(100) + sizeB := int64(200) + hashesA := map[string]string{"sha256": "sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} + hashesB := map[string]string{"sha256": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} + accessURLA := "s3://bucket/a" + accessURLB := "s3://bucket/b" + + response := map[string]any{ + "results": map[string]any{ + req.Hashes[0]: []any{ + map[string]any{ + "did": "obj-a", + "organization": org, + "project": project, + "size": sizeA, + "hashes": hashesA, + "access_methods": []any{ + map[string]any{ + "access_url": map[string]any{"url": accessURLA}, + }, + }, + }, + }, + req.Hashes[1]: []any{ + map[string]any{ + "did": "obj-b", + "organization": org, + "project": project, + "size": sizeB, + "hashes": hashesB, + "access_methods": []any{ + map[string]any{ + "access_url": map[string]any{"url": accessURLB}, + }, + }, + }, + }, + }, + } + body, err := json.Marshal(response) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + records, err := manager.BulkGetProjectRecordsByChecksum( + context.Background(), + "Bearer token", + "org", + "proj", + []string{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + ) + if err != nil { + t.Fatalf("BulkGetProjectRecordsByChecksum returned error: %v", err) + } + + if len(records) != 2 { + t.Fatalf("expected 2 checksum groups, got %#v", records) + } + if len(records["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]) != 1 { + t.Fatalf("expected first checksum match, got %#v", records) + } + if len(records["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"]) != 1 { + t.Fatalf("expected second checksum match, got %#v", records) + } + if got := records["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"][0].ObjectID; got != "obj-a" { + t.Fatalf("expected obj-a, got %q", got) + } + if got := records["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"][0].AccessURLs; !reflect.DeepEqual(got, []string{"s3://bucket/b"}) { + t.Fatalf("unexpected access urls: %#v", got) + } +} + +func TestBulkGetProjectRecordsByChecksumAllowsLegacyUnscopedResults(t *testing.T) { + t.Helper() + + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + response := map[string]any{ + "results": map[string]any{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": []any{ + map[string]any{ + "did": "obj-a", + "size": 100, + "hashes": map[string]string{"sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + }, + }, + }, + } + body, err := json.Marshal(response) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + records, err := manager.BulkGetProjectRecordsByChecksum( + context.Background(), + "Bearer token", + "org", + "proj", + []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + ) + if err != nil { + t.Fatalf("BulkGetProjectRecordsByChecksum returned error: %v", err) + } + if len(records["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]) != 1 { + t.Fatalf("expected fallback match for legacy unscoped result, got %#v", records) + } +} + +func TestBulkDeleteObjectsUsesBulkDRSDeleteEndpoint(t *testing.T) { + t.Helper() + + var requests []string + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + requests = append(requests, r.Method+" "+r.URL.Path) + if r.Method != http.MethodPut { + t.Fatalf("unexpected method %s", r.Method) + } + if r.URL.Path != "/ga4gh/drs/v1/objects/delete" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + + var req struct { + BulkObjectIDs []string `json:"bulk_object_ids"` + DeleteObjectMetadata *bool `json:"delete_object_metadata"` + DeleteStorageData *bool `json:"delete_storage_data"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if !reflect.DeepEqual(req.BulkObjectIDs, []string{"obj-a", "obj-b"}) { + t.Fatalf("unexpected bulk object ids: %#v", req.BulkObjectIDs) + } + if req.DeleteObjectMetadata == nil || !*req.DeleteObjectMetadata { + t.Fatalf("expected delete_object_metadata=true, got %#v", req.DeleteObjectMetadata) + } + if req.DeleteStorageData == nil || !*req.DeleteStorageData { + t.Fatalf("expected delete_storage_data=true, got %#v", req.DeleteStorageData) + } + + return &http.Response{ + StatusCode: http.StatusNoContent, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(nil)), + }, nil + })} + + manager := NewManager("http://syfon.example/data", client) + if err := manager.BulkDeleteObjects( + context.Background(), + "Bearer token", + []string{"obj-a", "obj-b", "obj-a"}, + true, + ); err != nil { + t.Fatalf("BulkDeleteObjects returned error: %v", err) + } + + expected := []string{"PUT /ga4gh/drs/v1/objects/delete"} + if !reflect.DeepEqual(requests, expected) { + t.Fatalf("unexpected delete requests: %#v", requests) + } +} + +func TestRegisterProjectObjectsUsesDRSRegistrationEndpoint(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/ga4gh/drs/v1/objects/register" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer token" { + t.Fatalf("expected authorization header") + } + var request struct { + Candidates []struct { + Name string `json:"name"` + Size int64 `json:"size"` + ControlledAccess []string `json:"controlled_access"` + Checksums []struct { + Type string `json:"type"` + Checksum string `json:"checksum"` + } `json:"checksums"` + AccessMethods []struct { + AccessURL struct { + URL string `json:"url"` + } `json:"access_url"` + } `json:"access_methods"` + } `json:"candidates"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode request: %v", err) + } + if len(request.Candidates) != 1 || request.Candidates[0].Name != "a.bin" || request.Candidates[0].Size != 100 || len(request.Candidates[0].Checksums) != 1 || request.Candidates[0].Checksums[0].Type != "sha256" || request.Candidates[0].Checksums[0].Checksum != strings.Repeat("a", 64) || len(request.Candidates[0].AccessMethods) != 1 || request.Candidates[0].AccessMethods[0].AccessURL.URL != "s3://bucket/root/a.bin" { + t.Fatalf("unexpected registration request: %+v", request) + } + return &http.Response{ + StatusCode: http.StatusCreated, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"objects":[{"id":"obj-created"}]}`)), + }, nil + })} + manager := NewManager("http://syfon.example", client) + results, err := manager.RegisterProjectObjects(context.Background(), "Bearer token", []ProjectObjectRegistration{{ + Name: "a.bin", + Checksum: strings.Repeat("a", 64), + Size: 100, + ControlledAccess: []string{"/organization/org/project/proj"}, + AccessURLs: []string{"s3://bucket/root/a.bin"}, + }}) + if err != nil { + t.Fatalf("RegisterProjectObjects returned error: %v", err) + } + if !reflect.DeepEqual(results, []ProjectObjectRegistrationResult{{ObjectID: "obj-created"}}) { + t.Fatalf("unexpected registration result: %+v", results) + } +} + +func TestBulkProbeStorageObjectsBatchesRequests(t *testing.T) { + t.Helper() + + requests := 0 + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/bulk" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + requests++ + var req struct { + Items []struct { + ID string `json:"id"` + ObjectURL string `json:"object_url"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes"` + ExpectedSHA256 string `json:"expected_sha256"` + } `json:"items"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if len(req.Items) == 0 || len(req.Items) > bulkStorageProbeBatchSize { + t.Fatalf("unexpected batch size %d", len(req.Items)) + } + respItems := make([]map[string]any, 0, len(req.Items)) + for _, item := range req.Items { + respItems = append(respItems, map[string]any{ + "id": item.ID, + "object_url": item.ObjectURL, + "exists": true, + "status": "present", + "validation_status": "matched", + }) + } + body, err := json.Marshal(map[string]any{"items": respItems}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + items := make([]BulkStorageProbeItem, 0, bulkStorageProbeBatchSize+1) + for i := 0; i < bulkStorageProbeBatchSize+1; i++ { + size := int64(i + 1) + items = append(items, BulkStorageProbeItem{ + ID: "item-" + strconv.Itoa(i), + ObjectURL: "s3://bucket/object-" + strconv.Itoa(i), + ExpectedSizeBytes: &size, + ExpectedSHA256: strings.Repeat("a", 64), + }) + } + + results, err := manager.BulkProbeStorageObjects(context.Background(), "Bearer token", items) + if err != nil { + t.Fatalf("BulkProbeStorageObjects returned error: %v", err) + } + if requests != 2 { + t.Fatalf("expected 2 bulk probe requests, got %d", requests) + } + if len(results) != len(items) { + t.Fatalf("expected %d results, got %d", len(items), len(results)) + } +} + +func TestBulkListStorageObjectsSendsExpectedName(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/bulk-list" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Items []struct { + ID string `json:"id"` + ObjectURL string `json:"object_url"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes"` + ExpectedName string `json:"expected_name"` + } `json:"items"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if len(req.Items) != 1 || req.Items[0].ExpectedName != "file.bin" { + t.Fatalf("expected request to include expected_name, got %+v", req.Items) + } + sizeMatch := true + nameMatch := true + body, err := json.Marshal(map[string]any{"items": []map[string]any{{ + "id": req.Items[0].ID, + "object_url": req.Items[0].ObjectURL, + "exists": true, + "status": "present", + "validation_status": "matched", + "size_match": sizeMatch, + "name_match": nameMatch, + }}}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + size := int64(17) + manager := NewManager("http://syfon.example", client) + results, err := manager.BulkListStorageObjects(context.Background(), "Bearer token", []BulkStorageProbeItem{{ + ID: "item-1", + ObjectURL: "s3://bucket/file.bin", + ExpectedSizeBytes: &size, + ExpectedName: "file.bin", + }}) + if err != nil { + t.Fatalf("BulkListStorageObjects returned error: %v", err) + } + if len(results) != 1 || results[0].NameMatch == nil || !*results[0].NameMatch { + t.Fatalf("expected parsed name_match result, got %+v", results) + } +} + +func TestBulkListStorageObjectsDeduplicatesIdenticalRequests(t *testing.T) { + requests := 0 + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/bulk-list" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + requests++ + var req struct { + Items []struct { + ID string `json:"id"` + ObjectURL string `json:"object_url"` + ExpectedSizeBytes *int64 `json:"expected_size_bytes"` + ExpectedName string `json:"expected_name"` + } `json:"items"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if len(req.Items) != 1 { + t.Fatalf("expected identical LIST items to be deduplicated, got %+v", req.Items) + } + sizeMatch := true + nameMatch := true + body, err := json.Marshal(map[string]any{"items": []map[string]any{{ + "id": req.Items[0].ID, + "object_url": req.Items[0].ObjectURL, + "exists": true, + "status": "present", + "validation_status": "matched", + "size_match": sizeMatch, + "name_match": nameMatch, + }}}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + size := int64(17) + manager := NewManager("http://syfon.example", client) + results, err := manager.BulkListStorageObjects(context.Background(), "Bearer token", []BulkStorageProbeItem{ + {ID: "item-1", ObjectURL: "s3://bucket/file.bin", ExpectedSizeBytes: &size, ExpectedName: "file.bin"}, + {ID: "item-2", ObjectURL: "s3://bucket/file.bin", ExpectedSizeBytes: &size, ExpectedName: "file.bin"}, + }) + if err != nil { + t.Fatalf("BulkListStorageObjects returned error: %v", err) + } + if requests != 1 { + t.Fatalf("expected one bulk-list request, got %d", requests) + } + if len(results) != 2 || results[0].ObjectURL != "s3://bucket/file.bin" || results[1].ObjectURL != "s3://bucket/file.bin" { + t.Fatalf("expected deduplicated response to fan out to all original items, got %+v", results) + } +} + +func TestListProjectAuditRecordsIncludesPathPrefix(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/project-records" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Organization string `json:"organization"` + Project string `json:"project"` + PathPrefix string `json:"path_prefix"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if req.Organization != "org" || req.Project != "proj" || req.PathPrefix != "CONFIG" { + t.Fatalf("unexpected request payload: %+v", req) + } + body, err := json.Marshal(map[string]any{"items": []map[string]any{}}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + if _, err := manager.ListProjectAuditRecords(context.Background(), "Bearer token", "org", "proj", "/CONFIG/"); err != nil { + t.Fatalf("ListProjectAuditRecords returned error: %v", err) + } +} + +func TestListProjectAuditRecordsFoldsAccessMethodURLsIntoAccessURLs(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodPost || r.URL.Path != "/data/inspect/project-records" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + body, err := json.Marshal(map[string]any{"items": []map[string]any{{ + "object_id": "obj-1", + "name": "file.txt", + "checksum": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "size": 10, + "access_methods": []map[string]any{ + {"access_id": "s3", "type": "s3", "url": "s3://bucket/path/file.txt"}, + {"access_id": "s3", "type": "s3", "url": "s3://bucket/path/file.txt"}, + }, + }}}) + if err != nil { + t.Fatalf("encode response: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + }, nil + })} + + manager := NewManager("http://syfon.example", client) + records, err := manager.ListProjectAuditRecords(context.Background(), "Bearer token", "org", "proj", "") + if err != nil { + t.Fatalf("ListProjectAuditRecords returned error: %v", err) + } + if len(records) != 1 { + t.Fatalf("expected one record, got %+v", records) + } + if !reflect.DeepEqual(records[0].AccessURLs, []string{"s3://bucket/path/file.txt"}) { + t.Fatalf("expected access method URL folded into access URLs, got %+v", records[0].AccessURLs) + } + if len(records[0].AccessMethods) != 2 { + t.Fatalf("expected access methods preserved, got %+v", records[0].AccessMethods) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} diff --git a/internal/presentation/interface.go b/internal/presentation/interface.go new file mode 100644 index 0000000..1e76faa --- /dev/null +++ b/internal/presentation/interface.go @@ -0,0 +1,8 @@ +package presentation + +// Manager defines the interface for project presentation HTML storage and retrieval. +type Manager interface { + Get(organization string, project string) (string, error) + Save(organization string, project string, content string) error + ProjectPresentationPath(organization string, project string) string +} diff --git a/internal/presentation/store.go b/internal/presentation/store.go new file mode 100644 index 0000000..8b9821f --- /dev/null +++ b/internal/presentation/store.go @@ -0,0 +1,70 @@ +package presentation + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +type FilesystemStore struct { + dataDir string +} + +func NewFilesystemStore(dataDir string) *FilesystemStore { + return &FilesystemStore{dataDir: strings.TrimSpace(dataDir)} +} + +func (s *FilesystemStore) projectPresentationDir(organization string) string { + return filepath.Join( + s.dataDir, + ProjectPresentationDirectory, + sanitizePathPart(strings.TrimSpace(organization)), + ) +} + +func (s *FilesystemStore) ProjectPresentationPath(organization string, project string) string { + return filepath.Join( + s.projectPresentationDir(organization), + sanitizePathPart(strings.TrimSpace(project))+"_presentation.html", + ) +} + +func (s *FilesystemStore) Get(organization string, project string) (string, error) { + if s.dataDir == "" { + return "", ErrDataDirRequired + } + data, err := os.ReadFile(s.ProjectPresentationPath(organization, project)) + if err != nil { + if os.IsNotExist(err) { + return "", ErrNoPresentation + } + return "", fmt.Errorf("read presentation HTML: %w", err) + } + return string(data), nil +} + +func (s *FilesystemStore) Save(organization string, project string, content string) error { + if s.dataDir == "" { + return ErrDataDirRequired + } + dir := s.projectPresentationDir(organization) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create presentation directory: %w", err) + } + path := s.ProjectPresentationPath(organization, project) + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, []byte(content), 0o644); err != nil { + return fmt.Errorf("write presentation HTML: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("persist presentation HTML: %w", err) + } + return nil +} + +func sanitizePathPart(value string) string { + replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_") + return replacer.Replace(value) +} diff --git a/internal/presentation/store_test.go b/internal/presentation/store_test.go new file mode 100644 index 0000000..d29a8ba --- /dev/null +++ b/internal/presentation/store_test.go @@ -0,0 +1,44 @@ +package presentation + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFilesystemStoreUsesOrgProjectFilenameLayout(t *testing.T) { + tempDir := t.TempDir() + store := NewFilesystemStore(tempDir) + + expectedPath := filepath.Join(tempDir, ProjectPresentationDirectory, "Org_A", "Project_1_presentation.html") + if got := store.ProjectPresentationPath("Org A", "Project/1"); got != expectedPath { + t.Fatalf("unexpected presentation path: got %q want %q", got, expectedPath) + } +} + +func TestFilesystemStoreSaveAndGet(t *testing.T) { + tempDir := t.TempDir() + store := NewFilesystemStore(tempDir) + + if err := store.Save("org-a", "proj-a", "

Hello

"); err != nil { + t.Fatalf("save failed: %v", err) + } + got, err := store.Get("org-a", "proj-a") + if err != nil { + t.Fatalf("get failed: %v", err) + } + if got != "

Hello

" { + t.Fatalf("unexpected content: %q", got) + } + if _, err := os.Stat(store.ProjectPresentationPath("org-a", "proj-a")); err != nil { + t.Fatalf("expected stored file: %v", err) + } +} + +func TestFilesystemStoreGetMissing(t *testing.T) { + store := NewFilesystemStore(t.TempDir()) + _, err := store.Get("org-a", "proj-a") + if err != ErrNoPresentation { + t.Fatalf("expected ErrNoPresentation, got %v", err) + } +} diff --git a/internal/presentation/types.go b/internal/presentation/types.go new file mode 100644 index 0000000..9a24a2f --- /dev/null +++ b/internal/presentation/types.go @@ -0,0 +1,12 @@ +package presentation + +import "errors" + +const ( + ProjectPresentationDirectory = "_project_presentations" +) + +var ( + ErrNoPresentation = errors.New("project has no presentation HTML") + ErrDataDirRequired = errors.New("presentation storage directory is required") +) diff --git a/internal/server/http/config/config.go b/internal/server/http/config/config.go index ca871e3..fefc2a8 100644 --- a/internal/server/http/config/config.go +++ b/internal/server/http/config/config.go @@ -145,16 +145,3 @@ func (handler *Handler) resolveProjectConfigParams(ctx fiber.Ctx) (string, strin } return handler.resolveConfigParams(ctx) } - -func mergeErrorDetails(base map[string]any, extra map[string]any) map[string]any { - if len(extra) == 0 { - return base - } - if base == nil { - base = map[string]any{} - } - for k, v := range extra { - base[k] = v - } - return base -} diff --git a/internal/server/http/directory/directory.go b/internal/server/http/directory/directory.go deleted file mode 100644 index 6ec7174..0000000 --- a/internal/server/http/directory/directory.go +++ /dev/null @@ -1,128 +0,0 @@ -package directory - -import ( - "fmt" - "net/http" - "path" - "strings" - - "github.com/bmeg/grip/gripql" - "github.com/calypr/gecko/apierror" - "github.com/calypr/gecko/internal/httputil" - servermw "github.com/calypr/gecko/internal/server/middleware" - "github.com/gofiber/fiber/v3" -) - -func (handler *Handler) registerDirectoryHandlers(app fiber.Router, authMiddleware fiber.Handler) { - app.Get("/dir", handler.handleListProjects) - app.Get("/dir/:projectId", authMiddleware, handler.handleDirGet) -} - -// handleListProjects godoc -// @Summary List authorized projects -// @Description Retrieve the set of projects visible to the current user. -// @Tags Directory -// @Produce json -// @Success 200 {array} string "Project resource paths" -// @Failure 401 {object} ErrorResponse "Unauthorized" -// @Failure 500 {object} ErrorResponse "Server error" -// @Router /dir [get] -func (handler *Handler) handleListProjects(ctx fiber.Ctx) error { - projs, errResponse := servermw.GetProjectsFromToken(ctx, servermw.NewFenceUserAccessHandler(nil), "read", "*") - if errResponse != nil { - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - q := buildListProjectsQuery(projs) - res, err := handler.gripqlClient.Traversal(ctx, &gripql.GraphQuery{Graph: handler.gripGraphName, Query: q.Statements}) - if err != nil { - errResponse = httputil.NewError(apierror.TypeGraphQueryFailed, "graph query failed", http.StatusInternalServerError, nil, &err) - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - out := []string{} - for r := range res { - renda, ok := r.GetRender().GetStructValue().AsMap()["project"].(string) - if ok { - out = append(out, renda) - } - } - return httputil.JSON(out, http.StatusOK).Write(ctx) -} - -// handleDirGet godoc -// @Summary Retrieve directory information for a project -// @Description Retrieve directory details for the given project ID and directory path. -// @Tags Directory -// @Produce json -// @Param projectId path string true "Project ID (format: program-project)" -// @Param directory query string true "Directory path (e.g. /data/my-dir)" -// @Success 200 {array} map[string]interface{} "Directory information" -// @Failure 400 {object} ErrorResponse "Invalid request body or directory path" -// @Failure 403 {object} ErrorResponse "User is not allowed on the resource path" -// @Failure 500 {object} ErrorResponse "Server error" -// @Router /dir/{projectId} [get] -func (handler *Handler) handleDirGet(ctx fiber.Ctx) error { - projectID := ctx.Params("projectId") - dirPath := ctx.Query("directory") - if dirPath == "" || !isValidPosixPath(&dirPath) { - errResponse := httputil.NewError(apierror.TypeInvalidDirectory, fmt.Sprintf("Invalid or missing Directory path: %s", dirPath), http.StatusBadRequest, map[string]any{"directory": dirPath}, nil) - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - - projectSplit := strings.Split(projectID, "-") - if len(projectSplit) != 2 { - errResponse := httputil.NewError(apierror.TypeInvalidProjectID, fmt.Sprintf("Failed to parse request body: %v", fmt.Sprintf("incorrect path %s", ctx.Path())), http.StatusNotFound, map[string]any{"project_id": projectID}, nil) - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - projectID = "/programs/" + projectSplit[0] + "/projects/" + projectSplit[1] - - q := buildDirGetQuery(projectID, dirPath) - res, err := handler.gripqlClient.Traversal(ctx, &gripql.GraphQuery{Graph: handler.gripGraphName, Query: q.Statements}) - if err != nil { - errResponse := httputil.NewError(apierror.TypeGraphQueryFailed, "graph query failed", http.StatusInternalServerError, nil, &err) - errResponse.WriteLog(handler.logger) - return errResponse.Write(ctx) - } - out := []any{} - for r := range res { - out = append(out, r.GetVertex()) - } - return httputil.JSON(out, http.StatusOK).Write(ctx) -} - -func isValidPosixPath(p *string) bool { - if strings.ContainsRune(*p, 000) || !path.IsAbs(*p) || strings.Contains(*p, "\\") { - return false - } - cleaned := path.Clean(*p) - if *p == "" || cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "/..") { - return false - } - return true -} - -func buildListProjectsQuery(projs []any) *gripql.Query { - return gripql.V(). - HasLabel("ResearchStudy"). - Has(gripql.Within("auth_resource_path", projs...)). - As("project"). - OutE("rootDir_Directory"). - Select("project"). - Distinct("auth_resource_path"). - Render(map[string]any{"project": "$project.auth_resource_path"}) -} - -func buildDirGetQuery(projectID, dirPath string) *gripql.Query { - q := gripql.V().HasLabel("ResearchStudy").Has(gripql.Eq("auth_resource_path", projectID)).OutE("rootDir_Directory").OutNull().OutNull() - if dirPath != "/" { - for splStr := range strings.SplitSeq(strings.Trim(dirPath, "/"), "/") { - q = q.Has(gripql.Eq("name", splStr)).Has(gripql.Eq("auth_resource_path", projectID)).OutNull() - } - } else { - q = q.Has(gripql.Eq("auth_resource_path", projectID)) - } - return q -} diff --git a/internal/server/http/directory/directory_test.go b/internal/server/http/directory/directory_test.go deleted file mode 100644 index 0237e40..0000000 --- a/internal/server/http/directory/directory_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package directory - -import ( - "encoding/json" - "testing" - - "github.com/bmeg/grip/gripql" - "github.com/stretchr/testify/assert" -) - -// Helper to convert query to string for assertion -func queryString(q *gripql.Query) string { - b, _ := json.Marshal(q.Statements) - return string(b) -} - -func TestBuildListProjectsQuery(t *testing.T) { - projs := []any{"PROG-PROJ1", "PROG-PROJ2"} - q := buildListProjectsQuery(projs) - - // Verify key components of the query - jsonStr := queryString(q) - assert.Contains(t, jsonStr, "ResearchStudy") - assert.Contains(t, jsonStr, "auth_resource_path") - assert.Contains(t, jsonStr, "rootDir_Directory") // Must have OutE - assert.Contains(t, jsonStr, "Distinct") // Must have Distinct -} - -func TestBuildDirGetQuery_Root(t *testing.T) { - projectId := "/programs/PROG/projects/PROJ" - dirPath := "/" - q := buildDirGetQuery(projectId, dirPath) - - jsonStr := queryString(q) - assert.Contains(t, jsonStr, "ResearchStudy") - // The query logic for root just filters by auth_resource_path - assert.Contains(t, jsonStr, "auth_resource_path") - assert.Contains(t, jsonStr, "rootDir_Directory") - // Verify it does NOT contain loop logic - assert.NotContains(t, jsonStr, "name") -} - -func TestBuildDirGetQuery_SubDir(t *testing.T) { - projectId := "/programs/PROG/projects/PROJ" - dirPath := "/data/foo" - q := buildDirGetQuery(projectId, dirPath) - - jsonStr := queryString(q) - // Verify traversal - assert.Contains(t, jsonStr, "data") - assert.Contains(t, jsonStr, "foo") - // Verify security check is present in loop (auth_resource_path appears multiple times) - // We can't easily count occurrences in JSON string without parsing, but Contains is a good smoke test - assert.Contains(t, jsonStr, "auth_resource_path") -} diff --git a/internal/server/http/directory/handler.go b/internal/server/http/directory/handler.go deleted file mode 100644 index c2351a8..0000000 --- a/internal/server/http/directory/handler.go +++ /dev/null @@ -1,23 +0,0 @@ -package directory - -import ( - "github.com/bmeg/grip/gripql" - "github.com/calypr/gecko/internal/server/http/shared" - "github.com/uc-cdis/arborist/arborist" -) - -type Handler struct { - *shared.Handler - logger arborist.Logger - gripqlClient *gripql.Client - gripGraphName string -} - -func NewHandler(sharedHandler *shared.Handler) *Handler { - return &Handler{ - Handler: sharedHandler, - logger: sharedHandler.Logger, - gripqlClient: sharedHandler.GripqlClient, - gripGraphName: sharedHandler.GripGraphName, - } -} diff --git a/internal/server/http/directory/register.go b/internal/server/http/directory/register.go deleted file mode 100644 index d361e5a..0000000 --- a/internal/server/http/directory/register.go +++ /dev/null @@ -1,18 +0,0 @@ -package directory - -import ( - "github.com/calypr/gecko/internal/server/http/shared" - servermw "github.com/calypr/gecko/internal/server/middleware" - "github.com/gofiber/fiber/v3" -) - -func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler servermw.ResourceAccessHandler) { - handler := NewHandler(sharedHandler) - if handler.GripqlClient == nil { - handler.Logger.Warning("Skipping gripql Directory endpoints — no database configured") - return - } - authMiddleware := servermw.GeneralAuth(handler.Logger, authzHandler, "read", "*") - app.Get("/dir", handler.handleListProjects) - app.Get("/dir/:projectId", authMiddleware, handler.handleDirGet) -} diff --git a/internal/server/http/git/handler.go b/internal/server/http/git/handler.go index 6ee4aa0..f8ee83a 100644 --- a/internal/server/http/git/handler.go +++ b/internal/server/http/git/handler.go @@ -3,6 +3,7 @@ package git import ( "github.com/bmeg/grip/gripql" "github.com/calypr/gecko/internal/git" + "github.com/calypr/gecko/internal/presentation" "github.com/calypr/gecko/internal/server/http/shared" "github.com/calypr/gecko/internal/thumbnail" "github.com/jmoiron/sqlx" @@ -12,30 +13,39 @@ import ( type Handler struct { *shared.Handler - db *sqlx.DB - logger arborist.Logger - jwtApp arborist.JWTDecoder - qdrantClient *qdrant.Client - gripqlClient *gripql.Client - gripGraphName string - gitService *git.GitService - projectSetup *git.SetupService - projectSync *git.ReconcileService - thumbnailStore thumbnail.Manager + db *sqlx.DB + logger arborist.Logger + jwtApp arborist.JWTDecoder + qdrantClient *qdrant.Client + gripqlClient *gripql.Client + gripGraphName string + gitService *git.GitService + projectSetup *git.SetupService + projectSync *git.ReconcileService + storageAnalytics *git.StorageAnalyticsService + thumbnailStore thumbnail.Manager + presentationStore presentation.Manager } func NewHandler(sharedHandler *shared.Handler) *Handler { + var storageAnalytics *git.StorageAnalyticsService + if sharedHandler.GitService != nil && sharedHandler.SyfonManager != nil { + storageAnalytics = git.NewStorageAnalyticsService(sharedHandler.SyfonManager) + storageAnalytics.EnableStorageChainAuditResponseCacheFromEnv() + } return &Handler{ - Handler: sharedHandler, - db: sharedHandler.DB, - logger: sharedHandler.Logger, - jwtApp: sharedHandler.JWTApp, - qdrantClient: sharedHandler.QdrantClient, - gripqlClient: sharedHandler.GripqlClient, - gripGraphName: sharedHandler.GripGraphName, - gitService: sharedHandler.GitService, - projectSetup: sharedHandler.ProjectSetup, - projectSync: sharedHandler.ProjectSync, - thumbnailStore: sharedHandler.ThumbnailStore, + Handler: sharedHandler, + db: sharedHandler.DB, + logger: sharedHandler.Logger, + jwtApp: sharedHandler.JWTApp, + qdrantClient: sharedHandler.QdrantClient, + gripqlClient: sharedHandler.GripqlClient, + gripGraphName: sharedHandler.GripGraphName, + gitService: sharedHandler.GitService, + projectSetup: sharedHandler.ProjectSetup, + projectSync: sharedHandler.ProjectSync, + storageAnalytics: storageAnalytics, + thumbnailStore: sharedHandler.ThumbnailStore, + presentationStore: sharedHandler.PresentationStore, } } diff --git a/internal/server/http/git/helpers.go b/internal/server/http/git/helpers.go index dd0d127..0c3e51b 100644 --- a/internal/server/http/git/helpers.go +++ b/internal/server/http/git/helpers.go @@ -9,14 +9,6 @@ import ( "github.com/gofiber/fiber/v3" ) -func (handler *Handler) authenticatedUserID(ctx fiber.Ctx) (string, *httputil.ErrorResponse) { - return handler.AuthenticatedUserID(ctx) -} - -func (handler *Handler) writeAppError(ctx fiber.Ctx, err error) error { - return handler.WriteAppError(ctx, err) -} - func writeAppError(ctx fiber.Ctx, logger any, err error) error { _ = logger if appErr, ok := err.(*git.Error); ok { diff --git a/internal/server/http/git/installation.go b/internal/server/http/git/installation.go index 2f25cc4..8d90e7d 100644 --- a/internal/server/http/git/installation.go +++ b/internal/server/http/git/installation.go @@ -46,7 +46,7 @@ func (handler *Handler) handleGitOrganizationInitConnectPOST(ctx fiber.Ctx) erro if redirectPath == "" { redirectPath = "/git" } - connectCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + connectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() redirectURL, err := handler.gitService.RequestInstallationURL( connectCtx, @@ -76,7 +76,7 @@ func (handler *Handler) handleGitOrganizationInitConnectPOST(ctx fiber.Ctx) erro targetID, repoID, resolveErr := handler.gitService.ResolveTargetAndRepositoryIDs(connectCtx, identity) if resolveErr != nil { handler.logger.Warning(fmt.Sprintf("skipping GitHub install redirect optimization for %s/%s: %v", identity.Owner, identity.Repo, resolveErr)) - if settingsURL, ok := handler.organizationInstallationSettingsURL(connectCtx, authorizationHeader, organization, identity.Owner); ok { + if settingsURL, ok := handler.organizationInstallationSettingsURL(connectCtx, authorizationHeader, organization, identity.Owner, redirectURL); ok { redirectURL = settingsURL } } else { @@ -90,7 +90,7 @@ func (handler *Handler) handleGitOrganizationInitConnectPOST(ctx fiber.Ctx) erro }, http.StatusOK).Write(ctx) } -func (handler *Handler) organizationInstallationSettingsURL(ctx context.Context, authorizationHeader string, organization string, owner string) (string, bool) { +func (handler *Handler) organizationInstallationSettingsURL(ctx context.Context, authorizationHeader string, organization string, owner string, redirectURL string) (string, bool) { installation, err := handler.gitService.RequestOrganizationInstallationStatus(ctx, authorizationHeader, organization, owner) if err != nil { handler.logger.Warning(fmt.Sprintf("failed to load GitHub organization installation for %s: %v", owner, err)) @@ -102,6 +102,11 @@ func (handler *Handler) organizationInstallationSettingsURL(ctx context.Context, if installation.InstallationID == nil || *installation.InstallationID <= 0 { return "", false } + + if appInstallationURL := appInstallationRedirectURL(strings.TrimSpace(redirectURL), *installation.InstallationID); appInstallationURL != "" { + return appInstallationURL, true + } + owner = strings.TrimSpace(owner) if owner == "" { return "", false @@ -124,6 +129,7 @@ func (handler *Handler) handleGitOrganizationConnectPOST(ctx fiber.Ctx) error { } type connectRequest struct { InstallationID *int64 `json:"installation_id"` + GitHubOwner string `json:"github_owner"` } requestBody := connectRequest{} if len(ctx.Body()) > 0 { @@ -137,11 +143,45 @@ func (handler *Handler) handleGitOrganizationConnectPOST(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - connectCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + githubOwner := strings.TrimSpace(requestBody.GitHubOwner) + if githubOwner == "" { + response := httputil.NewError(apierror.Type("invalid_request"), "github_owner is required", http.StatusBadRequest, map[string]any{"organization": organization}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + connectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() + installation, err := handler.gitService.RequestOrganizationInstallationStatus( + connectCtx, + authorizationHeader, + organization, + githubOwner, + ) + if err != nil { + if statusErr, ok := err.(*git.HTTPStatusError); ok { + response := httputil.NewError(apierror.Type(statusErr.Code), statusErr.Message, statusErr.StatusCode, map[string]any{"organization": organization, "github_owner": githubOwner}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + response := httputil.NewError(apierror.Type("integration_error"), fmt.Sprintf("failed to load GitHub organization installation status: %s", err), http.StatusBadGateway, map[string]any{"organization": organization, "github_owner": githubOwner}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + if !installation.Installed || installation.InstallationID == nil || *installation.InstallationID <= 0 { + response := httputil.NewError(apierror.Type("conflict"), "organization is not connected to the GitHub App", http.StatusConflict, map[string]any{"organization": organization, "github_owner": githubOwner}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + if err := handler.persistConnectedOrganizationState(organization, githubOwner, installation); err != nil { + response := httputil.NewError(apierror.TypeDatabaseError, fmt.Sprintf("failed to persist organization git state: %s", err), http.StatusInternalServerError, map[string]any{"organization": organization, "github_owner": githubOwner}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } repositories, err := handler.gitService.ListInstallationRepositories( connectCtx, authorizationHeader, + organization, + githubOwner, *requestBody.InstallationID, ) if err != nil { @@ -161,6 +201,42 @@ func (handler *Handler) handleGitOrganizationConnectPOST(ctx fiber.Ctx) error { }, http.StatusOK).Write(ctx) } +func (handler *Handler) persistConnectedOrganizationState(organization string, githubOwner string, installation git.GitRepositoryInstallationStatus) error { + now := time.Now().UTC() + state := geckodb.GitOrganizationState{ + Organization: organization, + Installed: installation.Installed, + UpdatedAt: now, + LastSeenAt: sql.NullTime{Time: now, Valid: true}, + ConfiguredAt: sql.NullTime{Time: now, Valid: installation.Installed}, + LastError: sql.NullString{}, + } + if installation.InstallationID != nil && *installation.InstallationID > 0 { + state.InstallationID = sql.NullInt64{Int64: *installation.InstallationID, Valid: true} + } + target := strings.TrimSpace(installation.Target) + if target == "" { + target = strings.TrimSpace(githubOwner) + } + if target != "" { + state.InstallationTarget = sql.NullString{String: target, Valid: true} + } + targetType := strings.TrimSpace(installation.TargetType) + if targetType == "" && target != "" { + targetType = "Organization" + } + if targetType != "" { + state.InstallationTargetType = sql.NullString{String: targetType, Valid: true} + } + if strings.TrimSpace(installation.HTMLURL) != "" { + state.HTMLURL = sql.NullString{String: strings.TrimSpace(installation.HTMLURL), Valid: true} + } + if strings.TrimSpace(installation.RepositorySelection) != "" { + state.RepositorySelection = sql.NullString{String: strings.TrimSpace(installation.RepositorySelection), Valid: true} + } + return geckodb.UpsertGitOrganizationState(handler.db, state) +} + func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { organization := strings.TrimSpace(ctx.Params("orgTitle")) project := strings.TrimSpace(ctx.Params("projectTitle")) @@ -179,11 +255,6 @@ func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { return errResponse.Write(ctx) } } - if strings.TrimSpace(requestBody.RepositoryFullName) == "" { - response := httputil.NewError(apierror.Type("invalid_request"), "repository_full_name is required", http.StatusBadRequest, map[string]any{"organization": organization, "project": project}, nil) - response.WriteLog(handler.logger) - return response.Write(ctx) - } projectID := organization + "/" + project authorizationHeader, tokenErr := servermw.ValidateAuthorizationHeader(ctx.Get("Authorization")) if tokenErr != nil { @@ -191,19 +262,31 @@ func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - connectCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + connectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() projectCfg, errResponse := handler.loadProjectConfig(connectCtx, projectID) if errResponse != nil { errResponse.WriteLog(handler.logger) return errResponse.Write(ctx) } - orgState, errResponse := handler.loadConnectedOrganizationState(connectCtx, organization, project) + if strings.TrimSpace(requestBody.RepositoryFullName) == "" { + if err := handler.unbindProjectRepository(connectCtx, projectID, projectCfg); err != nil { + response := httputil.NewError(apierror.TypeDatabaseError, fmt.Sprintf("failed to clear project repository binding: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + return httputil.JSON(git.GitOrganizationConnectResponse{Mode: "disconnected"}, http.StatusOK).Write(ctx) + } + orgState, errResponse := handler.loadOrBootstrapConnectedOrganizationState(connectCtx, authorizationHeader, organization, project, requestBody.RepositoryFullName) if errResponse != nil { errResponse.WriteLog(handler.logger) return errResponse.Write(ctx) } - repositories, errResponse := handler.listConnectedInstallationRepositories(connectCtx, authorizationHeader, organization, project, orgState.InstallationID.Int64) + owner := organization + if orgState.InstallationTarget.Valid { + owner = strings.TrimSpace(orgState.InstallationTarget.String) + } + repositories, errResponse := handler.listConnectedInstallationRepositories(connectCtx, authorizationHeader, organization, project, owner, orgState.InstallationID.Int64) if errResponse != nil { errResponse.WriteLog(handler.logger) return errResponse.Write(ctx) @@ -228,6 +311,40 @@ func (handler *Handler) handleGitProjectEditConnectPOST(ctx fiber.Ctx) error { return httputil.JSON(git.GitOrganizationConnectResponse{Mode: "connected"}, http.StatusOK).Write(ctx) } +func (handler *Handler) loadOrBootstrapConnectedOrganizationState(ctx context.Context, authorizationHeader string, organization string, project string, repositoryFullName string) (*geckodb.GitOrganizationState, *httputil.ErrorResponse) { + orgState, errResponse := handler.loadConnectedOrganizationState(ctx, organization, project) + if errResponse == nil { + return orgState, nil + } + if errResponse.Error.Code != http.StatusConflict { + return nil, errResponse + } + + identity, err := parseRequestedRepositoryIdentity(repositoryFullName) + if err != nil { + response := httputil.NewError(apierror.Type("invalid_request"), fmt.Sprintf("invalid repository_full_name %q: %s", repositoryFullName, err), http.StatusBadRequest, map[string]any{"organization": organization, "project": project}, nil) + return nil, response + } + installation, installErr := handler.gitService.RequestOrganizationInstallationStatus(ctx, authorizationHeader, organization, identity.Owner) + if installErr != nil { + if statusErr, ok := installErr.(*git.HTTPStatusError); ok { + response := httputil.NewError(apierror.Type(statusErr.Code), statusErr.Message, statusErr.StatusCode, map[string]any{"organization": organization, "project": project, "github_owner": identity.Owner}, nil) + return nil, response + } + response := httputil.NewError(apierror.Type("integration_error"), fmt.Sprintf("failed to load GitHub organization installation status: %s", installErr), http.StatusBadGateway, map[string]any{"organization": organization, "project": project, "github_owner": identity.Owner}, nil) + return nil, response + } + if !installation.Installed || installation.InstallationID == nil || *installation.InstallationID <= 0 { + response := httputil.NewError(apierror.Type("conflict"), "organization is not connected to the GitHub App", http.StatusConflict, map[string]any{"organization": organization, "project": project, "github_owner": identity.Owner}, nil) + return nil, response + } + if err := handler.persistConnectedOrganizationState(organization, identity.Owner, installation); err != nil { + response := httputil.NewError(apierror.TypeDatabaseError, fmt.Sprintf("failed to persist organization git state: %s", err), http.StatusInternalServerError, map[string]any{"organization": organization, "project": project, "github_owner": identity.Owner}, nil) + return nil, response + } + return handler.loadConnectedOrganizationState(ctx, organization, project) +} + func normalizeInstallationRepository(repositoryFullName string, repositories []git.GitHubInstallationRepository, organization string, project string) (git.GitRepositoryIdentity, bool, *httputil.ErrorResponse) { requested := strings.TrimSpace(repositoryFullName) for _, repository := range repositories { @@ -278,6 +395,21 @@ func decorateInstallationRedirectURL(redirectURL string, targetID int64, repoID return fmt.Sprintf("%s&repository_ids[]=%d", redirectURL, repoID) } +func appInstallationRedirectURL(redirectURL string, installationID int64) string { + if installationID <= 0 || strings.TrimSpace(redirectURL) == "" { + return "" + } + request, err := http.NewRequest(http.MethodGet, redirectURL, nil) + if err != nil || request.URL == nil { + return "" + } + pathParts := strings.Split(strings.Trim(request.URL.Path, "/"), "/") + if len(pathParts) < 2 || pathParts[0] != "apps" || strings.TrimSpace(pathParts[1]) == "" { + return "" + } + return fmt.Sprintf("%s://%s/apps/%s/installations/%d", request.URL.Scheme, request.URL.Host, pathParts[1], installationID) +} + func (handler *Handler) loadProjectConfig(ctx context.Context, projectID string) (appconfig.ProjectConfig, *httputil.ErrorResponse) { var projectCfg appconfig.ProjectConfig if err := geckodb.ConfigGETGenericContext(ctx, handler.db, projectID, string(appconfig.TypeProjects), &projectCfg); err != nil { @@ -304,8 +436,8 @@ func (handler *Handler) loadConnectedOrganizationState(ctx context.Context, orga return orgState, nil } -func (handler *Handler) listConnectedInstallationRepositories(ctx context.Context, authorizationHeader string, organization string, project string, installationID int64) ([]git.GitHubInstallationRepository, *httputil.ErrorResponse) { - repositories, err := handler.gitService.ListInstallationRepositories(ctx, authorizationHeader, installationID) +func (handler *Handler) listConnectedInstallationRepositories(ctx context.Context, authorizationHeader string, organization string, project string, owner string, installationID int64) ([]git.GitHubInstallationRepository, *httputil.ErrorResponse) { + repositories, err := handler.gitService.ListInstallationRepositories(ctx, authorizationHeader, organization, owner, installationID) if err != nil { if statusErr, ok := err.(*git.HTTPStatusError); ok { response := httputil.NewError(apierror.Type(statusErr.Code), statusErr.Message, statusErr.StatusCode, map[string]any{"organization": organization, "project": project, "installation_id": installationID}, nil) @@ -358,6 +490,29 @@ func (handler *Handler) bindProjectRepository(ctx context.Context, projectID str return nil } +func (handler *Handler) unbindProjectRepository(ctx context.Context, projectID string, projectCfg appconfig.ProjectConfig) error { + projectCfg.SrcRepo = "" + + tx, err := handler.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("begin project repository unbind transaction: %w", err) + } + defer func() { + _ = tx.Rollback() + }() + + if err := geckodb.ConfigPUTGenericTxContext(ctx, tx, projectID, string(appconfig.TypeProjects), &projectCfg); err != nil { + return fmt.Errorf("update project config: %w", err) + } + if _, err := tx.ExecContext(ctx, `DELETE FROM config_schema.git_project_state WHERE project_id = $1`, projectID); err != nil { + return fmt.Errorf("delete project git state: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit project repository unbind transaction: %w", err) + } + return nil +} + func (handler *Handler) handleGitProjectUpdatePOST(ctx fiber.Ctx) error { organization, project, projectID, cfg, identity, errResponse := handler.resolveGitProject(ctx) if errResponse != nil { diff --git a/internal/server/http/git/installation_test.go b/internal/server/http/git/installation_test.go index fef4da9..0c57e39 100644 --- a/internal/server/http/git/installation_test.go +++ b/internal/server/http/git/installation_test.go @@ -40,11 +40,17 @@ func newGitHandlerTestServer(t *testing.T, fenceServer *httptest.Server, githubT if githubTransport != nil { githubClient.Transport = githubTransport } + fenceClient := http.DefaultClient + fenceBaseURL := "" + if fenceServer != nil { + fenceClient = fenceServer.Client() + fenceBaseURL = fenceServer.URL + } gitSvc := gitservice.NewGitService(gitservice.GitServiceConfig{ DataDir: t.TempDir(), GitHubAPIBase: "https://api.github.com", HTTPClient: githubClient, - FenceClient: intfence.NewClient(fenceServer.Client(), intfence.Config{BaseURL: fenceServer.URL}), + FenceClient: intfence.NewClient(fenceClient, intfence.Config{BaseURL: fenceBaseURL}), }) handler := &Handler{ Handler: &shared.Handler{}, @@ -199,7 +205,7 @@ func TestGitOrganizationInitConnectFallsBackToPlainRedirectWhenRepositoryLookupF } } -func TestGitOrganizationInitConnectFallsBackToCleanOrgSettingsURLWhenRepositoryLookupFails(t *testing.T) { +func TestGitOrganizationInitConnectFallsBackToAppInstallationURLWhenRepositoryLookupFails(t *testing.T) { fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { var receivedBody map[string]any if err := json.NewDecoder(request.Body).Decode(&receivedBody); err != nil { @@ -262,8 +268,8 @@ func TestGitOrganizationInitConnectFallsBackToCleanOrgSettingsURLWhenRepositoryL if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { t.Fatalf("decode response: %v", err) } - if payload.RedirectURL != "https://github.com/organizations/EllrottLab/settings/installations/134470697" { - t.Fatalf("expected clean organization settings redirect when repo lookup fails, got %q", payload.RedirectURL) + if payload.RedirectURL != "https://github.com/apps/calypr-github/installations/134470697" { + t.Fatalf("expected app installation redirect when repo lookup fails, got %q", payload.RedirectURL) } if strings.Contains(payload.RedirectURL, "suggested_target_id=") || strings.Contains(payload.RedirectURL, "repository_ids") { t.Fatalf("did not expect partial redirect optimization or empty repository_ids, got %q", payload.RedirectURL) @@ -337,10 +343,16 @@ func TestGitProjectEditConnectUpdatesProjectConfigAndState(t *testing.T) { payload, _ := io.ReadAll(resp.Body) t.Fatalf("expected 200, got %d: %s", resp.StatusCode, payload) } - if receivedBody["action"] != "installation_repositories" { - t.Fatalf("expected installation_repositories action, got %#v", receivedBody) - } - var payload gitservice.GitOrganizationConnectResponse + if receivedBody["action"] != "installation_repositories" { + t.Fatalf("expected installation_repositories action, got %#v", receivedBody) + } + if receivedBody["organization"] != "TEST" { + t.Fatalf("expected organization in fence request body, got %#v", receivedBody) + } + if receivedBody["owner"] != "EllrottLab" { + t.Fatalf("expected owner in fence request body, got %#v", receivedBody) + } + var payload gitservice.GitOrganizationConnectResponse if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { t.Fatalf("decode response: %v", err) } @@ -352,6 +364,100 @@ func TestGitProjectEditConnectUpdatesProjectConfigAndState(t *testing.T) { } } +func TestGitOrganizationConnectRequiresGitHubOwner(t *testing.T) { + handler, _, cleanup := newGitHandlerTestServer(t, nil, nil) + defer cleanup() + + app := fiber.New() + app.Post("/git/organizations/:orgTitle/connect", handler.handleGitOrganizationConnectPOST) + req := httptest.NewRequest(http.MethodPost, "/git/organizations/TEST/connect", bytes.NewBufferString(`{"installation_id":42}`)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + + resp := runGitRequest(t, app, req) + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 400, got %d: %s", resp.StatusCode, payload) + } +} + +func TestGitOrganizationConnectForwardsGitHubOwner(t *testing.T) { + var receivedBody map[string]any + fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if err := json.NewDecoder(request.Body).Decode(&receivedBody); err != nil { + t.Fatalf("decode fence request body: %v", err) + } + writer.Header().Set("Content-Type", "application/json") + switch receivedBody["action"] { + case "organization_installation": + _ = json.NewEncoder(writer).Encode(map[string]any{ + "installed": true, + "installation_id": 42, + "target": "EllrottLab", + "target_type": "Organization", + "html_url": "https://github.com/organizations/EllrottLab/settings/installations/42", + "repository_selection": "selected", + }) + case "installation_repositories": + _ = json.NewEncoder(writer).Encode(map[string]any{ + "installation_id": 42, + "repositories": []map[string]any{{ + "id": 101, + "name": "git_drs_test", + "full_name": "EllrottLab/git_drs_test", + "html_url": "https://github.com/EllrottLab/git_drs_test", + "clone_url": "https://github.com/EllrottLab/git_drs_test.git", + }}, + }) + default: + t.Fatalf("unexpected action: %#v", receivedBody["action"]) + } + })) + defer fenceServer.Close() + + handler, mock, cleanup := newGitHandlerTestServer(t, fenceServer, nil) + defer cleanup() + + mock.ExpectExec(`INSERT INTO config_schema\.git_organization_state`). + WithArgs( + "TEST", + true, + int64(42), + "Organization", + "EllrottLab", + "https://github.com/organizations/EllrottLab/settings/installations/42", + "selected", + sqlmock.AnyArg(), + sqlmock.AnyArg(), + sqlmock.AnyArg(), + sql.NullString{}, + ). + WillReturnResult(sqlmock.NewResult(1, 1)) + + app := fiber.New() + app.Post("/git/organizations/:orgTitle/connect", handler.handleGitOrganizationConnectPOST) + req := httptest.NewRequest(http.MethodPost, "/git/organizations/TEST/connect", bytes.NewBufferString(`{"installation_id":42,"github_owner":"EllrottLab"}`)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + + resp := runGitRequest(t, app, req) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 200, got %d: %s", resp.StatusCode, payload) + } + if receivedBody["organization"] != "TEST" { + t.Fatalf("expected organization in fence request body, got %#v", receivedBody) + } + if receivedBody["owner"] != "EllrottLab" { + t.Fatalf("expected github owner in fence request body, got %#v", receivedBody) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + func TestGitProjectEditConnectRejectsRepositoryOutsideInstallation(t *testing.T) { fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { writer.Header().Set("Content-Type", "application/json") @@ -404,7 +510,17 @@ func TestGitProjectEditConnectRejectsRepositoryOutsideInstallation(t *testing.T) func TestGitProjectEditConnectRequiresConnectedOrganization(t *testing.T) { fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - writer.WriteHeader(http.StatusInternalServerError) + var receivedBody map[string]any + if err := json.NewDecoder(request.Body).Decode(&receivedBody); err != nil { + t.Fatalf("decode fence request body: %v", err) + } + if receivedBody["action"] != "organization_installation" { + t.Fatalf("expected organization_installation action, got %#v", receivedBody["action"]) + } + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]any{ + "installed": false, + }) })) defer fenceServer.Close() @@ -441,3 +557,121 @@ func TestGitProjectEditConnectRequiresConnectedOrganization(t *testing.T) { t.Fatalf("unmet sql expectations: %v", err) } } + +func TestGitProjectEditConnectClearsRepositoryBinding(t *testing.T) { + fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusInternalServerError) + })) + defer fenceServer.Close() + + handler, mock, cleanup := newGitHandlerTestServer(t, fenceServer, nil) + defer cleanup() + + projectCfg := appconfig.ProjectConfig{ + Title: "proj-a", + OrgTitle: "TEST", + ProjectTitle: "proj-a", + SrcRepo: "https://github.com/EllrottLab/git_drs_test", + } + projectContent, err := json.Marshal(projectCfg) + if err != nil { + t.Fatalf("marshal project config: %v", err) + } + mock.ExpectQuery(`SELECT name, content FROM config_schema\.projects WHERE name=\$1`). + WithArgs("TEST/proj-a"). + WillReturnRows(sqlmock.NewRows([]string{"name", "content"}).AddRow("TEST/proj-a", projectContent)) + updatedCfg := projectCfg + updatedCfg.SrcRepo = "" + updatedContent, err := json.Marshal(&updatedCfg) + if err != nil { + t.Fatalf("marshal updated project config: %v", err) + } + mock.ExpectBegin() + mock.ExpectExec(`INSERT INTO config_schema\.projects`). + WithArgs("TEST/proj-a", updatedContent). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`DELETE FROM config_schema\.git_project_state WHERE project_id = \$1`). + WithArgs("TEST/proj-a"). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/edit-connect", handler.handleGitProjectEditConnectPOST) + body := bytes.NewBufferString(`{"repository_full_name":""}`) + req := httptest.NewRequest(http.MethodPost, "/git/projects/TEST/proj-a/edit-connect", body) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + + resp := runGitRequest(t, app, req) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 200, got %d: %s", resp.StatusCode, payload) + } + var payload gitservice.GitOrganizationConnectResponse + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if payload.Mode != "disconnected" { + t.Fatalf("expected disconnected mode, got %+v", payload) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectUpdateRejectsSetupOnlyProjectUntilGitHubConnectCompletes(t *testing.T) { + fenceServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusInternalServerError) + })) + defer fenceServer.Close() + + handler, mock, cleanup := newGitHandlerTestServer(t, fenceServer, nil) + defer cleanup() + + projectCfg := appconfig.ProjectConfig{ + Title: "proj-a", + OrgTitle: "TEST", + ProjectTitle: "proj-a", + Description: "setup only", + ContactEmail: "test@example.org", + SrcRepo: "", + } + projectContent, err := json.Marshal(projectCfg) + if err != nil { + t.Fatalf("marshal project config: %v", err) + } + mock.ExpectQuery(`SELECT name, content FROM config_schema\.projects WHERE name=\$1`). + WithArgs("TEST/proj-a"). + WillReturnRows(sqlmock.NewRows([]string{"name", "content"}).AddRow("TEST/proj-a", projectContent)) + + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/update", handler.handleGitProjectUpdatePOST) + req := httptest.NewRequest(http.MethodPost, "/git/projects/TEST/proj-a/update", nil) + req.Header.Set("Authorization", "Bearer test") + + resp := runGitRequest(t, app, req) + defer resp.Body.Close() + if resp.StatusCode != http.StatusConflict { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 409, got %d: %s", resp.StatusCode, payload) + } + var payload struct { + Error struct { + Message string `json:"message"` + Details map[string]any `json:"details"` + } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if !strings.Contains(payload.Error.Message, "GitHub connection has not been completed") { + t.Fatalf("unexpected error message: %q", payload.Error.Message) + } + if got := payload.Error.Details["workflow_stage"]; got != gitservice.GitWorkflowStageAwaitingGitHubConnect { + t.Fatalf("expected workflow stage %q, got %#v", gitservice.GitWorkflowStageAwaitingGitHubConnect, got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} diff --git a/internal/server/http/git/presentation_config.go b/internal/server/http/git/presentation_config.go new file mode 100644 index 0000000..1317948 --- /dev/null +++ b/internal/server/http/git/presentation_config.go @@ -0,0 +1,64 @@ +package git + +import ( + "errors" + "fmt" + "net/http" + + appconfig "github.com/calypr/gecko/config" + "github.com/calypr/gecko/internal/httputil" + "github.com/calypr/gecko/internal/presentation" + "github.com/gofiber/fiber/v3" +) + +func (handler *Handler) handleGitProjectPresentationConfigGET(ctx fiber.Ctx) error { + organization, project, projectID, errResponse := handler.resolveExistingProject(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + if handler.presentationStore == nil { + errResponse = httputil.NewError("internal_error", "presentation storage is not configured", http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + + content, err := handler.presentationStore.Get(organization, project) + if err != nil { + if errors.Is(err, presentation.ErrNoPresentation) { + return httputil.JSON(appconfig.PresentationConfig{}, http.StatusOK).Write(ctx) + } + errResponse = httputil.NewError("internal_error", fmt.Sprintf("failed to load project presentation config: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + return httputil.JSON(appconfig.PresentationConfig{PresentationConfig: content}, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectPresentationConfigPUT(ctx fiber.Ctx) error { + organization, project, projectID, errResponse := handler.resolveExistingProject(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + if handler.presentationStore == nil { + errResponse = httputil.NewError("internal_error", "presentation storage is not configured", http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + + cfg := &appconfig.PresentationConfig{} + if errResponse = httputil.ParseJSONBody(ctx.Body(), cfg, map[string]any{"project_id": projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + if err := cfg.Validate(); err != nil { + errResponse = httputil.NewError("validation_failed", fmt.Sprintf("body data validation failed: %s", err), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + if err := handler.presentationStore.Save(organization, project, cfg.PresentationConfig); err != nil { + errResponse = httputil.NewError("internal_error", fmt.Sprintf("failed to persist project presentation config: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + return httputil.JSON(cfg, http.StatusOK).Write(ctx) +} diff --git a/internal/server/http/git/presentation_config_test.go b/internal/server/http/git/presentation_config_test.go new file mode 100644 index 0000000..abbf3c6 --- /dev/null +++ b/internal/server/http/git/presentation_config_test.go @@ -0,0 +1,218 @@ +package git + +import ( + "bytes" + "encoding/json" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + appconfig "github.com/calypr/gecko/config" + geckologging "github.com/calypr/gecko/internal/logging" + "github.com/calypr/gecko/internal/presentation" + servermw "github.com/calypr/gecko/internal/server/middleware" + "github.com/gofiber/fiber/v3" + "github.com/jmoiron/sqlx" +) + +type fakePresentationAccessHandler struct { + resources []any +} + +func (handler fakePresentationAccessHandler) GetAllowedResources(_ string, _ string, _ string) ([]any, error) { + return handler.resources, nil +} + +func (handler fakePresentationAccessHandler) CheckResourceServiceAccess(_ string, _ string, _ string, resourcePath string) (bool, error) { + for _, resource := range handler.resources { + if value, ok := resource.(string); ok && value == resourcePath { + return true, nil + } + } + return false, nil +} + +func newPresentationConfigTestServer(t *testing.T) (*Handler, sqlmock.Sqlmock, func()) { + t.Helper() + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + handler := &Handler{ + db: sqlx.NewDb(db, "sqlmock"), + logger: &geckologging.Handler{Logger: log.New(os.Stdout, "", 0)}, + presentationStore: presentation.NewFilesystemStore(t.TempDir()), + } + return handler, mock, func() { _ = db.Close() } +} + +func newPresentationConfigApp(handler *Handler, authz servermw.ResourceAccessHandler) *fiber.App { + app := fiber.New() + app.Get("/git/projects/:orgTitle/:projectTitle/presentationConfig", servermw.ProjectConfigAuth(handler.logger, authz, "read"), handler.handleGitProjectPresentationConfigGET) + app.Put("/git/projects/:orgTitle/:projectTitle/presentationConfig", servermw.ProjectConfigAuth(handler.logger, authz, "update"), handler.handleGitProjectPresentationConfigPUT) + app.Post("/git/projects/:orgTitle/:projectTitle/presentationConfig", servermw.ProjectConfigAuth(handler.logger, authz, "update"), handler.handleGitProjectPresentationConfigPUT) + return app +} + +func runPresentationConfigRequest(t *testing.T, app *fiber.App, req *http.Request) *http.Response { + t.Helper() + resp, err := app.Test(req, fiber.TestConfig{Timeout: 0, FailOnTimeout: false}) + if err != nil { + t.Fatalf("fiber test request failed: %v", err) + } + return resp +} + +func expectProjectLookup(mock sqlmock.Sqlmock, organization string, project string) { + projectCfg := appconfig.ProjectConfig{ + Title: project, + ContactEmail: "owner@example.org", + SrcRepo: "github.com/example/" + project, + OrgTitle: organization, + Description: "project", + ProjectTitle: project, + } + content, _ := json.Marshal(projectCfg) + mock.ExpectQuery(`SELECT name, content FROM config_schema\.projects WHERE name=\$1`). + WithArgs(organization + "/" + project). + WillReturnRows(sqlmock.NewRows([]string{"name", "content"}).AddRow(organization+"/"+project, content)) +} + +func decodePresentationResponse(t *testing.T, body io.Reader) appconfig.PresentationConfig { + t.Helper() + var payload appconfig.PresentationConfig + if err := json.NewDecoder(body).Decode(&payload); err != nil { + t.Fatalf("decode response: %v", err) + } + return payload +} + +func TestGitProjectPresentationConfigGETNoConfigFallsBackToEmpty(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + expectProjectLookup(mock, "org-a", "proj-a") + + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) + req := httptest.NewRequest(http.MethodGet, "/git/projects/org-a/proj-a/presentationConfig", nil) + req.Header.Set("Authorization", "Bearer test") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + payload := decodePresentationResponse(t, resp.Body) + if payload.PresentationConfig != "" { + t.Fatalf("expected empty presentationConfig, got %q", payload.PresentationConfig) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectPresentationConfigPUTPersistsRawHTML(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + expectProjectLookup(mock, "org-a", "proj-a") + + requestPayload := appconfig.PresentationConfig{ + PresentationConfig: `

Hello

`, + } + requestBody, _ := json.Marshal(requestPayload) + expectedStored := `

Hello

` + + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) + req := httptest.NewRequest(http.MethodPut, "/git/projects/org-a/proj-a/presentationConfig", bytes.NewReader(requestBody)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + payload := decodePresentationResponse(t, resp.Body) + if payload.PresentationConfig != expectedStored { + t.Fatalf("unexpected sanitized response: %q", payload.PresentationConfig) + } + data, err := os.ReadFile(handler.presentationStore.ProjectPresentationPath("org-a", "proj-a")) + if err != nil { + t.Fatalf("read stored presentation: %v", err) + } + if string(data) != expectedStored { + t.Fatalf("unexpected stored HTML: %q", string(data)) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectPresentationConfigPOSTUsesSameWritePath(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + expectProjectLookup(mock, "org-a", "proj-a") + + requestPayload := appconfig.PresentationConfig{PresentationConfig: `

Hello

`} + requestBody, _ := json.Marshal(requestPayload) + + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) + req := httptest.NewRequest(http.MethodPost, "/git/projects/org-a/proj-a/presentationConfig", bytes.NewReader(requestBody)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + if filepath.Base(handler.presentationStore.ProjectPresentationPath("org-a", "proj-a")) != "proj-a_presentation.html" { + t.Fatalf("unexpected presentation filename: %q", filepath.Base(handler.presentationStore.ProjectPresentationPath("org-a", "proj-a"))) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectPresentationConfigPUTRejectsUnknownFields(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + expectProjectLookup(mock, "org-a", "proj-a") + + requestBody := []byte(`{"presentationConfig":"

Hello

","extra":"nope"}`) + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/proj-a"}}) + req := httptest.NewRequest(http.MethodPut, "/git/projects/org-a/proj-a/presentationConfig", bytes.NewReader(requestBody)) + req.Header.Set("Authorization", "Bearer test") + req.Header.Set("Content-Type", "application/json") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected status 400, got %d", resp.StatusCode) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestGitProjectPresentationConfigRouteRequiresProjectAccess(t *testing.T) { + handler, mock, cleanup := newPresentationConfigTestServer(t) + defer cleanup() + + app := newPresentationConfigApp(handler, fakePresentationAccessHandler{resources: []any{"/programs/org-a/projects/other"}}) + req := httptest.NewRequest(http.MethodGet, "/git/projects/org-a/proj-a/presentationConfig", nil) + req.Header.Set("Authorization", "Bearer test") + resp := runPresentationConfigRequest(t, app, req) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("expected status 403, got %d", resp.StatusCode) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} diff --git a/internal/server/http/git/project.go b/internal/server/http/git/project.go index 2ce4be7..1511e08 100644 --- a/internal/server/http/git/project.go +++ b/internal/server/http/git/project.go @@ -66,6 +66,14 @@ func (handler *Handler) resolveGitProject(ctx fiber.Ctx) (string, string, string response.WriteLog(handler.logger) return "", "", "", appconfig.ProjectConfig{}, git.GitRepositoryIdentity{}, response } + if strings.TrimSpace(cfg.SrcRepo) == "" { + response := httputil.NewError("conflict", fmt.Sprintf("GitHub connection has not been completed for %s", projectID), http.StatusConflict, map[string]any{ + "project_id": projectID, + "workflow_stage": git.GitWorkflowStageAwaitingGitHubConnect, + }, nil) + response.WriteLog(handler.logger) + return "", "", "", appconfig.ProjectConfig{}, git.GitRepositoryIdentity{}, response + } identity, err := git.ParseRepositoryIdentity(cfg.SrcRepo) if err != nil { response := httputil.NewError("validation_failed", fmt.Sprintf("invalid src_repo for %s: %s", projectID, err), http.StatusBadRequest, map[string]any{"project_id": projectID, "src_repo": cfg.SrcRepo}, nil) @@ -97,14 +105,14 @@ func (handler *Handler) loadGitProjectState(projectID string, identity git.GitRe return state, nil } -func (handler *Handler) ensureMirrorReadyForRead(ctx context.Context, authorizationHeader string, projectID string, identity git.GitRepositoryIdentity, state *geckodb.GitProjectState) (*geckodb.GitProjectState, error) { +func (handler *Handler) ensureMirrorReadyForRead(ctx context.Context, authorizationHeader string, projectID string, identity git.GitRepositoryIdentity, state *geckodb.GitProjectState, requireCurrentMirror bool) (*geckodb.GitProjectState, error) { if state == nil || !state.InstallationID.Valid { return state, nil } if strings.TrimSpace(state.MirrorPath) == "" { return state, nil } - if _, err := os.Stat(state.MirrorPath); err == nil { + if !shouldRefreshMirrorForRead(state.MirrorPath, requireCurrentMirror) { return state, nil } org, project, _ := strings.Cut(projectID, "/") @@ -128,6 +136,15 @@ func (handler *Handler) ensureMirrorReadyForRead(ctx context.Context, authorizat return updatedState, nil } +func mirrorExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func shouldRefreshMirrorForRead(mirrorPath string, requireCurrentMirror bool) bool { + return requireCurrentMirror || !mirrorExists(mirrorPath) +} + func (handler *Handler) handleGitProjectsGET(ctx fiber.Ctx) error { states, err := geckodb.ListGitProjectStates(handler.db) if err != nil { @@ -211,7 +228,7 @@ func (handler *Handler) handleGitProjectGET(ctx fiber.Ctx) error { if authorizationHeader != "" { refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, false) if err != nil { handler.logger.Warning("failed to warm git mirror for %s: %v", projectID, err) } diff --git a/internal/server/http/git/project_test.go b/internal/server/http/git/project_test.go new file mode 100644 index 0000000..9f9488a --- /dev/null +++ b/internal/server/http/git/project_test.go @@ -0,0 +1,43 @@ +package git + +import ( + "path/filepath" + "testing" +) + +func TestShouldRefreshMirrorForRead(t *testing.T) { + existingMirrorPath := t.TempDir() + missingMirrorPath := filepath.Join(t.TempDir(), "missing.git") + + tests := []struct { + name string + mirrorPath string + requireCurrentMirror bool + want bool + }{ + { + name: "existing mirror can serve regular reads", + mirrorPath: existingMirrorPath, + want: false, + }, + { + name: "storage audit refreshes existing mirror", + mirrorPath: existingMirrorPath, + requireCurrentMirror: true, + want: true, + }, + { + name: "missing mirror refreshes regular reads", + mirrorPath: missingMirrorPath, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := shouldRefreshMirrorForRead(test.mirrorPath, test.requireCurrentMirror); got != test.want { + t.Fatalf("expected refresh=%t, got %t", test.want, got) + } + }) + } +} diff --git a/internal/server/http/git/register.go b/internal/server/http/git/register.go index 4571184..d9a923c 100644 --- a/internal/server/http/git/register.go +++ b/internal/server/http/git/register.go @@ -21,23 +21,42 @@ func RegisterRoutes(app *fiber.App, sharedHandler *shared.Handler, authzHandler gitGroup.Post("/organizations/:orgTitle/reconcile", servermw.GitOrganizationAuth(handler.Logger, authzHandler), handler.handleGitOrganizationReconcilePOST) projectReadAuth := servermw.GitProjectAuth(handler.Logger, authzHandler) + projectSetupAuth := servermw.GitProjectSetupAuth(handler.Logger, authzHandler) + projectWriteAuth := servermw.GitProjectMutationAuth(handler.Logger, authzHandler, "update") + projectConfigReadAuth := servermw.ProjectConfigAuth(handler.Logger, authzHandler, "read") + projectConfigWriteAuth := servermw.ProjectConfigAuth(handler.Logger, authzHandler, "update") gitGroup.Get("/projects/:orgTitle/:projectTitle", projectReadAuth, handler.handleGitProjectGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/refs", projectReadAuth, handler.handleGitProjectRefsGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/tree", projectReadAuth, handler.handleGitProjectTreeGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/tree/*", projectReadAuth, handler.handleGitProjectTreeGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/manifest", projectReadAuth, handler.handleGitProjectManifestGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/manifest/*", projectReadAuth, handler.handleGitProjectManifestGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/file/*", projectReadAuth, handler.handleGitProjectFileGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/download/*", projectReadAuth, handler.handleGitProjectDownloadGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/storage/summary", projectReadAuth, handler.handleGitProjectStorageSummaryGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/storage/children", projectReadAuth, handler.handleGitProjectStorageChildrenGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/storage/folder", projectReadAuth, handler.handleGitProjectStorageFolderGET) gitGroup.Get("/projects/:orgTitle/:projectTitle/thumbnail", handler.handleGitProjectThumbnailGET) + gitGroup.Get("/projects/:orgTitle/:projectTitle/presentationConfig", projectConfigReadAuth, handler.handleGitProjectPresentationConfigGET) projectGitWrite := gitGroup.Group("/projects/:orgTitle/:projectTitle", servermw.RequireAuthorization(handler.Logger)) + // Setup must stay auth-only so a brand-new organization can be bootstrapped + // before any org/project Arborist resources exist for the caller. projectGitWrite.Put("/setup", handler.handleCalyprProjectSetupPUT) - projectGitWrite.Put("/storage", handler.handleCalyprProjectStoragePUT) - projectGitWrite.Put("/thumbnail", handler.handleGitProjectThumbnailPUT) - projectGitWrite.Delete("/thumbnail", handler.handleGitProjectThumbnailDELETE) - projectGitWrite.Post("/edit-connect", handler.handleGitProjectEditConnectPOST) - projectGitWrite.Post("/update", handler.handleGitProjectUpdatePOST) - projectGitWrite.Post("/uploads/session", handler.handleGitProjectUploadSessionPOST) - projectGitWrite.Get("/uploads/session/:sessionID", handler.handleGitProjectUploadSessionGET) - projectGitWrite.Post("/uploads/session/:sessionID/files", handler.handleGitProjectUploadSessionFilesPOST) - projectGitWrite.Post("/uploads/session/:sessionID/finalize", handler.handleGitProjectUploadSessionFinalizePOST) + projectGitWrite.Put("/storage", projectSetupAuth, handler.handleCalyprProjectStoragePUT) + projectGitWrite.Put("/thumbnail", projectWriteAuth, handler.handleGitProjectThumbnailPUT) + projectGitWrite.Delete("/thumbnail", projectWriteAuth, handler.handleGitProjectThumbnailDELETE) + projectGitWrite.Put("/presentationConfig", projectConfigWriteAuth, handler.handleGitProjectPresentationConfigPUT) + projectGitWrite.Post("/presentationConfig", projectConfigWriteAuth, handler.handleGitProjectPresentationConfigPUT) + projectGitWrite.Post("/edit-connect", projectWriteAuth, handler.handleGitProjectEditConnectPOST) + projectGitWrite.Post("/update", projectWriteAuth, handler.handleGitProjectUpdatePOST) + projectGitWrite.Post("/repair/project-diff/audit", projectReadAuth, handler.handleGitProjectDiffAuditPOST) + projectGitWrite.Post("/repair/storage-chain/audit", projectReadAuth, handler.handleGitProjectStorageChainAuditPOST) + projectGitWrite.Post("/repair/storage-chain/register-git-only", projectWriteAuth, handler.handleGitProjectStorageChainRegisterGitOnlyPOST) + projectGitWrite.Post("/repair/storage-cleanup/audit", projectReadAuth, handler.handleGitProjectStorageCleanupAuditPOST) + projectGitWrite.Post("/repair/storage-cleanup/apply", projectReadAuth, handler.handleGitProjectStorageCleanupApplyPOST) + projectGitWrite.Post("/uploads/session", projectWriteAuth, handler.handleGitProjectUploadSessionPOST) + projectGitWrite.Get("/uploads/session/:sessionID", projectWriteAuth, handler.handleGitProjectUploadSessionGET) + projectGitWrite.Post("/uploads/session/:sessionID/files", projectWriteAuth, handler.handleGitProjectUploadSessionFilesPOST) + projectGitWrite.Post("/uploads/session/:sessionID/finalize", projectWriteAuth, handler.handleGitProjectUploadSessionFinalizePOST) } diff --git a/internal/server/http/git/repository.go b/internal/server/http/git/repository.go index 7863777..d7d7b44 100644 --- a/internal/server/http/git/repository.go +++ b/internal/server/http/git/repository.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "net/http" + "net/url" + "strconv" "strings" "time" @@ -34,7 +36,7 @@ func (handler *Handler) handleGitProjectRefsGET(ctx fiber.Ctx) error { if authorizationHeader != "" { refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, false) if err != nil { handler.logger.Warning("failed to warm git mirror for %s refs: %v", projectID, err) } @@ -74,7 +76,7 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { if authorizationHeader != "" { refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state) + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, false) if err != nil { handler.logger.Warning("failed to warm git mirror for %s tree: %v", projectID, err) } @@ -90,11 +92,18 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { if refName == "" { refName = state.DefaultBranch.String } + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } return httputil.JSON(&git.GitProjectTreeResponse{ - ProjectID: projectID, - Ref: refName, - Path: strings.Trim(ctx.Params("*"), "/"), - Entries: []git.GitTreeEntry{}, + ProjectID: projectID, + Ref: refName, + Path: path, + EntryCount: 0, + Entries: []git.GitTreeEntry{}, }, http.StatusOK).Write(ctx) } refName, hash, err := git.ResolveGitReference(repo, strings.TrimSpace(ctx.Query("ref")), state.DefaultBranch.String) @@ -103,8 +112,19 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - path := strings.Trim(ctx.Params("*"), "/") - treeResponse, err := git.BuildGitTreeResponse(projectID, refName, path, repo, hash) + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + treeOptions, treeOptionErr := buildGitTreeResponseOptions(ctx) + if treeOptionErr != nil { + response := httputil.NewError("invalid_request", treeOptionErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + treeResponse, err := git.BuildGitTreeResponse(projectID, refName, path, repo, hash, treeOptions) if err != nil { response := httputil.NewError("not_found", fmt.Sprintf("failed to read git tree: %s", err), http.StatusNotFound, map[string]any{"project_id": projectID, "ref": refName, "path": path}, nil) response.WriteLog(handler.logger) @@ -113,6 +133,164 @@ func (handler *Handler) handleGitProjectTreeGET(ctx fiber.Ctx) error { return httputil.JSON(treeResponse, http.StatusOK).Write(ctx) } +func (handler *Handler) handleGitProjectManifestGET(ctx fiber.Ctx) error { + _, _, projectID, _, identity, errResponse := handler.resolveGitProject(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + state, err := handler.loadGitProjectState(projectID, identity) + if err != nil { + response := httputil.NewError("database_error", fmt.Sprintf("failed to read git state: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + if state == nil || state.MirrorPath == "" { + response := httputil.NewError("conflict", fmt.Sprintf("project %s has not been refreshed yet", projectID), http.StatusConflict, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + authorizationHeader := strings.TrimSpace(ctx.Get("Authorization")) + if authorizationHeader != "" { + refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, false) + if err != nil { + handler.logger.Warning("failed to warm git mirror for %s manifest: %v", projectID, err) + } + } + repo, err := git.OpenRepository(state.MirrorPath) + if err != nil { + response := httputil.NewError("integration_error", fmt.Sprintf("failed to open git mirror: %s", err), http.StatusBadGateway, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + if git.RepositoryIsEmpty(repo) { + refName := strings.TrimSpace(ctx.Query("ref")) + if refName == "" { + refName = state.DefaultBranch.String + } + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + return httputil.JSON(&git.GitProjectManifestResponse{ + ProjectID: projectID, + Ref: refName, + Path: path, + EntryCount: 0, + HasMore: false, + Entries: []git.GitTreeEntry{}, + }, http.StatusOK).Write(ctx) + } + refName, hash, err := git.ResolveGitReference(repo, strings.TrimSpace(ctx.Query("ref")), state.DefaultBranch.String) + if err != nil { + response := httputil.NewError("not_found", fmt.Sprintf("failed to resolve git ref: %s", err), http.StatusNotFound, map[string]any{"project_id": projectID, "ref": ctx.Query("ref")}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + manifestOptions, manifestOptionErr := buildGitManifestResponseOptions(ctx) + if manifestOptionErr != nil { + response := httputil.NewError("invalid_request", manifestOptionErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + manifestResponse, err := git.BuildGitManifestResponse(projectID, refName, path, repo, hash, manifestOptions) + if err != nil { + statusCode := http.StatusNotFound + if strings.Contains(strings.ToLower(err.Error()), "cursor") || strings.Contains(strings.ToLower(err.Error()), "limit") { + statusCode = http.StatusBadRequest + } + response := httputil.NewError("not_found", fmt.Sprintf("failed to read git manifest: %s", err), statusCode, map[string]any{"project_id": projectID, "ref": refName, "path": path}, nil) + if statusCode == http.StatusBadRequest { + response = httputil.NewError("invalid_request", err.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID, "ref": refName, "path": path}, nil) + } + response.WriteLog(handler.logger) + return response.Write(ctx) + } + return httputil.JSON(manifestResponse, http.StatusOK).Write(ctx) +} + +func parseOptionalBoolQuery(ctx fiber.Ctx, key string, defaultValue bool) (bool, error) { + value := strings.TrimSpace(ctx.Query(key)) + if value == "" { + return defaultValue, nil + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf("%s must be true or false", key) + } + return parsed, nil +} + +func buildGitTreeResponseOptions(ctx fiber.Ctx) (git.GitTreeResponseOptions, error) { + view := strings.TrimSpace(ctx.Query("view")) + if strings.EqualFold(view, "manifest") { + return git.GitTreeResponseOptions{}, nil + } + + includeSize, err := parseOptionalBoolQuery(ctx, "include_size", false) + if err != nil { + return git.GitTreeResponseOptions{}, err + } + includeLastModified, err := parseOptionalBoolQuery(ctx, "include_last_modified", false) + if err != nil { + return git.GitTreeResponseOptions{}, err + } + includeLFSPointer, err := parseOptionalBoolQuery(ctx, "include_lfs_pointer", false) + if err != nil { + return git.GitTreeResponseOptions{}, err + } + + limitValue := strings.TrimSpace(ctx.Query("limit")) + limit := 0 + if limitValue != "" { + parsedLimit, parseErr := strconv.Atoi(limitValue) + if parseErr != nil || parsedLimit < 0 { + return git.GitTreeResponseOptions{}, fmt.Errorf("limit must be a non-negative integer") + } + limit = parsedLimit + } + + return git.GitTreeResponseOptions{ + IncludeSize: includeSize, + IncludeLastModified: includeLastModified, + IncludeLFSPointer: includeLFSPointer, + Limit: limit, + }, nil +} + +func buildGitManifestResponseOptions(ctx fiber.Ctx) (git.GitManifestResponseOptions, error) { + filesOnly, err := parseOptionalBoolQuery(ctx, "files_only", true) + if err != nil { + return git.GitManifestResponseOptions{}, err + } + limit := 5000 + limitValue := strings.TrimSpace(ctx.Query("limit")) + if limitValue != "" { + parsedLimit, parseErr := strconv.Atoi(limitValue) + if parseErr != nil || parsedLimit <= 0 { + return git.GitManifestResponseOptions{}, fmt.Errorf("limit must be a positive integer") + } + if parsedLimit > 20000 { + return git.GitManifestResponseOptions{}, fmt.Errorf("limit must be less than or equal to 20000") + } + limit = parsedLimit + } + return git.GitManifestResponseOptions{ + Limit: limit, + Cursor: strings.TrimSpace(ctx.Query("cursor")), + FilesOnly: filesOnly, + }, nil +} + func (handler *Handler) handleGitProjectFileGET(ctx fiber.Ctx) error { organization, project, projectID, _, identity, errResponse := handler.resolveGitProject(ctx) if errResponse != nil { @@ -124,7 +302,12 @@ func (handler *Handler) handleGitProjectFileGET(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - path := strings.Trim(ctx.Params("*"), "/") + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } requestedRef := strings.TrimSpace(ctx.Query("ref")) metadata, contentBytes, err := handler.gitService.GetGitHubFileMetadata(ctx.Context(), authorizationHeader, organization, project, identity, requestedRef, path) if err != nil { @@ -155,7 +338,12 @@ func (handler *Handler) handleGitProjectDownloadGET(ctx fiber.Ctx) error { response.WriteLog(handler.logger) return response.Write(ctx) } - path := strings.Trim(ctx.Params("*"), "/") + path, pathErr := decodeGitWildcardPath(ctx) + if pathErr != nil { + response := httputil.NewError("invalid_request", pathErr.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } requestedRef := strings.TrimSpace(ctx.Query("ref")) metadata, _, err := handler.gitService.GetGitHubFileMetadata(ctx.Context(), authorizationHeader, organization, project, identity, requestedRef, path) if err != nil { @@ -178,3 +366,15 @@ func (handler *Handler) handleGitProjectDownloadGET(ctx fiber.Ctx) error { } return ctx.Redirect().To(strings.TrimSpace(metadata.GetDownloadURL())) } + +func decodeGitWildcardPath(ctx fiber.Ctx) (string, error) { + rawPath := strings.Trim(ctx.Params("*"), "/") + if rawPath == "" { + return "", nil + } + decodedPath, err := url.PathUnescape(rawPath) + if err != nil { + return "", fmt.Errorf("invalid git path") + } + return strings.Trim(decodedPath, "/"), nil +} diff --git a/internal/server/http/git/storage_analytics.go b/internal/server/http/git/storage_analytics.go new file mode 100644 index 0000000..c98aea6 --- /dev/null +++ b/internal/server/http/git/storage_analytics.go @@ -0,0 +1,669 @@ +package git + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/calypr/gecko/apierror" + gitcore "github.com/calypr/gecko/internal/git" + "github.com/calypr/gecko/internal/httputil" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/gofiber/fiber/v3" +) + +const defaultStorageChildrenLimit = 100 +const maxStorageChildrenLimit = 1000 + +type storageChildrenRequestOptions struct { + limit int + cursor string + sortBy string + sortOrder string + summaryMode string +} + +func (handler *Handler) handleGitProjectStorageSummaryGET(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + gitSubpath := normalizeAnalyticsSubpath(strings.TrimSpace(ctx.Query("git_subpath"))) + response, err := handler.storageAnalytics.BuildStorageSummary(ctx.Context(), projectCtx.authorizationHeader, projectCtx.organization, projectCtx.project, projectCtx.refName, gitSubpath, projectCtx.mirrorPath, projectCtx.repo, projectCtx.hash) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectStorageChildrenGET(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + gitSubpath := normalizeAnalyticsSubpath(strings.TrimSpace(ctx.Query("git_subpath"))) + options, errResponse := parseStorageChildrenRequestOptions(ctx, projectCtx.projectID) + if errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + response, err := handler.storageAnalytics.BuildStorageChildren( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + gitSubpath, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + options.limit, + options.sortBy, + options.sortOrder, + options.cursor, + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectStorageFolderGET(ctx fiber.Ctx) error { + resolveStart := time.Now() + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + resolveDuration := time.Since(resolveStart) + gitSubpath := normalizeAnalyticsSubpath(strings.TrimSpace(ctx.Query("git_subpath"))) + options, errResponse := parseStorageChildrenRequestOptions(ctx, projectCtx.projectID) + if errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + summaryMode := strings.TrimSpace(options.summaryMode) + if summaryMode == "" { + summaryMode = gitcore.StorageFolderSummarySourceGitIndex + } + timings := &gitcore.StorageFolderTimings{ + DebugPrefix: fmt.Sprintf( + "project_id=%s ref=%s git_subpath=%q limit=%d sort_by=%q sort_order=%q cursor=%t summary_mode=%s", + projectCtx.projectID, + projectCtx.refName, + gitSubpath, + options.limit, + options.sortBy, + options.sortOrder, + options.cursor != "", + summaryMode, + ), + Logf: handler.logger.Info, + } + timings.Record("resolve_git_context", resolveDuration) + forceRefresh := parseBoolQuery(ctx.Query("force_refresh")) || parseBoolQuery(ctx.Query("force_audit_refresh")) || parseBoolQuery(ctx.Query("force_bucket_inventory_refresh")) + buildStart := time.Now() + response, err := handler.storageAnalytics.BuildStorageFolder( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + gitSubpath, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + options.limit, + options.sortBy, + options.sortOrder, + options.cursor, + options.summaryMode, + forceRefresh, + timings, + ) + timings.Record("build_folder", time.Since(buildStart)) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + writeStart := time.Now() + writeErr := httputil.JSON(response, http.StatusOK).Write(ctx) + timings.Record("write_response", time.Since(writeStart)) + handler.logger.Info("storage_folder_request_complete %s", timings.DebugPrefix) + return writeErr +} + +func parseStorageChildrenRequestOptions(ctx fiber.Ctx, projectID string) (storageChildrenRequestOptions, *httputil.ErrorResponse) { + limit, err := parseStorageChildrenLimit(strings.TrimSpace(ctx.Query("limit"))) + if err != nil { + return storageChildrenRequestOptions{}, httputil.NewError("invalid_request", err.Error(), http.StatusBadRequest, map[string]any{"project_id": projectID}, nil) + } + return storageChildrenRequestOptions{ + limit: limit, + cursor: strings.TrimSpace(ctx.Query("cursor")), + sortBy: strings.TrimSpace(ctx.Query("sort_by")), + sortOrder: strings.TrimSpace(ctx.Query("sort_order")), + summaryMode: strings.TrimSpace(ctx.Query("summary_mode")), + }, nil +} + +func parseBoolQuery(raw string) bool { + parsed, err := strconv.ParseBool(strings.TrimSpace(raw)) + return err == nil && parsed +} + +func parseStorageChildrenLimit(rawLimit string) (int, error) { + if rawLimit == "" { + return defaultStorageChildrenLimit, nil + } + parsed, err := strconv.Atoi(rawLimit) + if err != nil || parsed <= 0 { + return 0, fmt.Errorf("limit must be a positive integer") + } + if parsed > maxStorageChildrenLimit { + return maxStorageChildrenLimit, nil + } + return parsed, nil +} + +func (handler *Handler) handleGitProjectDiffAuditPOST(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + requestBody := gitcore.GitProjectDiffAuditRequest{} + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + projectCtx.applyRequestRef(requestBody.Ref) + gitSubpath := normalizeAnalyticsSubpath(requestBody.GitSubpath) + response, err := handler.storageAnalytics.BuildProjectDiffAudit(ctx.Context(), projectCtx.authorizationHeader, projectCtx.organization, projectCtx.project, projectCtx.refName, gitSubpath, projectCtx.mirrorPath, projectCtx.repo, projectCtx.hash) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectStorageCleanupAuditPOST(ctx fiber.Ctx) error { + projectCtx, requestBody, errResponse := handler.parseCleanupAnalyticsRequest(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + response, _, err := handler.storageAnalytics.BuildStorageCleanupAudit( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + requestBody.GitSubpath, + requestBody.SelectedRepoPaths, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + requestBody.CheckStorage, + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, requestBody.GitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func (handler *Handler) handleGitProjectStorageChainAuditPOST(ctx fiber.Ctx) error { + routeOrg := strings.TrimSpace(ctx.Params("orgTitle")) + routeProject := strings.TrimSpace(ctx.Params("projectTitle")) + handler.logger.Info("storage_chain_audit_request_received org=%s project=%s path=%s query=%s", routeOrg, routeProject, ctx.Path(), string(ctx.Request().URI().QueryString())) + timings := &gitcore.StorageChainAuditTimings{ + Logf: handler.logger.Info, + DebugPrefix: fmt.Sprintf("org=%s project=%s", routeOrg, routeProject), + } + resolveStart := time.Now() + timings.StageStart("resolve_git_context") + projectCtx, errResponse := handler.resolveGitAnalyticsContextWithTimings(ctx, timings) + if errResponse != nil { + return errResponse.Write(ctx) + } + timings.Record("resolve_git_context", time.Since(resolveStart)) + timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q", projectCtx.projectID, projectCtx.refName, "") + requestBody := gitcore.GitStorageChainAuditRequest{} + parseStart := time.Now() + timings.StageStart("parse_request_body") + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + timings.Record("parse_request_body", time.Since(parseStart)) + probeMode, ok := gitcore.NormalizeStorageChainProbeMode(requestBody.ProbeMode) + if !ok { + response := httputil.NewError("invalid_request", "probe_mode must be either full or inventory_only", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + validationMode := gitcore.StorageChainValidationModeList + if strings.TrimSpace(requestBody.ValidationMode) != "" { + var valid bool + validationMode, valid = gitcore.NormalizeStorageChainValidationMode(requestBody.ValidationMode) + if !valid { + response := httputil.NewError("invalid_request", "validation_mode must be either list, metadata, or inventory", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + } else if strings.TrimSpace(requestBody.ProbeMode) != "" { + validationMode = gitcore.DefaultStorageChainValidationMode(probeMode, "") + } + bucketMode := gitcore.StorageChainBucketModeValidate + if strings.TrimSpace(requestBody.BucketInventoryMode) != "" { + var valid bool + bucketMode, valid = gitcore.NormalizeStorageChainBucketInventoryMode(requestBody.BucketInventoryMode) + if !valid { + response := httputil.NewError("invalid_request", "bucket_inventory_mode must be either validate or items", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + } + if validationMode == gitcore.StorageChainValidationModeInventory && strings.TrimSpace(requestBody.BucketInventoryMode) == "" { + bucketMode = gitcore.StorageChainBucketModeItems + } + findingLimit := requestBody.FindingLimit + if findingLimit < -1 { + response := httputil.NewError("invalid_request", "finding_limit must be -1, 0, or a positive integer", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + bucketPathPrefix := normalizeAnalyticsSubpath(requestBody.BucketPathPrefix) + if bucketPathPrefix != "" && bucketMode != gitcore.StorageChainBucketModeItems { + response := httputil.NewError("invalid_request", "bucket_path_prefix requires bucket_inventory_mode=items", http.StatusBadRequest, map[string]any{"project_id": projectCtx.projectID}, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) + } + projectCtx.applyRequestRef(requestBody.Ref) + gitSubpath := normalizeAnalyticsSubpath(requestBody.GitSubpath) + findingKind := strings.TrimSpace(requestBody.FindingKind) + timings.DebugPrefix = fmt.Sprintf("project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_prefix=%q finding_kind=%q finding_limit=%d", projectCtx.projectID, projectCtx.refName, gitSubpath, validationMode, probeMode, bucketMode, bucketPathPrefix, findingKind, findingLimit) + handler.logger.Info("storage_chain_audit_request_start %s", timings.DebugPrefix) + forceAuditRefresh := requestBody.Refresh || requestBody.ForceAuditRefresh || requestBody.ForceBucketInventoryRefresh + response, err := handler.storageAnalytics.BuildStorageChainAuditWithOptions( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + gitSubpath, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + gitcore.StorageChainAuditOptions{ + ProbeMode: probeMode, + ValidationMode: validationMode, + BucketInventoryMode: bucketMode, + BucketPathPrefix: bucketPathPrefix, + FindingKind: findingKind, + FindingLimit: findingLimit, + ForceAuditRefresh: forceAuditRefresh, + Timings: timings, + }, + ) + if err != nil { + handler.logger.Info("storage_chain_audit_request_error %s error=%q timings=%s", timings.DebugPrefix, err.Error(), formatStorageChainTimingSnapshot(timings)) + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, gitSubpath, err) + } + writeStart := time.Now() + timings.StageStart("json_response") + writeErr := httputil.JSON(response, http.StatusOK).Write(ctx) + timings.Record("json_response", time.Since(writeStart)) + timings.RecordMemory( + "json_response", + "total_findings", response.Summary.TotalFindings, + "returned_findings", response.Summary.ReturnedFindings, + "bucket_objects", response.Summary.BucketObjectCount, + "syfon_records", response.Summary.SyfonRecordCount, + "git_files", response.Summary.GitTrackedFileCount, + ) + handler.logStorageChainAuditTimings(projectCtx, gitSubpath, probeMode, bucketMode, response, timings) + return writeErr +} + +func (handler *Handler) handleGitProjectStorageChainRegisterGitOnlyPOST(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + requestBody := gitcore.GitOnlySyfonRegistrationRequest{} + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + projectCtx.applyRequestRef(requestBody.Ref) + requestBody.Ref = projectCtx.refName + requestBody.RepoPaths = normalizeAnalyticsPathList(requestBody.RepoPaths) + response, err := handler.storageAnalytics.RegisterGitOnlySyfonRecords( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + projectCtx.refName, + projectCtx.mirrorPath, + projectCtx.repo, + projectCtx.hash, + requestBody, + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, "", err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +func formatStorageChainTimingSnapshot(timings *gitcore.StorageChainAuditTimings) string { + parts := make([]string, 0) + for _, stage := range timings.Snapshot() { + if stage.Stage == "" { + continue + } + parts = append(parts, fmt.Sprintf("%s_ms=%d", stage.Stage, stage.Duration.Milliseconds())) + } + return strings.Join(parts, ",") +} + +func (handler *Handler) handleGitProjectStorageCleanupApplyPOST(ctx fiber.Ctx) error { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return errResponse.Write(ctx) + } + requestBody := gitcore.GitStorageCleanupApplyRequest{} + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return errResponse.Write(ctx) + } + projectCtx.applyRequestRef(requestBody.Ref) + requestBody.GitSubpath = normalizeAnalyticsSubpath(requestBody.GitSubpath) + requestBody.SelectedRepoPaths = normalizeAnalyticsPathList(requestBody.SelectedRepoPaths) + for i := range requestBody.Actions { + requestBody.Actions[i].NormalizedPath = normalizeAnalyticsSubpath(requestBody.Actions[i].NormalizedPath) + requestBody.Actions[i].Kind = strings.TrimSpace(requestBody.Actions[i].Kind) + requestBody.Actions[i].Action = strings.TrimSpace(requestBody.Actions[i].Action) + } + for i := range requestBody.Findings { + requestBody.Findings[i].Kind = strings.TrimSpace(requestBody.Findings[i].Kind) + requestBody.Findings[i].NormalizedPath = normalizeAnalyticsSubpath(requestBody.Findings[i].NormalizedPath) + requestBody.Findings[i].ObjectIDs = normalizeStringList(requestBody.Findings[i].ObjectIDs) + requestBody.Findings[i].BucketObjectURL = strings.TrimSpace(requestBody.Findings[i].BucketObjectURL) + requestBody.Findings[i].BucketObjectURLs = normalizeAnalyticsPathList(requestBody.Findings[i].BucketObjectURLs) + requestBody.Findings[i].AccessURLs = normalizeAnalyticsPathList(requestBody.Findings[i].AccessURLs) + requestBody.Findings[i].AvailableActions = normalizeStringList(requestBody.Findings[i].AvailableActions) + requestBody.Findings[i].DefaultAction = strings.TrimSpace(requestBody.Findings[i].DefaultAction) + requestBody.Findings[i].SuggestedAction = strings.TrimSpace(requestBody.Findings[i].SuggestedAction) + if requestBody.Findings[i].Evidence != nil { + requestBody.Findings[i].Evidence.ObjectIDs = normalizeStringList(requestBody.Findings[i].Evidence.ObjectIDs) + requestBody.Findings[i].Evidence.AccessURLs = normalizeAnalyticsPathList(requestBody.Findings[i].Evidence.AccessURLs) + requestBody.Findings[i].Evidence.BucketObjectURLs = normalizeAnalyticsPathList(requestBody.Findings[i].Evidence.BucketObjectURLs) + requestBody.Findings[i].Evidence.SourcePaths = normalizeAnalyticsPathList(requestBody.Findings[i].Evidence.SourcePaths) + } + for j := range requestBody.Findings[i].Records { + record := &requestBody.Findings[i].Records[j] + record.ObjectID = strings.TrimSpace(record.ObjectID) + record.Checksum = strings.TrimSpace(record.Checksum) + record.NormalizedPath = normalizeAnalyticsSubpath(record.NormalizedPath) + record.CleanupScope = strings.TrimSpace(record.CleanupScope) + record.AccessURLs = normalizeAnalyticsPathList(record.AccessURLs) + for k := range record.AccessMethods { + record.AccessMethods[k].AccessID = strings.TrimSpace(record.AccessMethods[k].AccessID) + record.AccessMethods[k].Type = strings.TrimSpace(record.AccessMethods[k].Type) + record.AccessMethods[k].URL = strings.TrimSpace(record.AccessMethods[k].URL) + record.AccessMethods[k].Headers = normalizeStringList(record.AccessMethods[k].Headers) + } + for k := range record.AccessProbes { + record.AccessProbes[k].URL = strings.TrimSpace(record.AccessProbes[k].URL) + record.AccessProbes[k].Status = strings.TrimSpace(record.AccessProbes[k].Status) + record.AccessProbes[k].ErrorKind = strings.TrimSpace(record.AccessProbes[k].ErrorKind) + } + } + } + response, err := handler.storageAnalytics.ApplyStorageCleanup( + ctx.Context(), + projectCtx.authorizationHeader, + projectCtx.organization, + projectCtx.project, + requestBody.SelectedRepoPaths, + requestBody.Actions, + requestBody.Findings, + requestBody.DeleteRepoOrphans, + requestBody.DeleteStaleDuplicates, + requestBody.DeleteBucketOnlyObjects, + requestBody.RepairBrokenBucketMappings, + requestBody.DryRun, + ) + if err != nil { + return handler.writeGitAnalyticsError(ctx, projectCtx.projectID, projectCtx.refName, requestBody.GitSubpath, err) + } + return httputil.JSON(response, http.StatusOK).Write(ctx) +} + +type gitAnalyticsContext struct { + organization string + project string + projectID string + authorizationHeader string + defaultBranch string + refName string + mirrorPath string + repo *gogit.Repository + hash plumbing.Hash +} + +func (ctx *gitAnalyticsContext) applyRequestRef(ref string) { + ref = strings.TrimSpace(ref) + if ref == "" || ref == ctx.refName { + return + } + refName, hash, err := gitcore.ResolveGitReference(ctx.repo, ref, ctx.defaultBranch) + if err != nil { + return + } + ctx.refName = refName + ctx.hash = hash +} + +func (handler *Handler) parseCleanupAnalyticsRequest(ctx fiber.Ctx) (*gitAnalyticsContext, gitcore.GitStorageCleanupAuditRequest, *httputil.ErrorResponse) { + projectCtx, errResponse := handler.resolveGitAnalyticsContext(ctx) + if errResponse != nil { + return nil, gitcore.GitStorageCleanupAuditRequest{}, errResponse + } + requestBody := gitcore.GitStorageCleanupAuditRequest{} + if errResponse := parseOptionalAnalyticsBody(ctx, &requestBody, map[string]any{"project_id": projectCtx.projectID}); errResponse != nil { + errResponse.WriteLog(handler.logger) + return nil, gitcore.GitStorageCleanupAuditRequest{}, errResponse + } + projectCtx.applyRequestRef(requestBody.Ref) + requestBody.GitSubpath = normalizeAnalyticsSubpath(requestBody.GitSubpath) + requestBody.SelectedRepoPaths = normalizeAnalyticsPathList(requestBody.SelectedRepoPaths) + return projectCtx, requestBody, nil +} + +func (handler *Handler) resolveGitAnalyticsContext(ctx fiber.Ctx) (*gitAnalyticsContext, *httputil.ErrorResponse) { + return handler.resolveGitAnalyticsContextWithMirrorPolicy(ctx, nil, false) +} + +func (handler *Handler) resolveGitAnalyticsContextWithTimings(ctx fiber.Ctx, timings *gitcore.StorageChainAuditTimings) (*gitAnalyticsContext, *httputil.ErrorResponse) { + return handler.resolveGitAnalyticsContextWithMirrorPolicy(ctx, timings, true) +} + +func (handler *Handler) resolveGitAnalyticsContextWithMirrorPolicy(ctx fiber.Ctx, timings *gitcore.StorageChainAuditTimings, requireCurrentMirror bool) (*gitAnalyticsContext, *httputil.ErrorResponse) { + if handler.storageAnalytics == nil { + response := httputil.NewError("internal_error", "storage analytics service is not configured", http.StatusInternalServerError, nil, nil) + response.WriteLog(handler.logger) + return nil, response + } + organization, project, projectID, _, identity, errResponse := handler.resolveGitProject(ctx) + if errResponse != nil { + return nil, errResponse + } + state, err := handler.loadGitProjectState(projectID, identity) + if err != nil { + response := httputil.NewError("database_error", fmt.Sprintf("failed to read git state: %s", err), http.StatusInternalServerError, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } + if state == nil || state.MirrorPath == "" { + response := httputil.NewError("conflict", fmt.Sprintf("project %s has not been refreshed yet", projectID), http.StatusConflict, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } + authorizationHeader := strings.TrimSpace(ctx.Get("Authorization")) + if authorizationHeader != "" { + start := time.Now() + timings.StageStart("mirror_warmup") + refreshCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + state, err = handler.ensureMirrorReadyForRead(refreshCtx, authorizationHeader, projectID, identity, state, requireCurrentMirror) + timings.Record("mirror_warmup", time.Since(start)) + if err != nil { + if requireCurrentMirror { + response := httputil.NewError("integration_error", fmt.Sprintf("storage audit requires a current Git mirror: %s", err), http.StatusBadGateway, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } + handler.logger.Warning("failed to warm git mirror for %s analytics: %v", projectID, err) + } + } + repo, err := gitcore.OpenRepository(state.MirrorPath) + if err != nil { + response := httputil.NewError("integration_error", fmt.Sprintf("failed to open git mirror: %s", err), http.StatusBadGateway, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } + if gitcore.RepositoryIsEmpty(repo) { + response := httputil.NewError("conflict", fmt.Sprintf("project %s has no Git content yet", projectID), http.StatusConflict, map[string]any{"project_id": projectID}, nil) + response.WriteLog(handler.logger) + return nil, response + } + defaultBranch := state.DefaultBranch.String + refName, hash, err := gitcore.ResolveGitReference(repo, strings.TrimSpace(ctx.Query("ref")), defaultBranch) + if err != nil { + response := httputil.NewError("not_found", fmt.Sprintf("failed to resolve git ref: %s", err), http.StatusNotFound, map[string]any{"project_id": projectID, "ref": ctx.Query("ref")}, nil) + response.WriteLog(handler.logger) + return nil, response + } + return &gitAnalyticsContext{ + organization: organization, + project: project, + projectID: projectID, + authorizationHeader: authorizationHeader, + defaultBranch: defaultBranch, + refName: refName, + mirrorPath: state.MirrorPath, + repo: repo, + hash: hash, + }, nil +} + +func parseOptionalAnalyticsBody(ctx fiber.Ctx, target any, details map[string]any) *httputil.ErrorResponse { + body := ctx.Body() + if len(body) == 0 { + return nil + } + return httputil.ParseJSONBody(body, target, details) +} + +func normalizeAnalyticsSubpath(raw string) string { + return strings.Trim(strings.TrimSpace(raw), "/") +} + +func normalizeAnalyticsPathList(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + normalized := normalizeAnalyticsSubpath(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + +func normalizeStringList(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + normalized := strings.TrimSpace(value) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out +} + +func (handler *Handler) writeGitAnalyticsError(ctx fiber.Ctx, projectID string, ref string, gitSubpath string, err error) error { + statusCode := http.StatusBadGateway + errorType := "integration_error" + errorMessage := strings.ToLower(err.Error()) + if strings.Contains(errorMessage, "storage children cursor") { + statusCode = http.StatusBadRequest + errorType = "invalid_request" + } else if strings.Contains(errorMessage, "cleanup apply") || strings.Contains(errorMessage, "cleanup finding") || strings.Contains(errorMessage, "cleanup action") || strings.Contains(errorMessage, "unsupported cleanup") || strings.Contains(errorMessage, "selected cleanup paths") || strings.Contains(errorMessage, "git-only syfon registration") { + statusCode = http.StatusBadRequest + errorType = "invalid_request" + } else if strings.Contains(errorMessage, "git tree path") { + statusCode = http.StatusNotFound + errorType = "not_found" + } + response := httputil.NewError(apierror.Type(errorType), err.Error(), statusCode, map[string]any{ + "project_id": projectID, + "ref": ref, + "git_subpath": gitSubpath, + }, nil) + response.WriteLog(handler.logger) + return response.Write(ctx) +} + +func (handler *Handler) logStorageChainAuditTimings(projectCtx *gitAnalyticsContext, gitSubpath string, probeMode string, bucketMode string, response *gitcore.GitStorageChainAuditResponse, timings *gitcore.StorageChainAuditTimings) { + if projectCtx == nil || response == nil { + return + } + parts := make([]string, 0) + for _, stage := range timings.Snapshot() { + if stage.Stage == "" { + continue + } + parts = append(parts, fmt.Sprintf("%s_ms=%d", stage.Stage, stage.Duration.Milliseconds())) + } + bucketPathExists := "unknown" + if response.Summary.BucketPathExists != nil { + bucketPathExists = strconv.FormatBool(*response.Summary.BucketPathExists) + } + handler.logger.Info( + "storage_chain_audit project_id=%s ref=%s git_subpath=%q validation_mode=%s probe_mode=%s bucket_inventory_mode=%s bucket_path_exists=%s bucket_summary_mode=%s bucket_inventory_available=%t audit_cache_hit=%t audit_cache_source=%s audit_cache_age_seconds=%d audit_refresh_duration_ms=%d findings=%d returned_findings=%d findings_truncated=%t finding_limit=%d bucket_objects=%d syfon_records=%d git_files=%d %s", + projectCtx.projectID, + projectCtx.refName, + gitSubpath, + response.Summary.ValidationMode, + probeMode, + bucketMode, + bucketPathExists, + response.Summary.BucketSummaryMode, + response.Summary.BucketInventoryAvailable, + response.Summary.AuditCacheHit, + response.Summary.AuditCacheSource, + response.Summary.AuditCacheAgeSeconds, + response.Summary.AuditRefreshDurationMs, + response.Summary.TotalFindings, + response.Summary.ReturnedFindings, + response.Summary.FindingsTruncated, + response.Summary.FindingLimit, + response.Summary.BucketObjectCount, + response.Summary.SyfonRecordCount, + response.Summary.GitTrackedFileCount, + strings.Join(parts, " "), + ) +} diff --git a/internal/server/http/git/storage_analytics_options_test.go b/internal/server/http/git/storage_analytics_options_test.go new file mode 100644 index 0000000..afdaf39 --- /dev/null +++ b/internal/server/http/git/storage_analytics_options_test.go @@ -0,0 +1,102 @@ +package git + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "net/http/httptest" + "os" + "testing" + + geckologging "github.com/calypr/gecko/internal/logging" + "github.com/gofiber/fiber/v3" +) + +func TestParseStorageChildrenLimit(t *testing.T) { + tests := []struct { + name string + raw string + want int + wantError bool + }{ + {name: "default", raw: "", want: defaultStorageChildrenLimit}, + {name: "explicit", raw: "25", want: 25}, + {name: "cap", raw: "5000", want: maxStorageChildrenLimit}, + {name: "invalid", raw: "nope", wantError: true}, + {name: "zero", raw: "0", wantError: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := parseStorageChildrenLimit(test.raw) + if test.wantError { + if err == nil { + t.Fatalf("expected error for limit %q", test.raw) + } + return + } + if err != nil { + t.Fatalf("parse limit: %v", err) + } + if got != test.want { + t.Fatalf("expected limit %d, got %d", test.want, got) + } + }) + } +} + +func TestParseStorageChildrenRequestOptionsReadsCursorAndSort(t *testing.T) { + app := fiber.New() + app.Get("/storage/children", func(ctx fiber.Ctx) error { + options, errResponse := parseStorageChildrenRequestOptions(ctx, "org/proj") + if errResponse != nil { + return errResponse.Write(ctx) + } + return ctx.JSON(map[string]any{ + "cursor": options.cursor, + "limit": options.limit, + "sort_by": options.sortBy, + "sort_order": options.sortOrder, + }) + }) + req := httptest.NewRequest(http.MethodGet, "/storage/children?limit=15&cursor=abc123&sort_by=name&sort_order=asc", nil) + resp, err := app.Test(req) + if err != nil { + t.Fatalf("run request: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + var body map[string]any + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + if body["cursor"] != "abc123" || body["sort_by"] != "name" || body["sort_order"] != "asc" || body["limit"].(float64) != 15 { + t.Fatalf("unexpected parsed options: %+v", body) + } +} + +func TestWriteGitAnalyticsErrorClassifiesCleanupApplyValidation(t *testing.T) { + handler := &Handler{logger: &geckologging.Handler{Logger: log.New(os.Stdout, "", 0)}} + app := fiber.New() + app.Post("/apply", func(ctx fiber.Ctx) error { + return handler.writeGitAnalyticsError(ctx, "org/proj", "main", "data", fmt.Errorf("cleanup apply requires findings from a prior audit; refusing to rebuild audit during apply")) + }) + resp, err := app.Test(httptest.NewRequest(http.MethodPost, "/apply", nil)) + if err != nil { + t.Fatalf("run request: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected status 400, got %d", resp.StatusCode) + } + var body map[string]any + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + errorBody, ok := body["error"].(map[string]any) + if !ok || errorBody["type"] != "invalid_request" { + t.Fatalf("expected invalid_request error, got %+v", body) + } +} diff --git a/internal/server/http/register.go b/internal/server/http/register.go index d9417ae..9a89aaa 100644 --- a/internal/server/http/register.go +++ b/internal/server/http/register.go @@ -1,12 +1,10 @@ package httpapi import ( - "net/http" "strings" "github.com/calypr/gecko/internal/httputil" "github.com/calypr/gecko/internal/server/http/config" - "github.com/calypr/gecko/internal/server/http/directory" "github.com/calypr/gecko/internal/server/http/git" "github.com/calypr/gecko/internal/server/http/health" "github.com/calypr/gecko/internal/server/http/shared" @@ -19,14 +17,13 @@ type Dependencies = shared.Dependencies func Register(app *fiber.App, deps Dependencies) { handler := shared.NewHandler(deps) - authzHandler := servermw.NewFenceUserAccessHandler(http.DefaultClient) + authzHandler := servermw.NewFenceUserAccessHandler(nil) app.Get("/swagger/doc.json", func(ctx fiber.Ctx) error { return ctx.SendFile("./docs/swagger.json") }) health.RegisterRoutes(app, handler) - directory.RegisterRoutes(app, handler, authzHandler) config.RegisterRoutes(app, handler, authzHandler) git.RegisterRoutes(app, handler, authzHandler) vector.RegisterRoutes(app, handler) diff --git a/internal/server/http/shared/handler.go b/internal/server/http/shared/handler.go index fdceeab..a2214b3 100644 --- a/internal/server/http/shared/handler.go +++ b/internal/server/http/shared/handler.go @@ -1,13 +1,13 @@ package shared import ( - "net/http" "os" "strings" "github.com/bmeg/grip/gripql" "github.com/calypr/gecko/internal/git" gintegrationsyfon "github.com/calypr/gecko/internal/integrations/syfon" + "github.com/calypr/gecko/internal/presentation" servermw "github.com/calypr/gecko/internal/server/middleware" "github.com/calypr/gecko/internal/thumbnail" "github.com/jmoiron/sqlx" @@ -16,34 +16,38 @@ import ( ) type Dependencies struct { - DB *sqlx.DB - Logger arborist.Logger - JWTApp arborist.JWTDecoder - QdrantClient *qdrant.Client - GripqlClient *gripql.Client - GripGraphName string - GitService *git.GitService - ThumbnailStore thumbnail.Manager + DB *sqlx.DB + Logger arborist.Logger + JWTApp arborist.JWTDecoder + QdrantClient *qdrant.Client + GripqlClient *gripql.Client + GripGraphName string + GitService *git.GitService + ThumbnailStore thumbnail.Manager + PresentationStore presentation.Manager } type Handler struct { - DB *sqlx.DB - Logger arborist.Logger - JWTApp arborist.JWTDecoder - QdrantClient *qdrant.Client - GripqlClient *gripql.Client - GripGraphName string - GitService *git.GitService - ProjectSetup *git.SetupService - ProjectSync *git.ReconcileService - ThumbnailStore thumbnail.Manager + DB *sqlx.DB + Logger arborist.Logger + JWTApp arborist.JWTDecoder + QdrantClient *qdrant.Client + GripqlClient *gripql.Client + GripGraphName string + GitService *git.GitService + ProjectSetup *git.SetupService + ProjectSync *git.ReconcileService + SyfonManager *gintegrationsyfon.Manager + ThumbnailStore thumbnail.Manager + PresentationStore presentation.Manager } func NewHandler(deps Dependencies) *Handler { var projectSetup *git.SetupService var projectSync *git.ReconcileService + var storageManager *gintegrationsyfon.Manager if deps.GitService != nil { - storageManager := gintegrationsyfon.NewManager(strings.TrimSpace(os.Getenv("SYFON_DATA_API_BASE_URL")), http.DefaultClient) + storageManager = gintegrationsyfon.NewManager(strings.TrimSpace(os.Getenv("SYFON_DATA_API_BASE_URL")), nil) projectSetup = git.NewSetupService(deps.DB, deps.GitService, storageManager, servermw.NewFenceUserAccessHandler(nil)) projectSync = git.NewReconcileService( deps.DB, @@ -52,16 +56,17 @@ func NewHandler(deps Dependencies) *Handler { ) } return &Handler{ - DB: deps.DB, - Logger: deps.Logger, - JWTApp: deps.JWTApp, - QdrantClient: deps.QdrantClient, - GripqlClient: deps.GripqlClient, - GripGraphName: deps.GripGraphName, - GitService: deps.GitService, - ProjectSetup: projectSetup, - ProjectSync: projectSync, - ThumbnailStore: deps.ThumbnailStore, + DB: deps.DB, + Logger: deps.Logger, + JWTApp: deps.JWTApp, + QdrantClient: deps.QdrantClient, + GripqlClient: deps.GripqlClient, + GripGraphName: deps.GripGraphName, + GitService: deps.GitService, + ProjectSetup: projectSetup, + ProjectSync: projectSync, + SyfonManager: storageManager, + ThumbnailStore: deps.ThumbnailStore, + PresentationStore: deps.PresentationStore, } } - diff --git a/internal/server/http/shared/helpers.go b/internal/server/http/shared/helpers.go index a71d4dc..2686d92 100644 --- a/internal/server/http/shared/helpers.go +++ b/internal/server/http/shared/helpers.go @@ -1,16 +1,6 @@ package shared -import ( - "fmt" - "net/http" - "strings" - - "github.com/calypr/gecko/apierror" - "github.com/calypr/gecko/internal/git" - "github.com/calypr/gecko/internal/httputil" - servermw "github.com/calypr/gecko/internal/server/middleware" - "github.com/gofiber/fiber/v3" -) +import "github.com/gofiber/fiber/v3" func ConfigTypeMiddleware(configType string) fiber.Handler { return func(ctx fiber.Ctx) error { @@ -18,57 +8,3 @@ func ConfigTypeMiddleware(configType string) fiber.Handler { return ctx.Next() } } - -func (handler *Handler) WriteAppError(ctx fiber.Ctx, err error) error { - if err == nil { - return nil - } - appErr, ok := err.(*git.Error) - if !ok { - response := httputil.NewError(apierror.Type("internal_error"), err.Error(), http.StatusInternalServerError, nil, nil) - response.WriteLog(handler.Logger) - return response.Write(ctx) - } - errorType := apierror.Type("internal_error") - switch appErr.Kind { - case git.ErrorKindValidation: - errorType = apierror.TypeValidationFailed - case git.ErrorKindForbidden: - errorType = apierror.TypeForbidden - case git.ErrorKindIntegration: - errorType = apierror.Type("integration_error") - case git.ErrorKindNotFound: - errorType = apierror.TypeNotFound - case git.ErrorKindDatabase: - errorType = apierror.TypeDatabaseError - case git.ErrorKindUnauthorized: - errorType = apierror.TypeMissingAuthorization - } - statusCode := appErr.StatusCode - if statusCode == 0 { - statusCode = http.StatusInternalServerError - } - response := httputil.NewError(errorType, appErr.Error(), statusCode, appErr.Details, nil) - response.WriteLog(handler.Logger) - return response.Write(ctx) -} - -func (handler *Handler) AuthenticatedUserID(ctx fiber.Ctx) (string, *httputil.ErrorResponse) { - authorizationHeader, tokenErr := servermw.ValidateAuthorizationHeader(ctx.Get("Authorization")) - if tokenErr != nil { - return "", httputil.NewError(apierror.TypeMissingAuthorization, tokenErr.Error(), http.StatusUnauthorized, nil, nil) - } - if handler.JWTApp == nil { - return "", httputil.NewError(apierror.TypeInvalidJWTHandler, "JWT validation is not configured", http.StatusUnauthorized, nil, nil) - } - claims, err := handler.JWTApp.Decode(servermw.CleanAccessToken(authorizationHeader)) - if err != nil { - return "", httputil.NewError(apierror.TypeUnauthorized, fmt.Sprintf("failed to decode authorization token: %s", err), http.StatusUnauthorized, nil, nil) - } - for _, claim := range []string{"sub", "username", "email"} { - if value, ok := (*claims)[claim].(string); ok && strings.TrimSpace(value) != "" { - return strings.TrimSpace(value), nil - } - } - return "", httputil.NewError(apierror.TypeUnauthorized, "authorization token does not include a stable user id", http.StatusUnauthorized, nil, nil) -} diff --git a/internal/server/middleware/access.go b/internal/server/middleware/access.go index bf6faba..3e58cd6 100644 --- a/internal/server/middleware/access.go +++ b/internal/server/middleware/access.go @@ -3,7 +3,6 @@ package middleware import ( "fmt" "net/url" - "sort" "strings" ) @@ -63,18 +62,3 @@ func ResourceListAllowsOrganization(resources []string, organization string) boo } return false } - -func ResourceListAllowedOrganizations(resources []string) []string { - seen := make(map[string]struct{}) - for _, resource := range resources { - if organization, ok := ResourcePathOrganization(resource); ok { - seen[organization] = struct{}{} - } - } - organizations := make([]string, 0, len(seen)) - for organization := range seen { - organizations = append(organizations, organization) - } - sort.Strings(organizations) - return organizations -} diff --git a/internal/server/middleware/auth.go b/internal/server/middleware/auth.go index c64abb1..366e8a4 100644 --- a/internal/server/middleware/auth.go +++ b/internal/server/middleware/auth.go @@ -115,34 +115,6 @@ func GeneralAuth(logger arborist.Logger, authzHandler ResourceAccessHandler, met } } -func BaseConfigsAuth(logger arborist.Logger, authzHandler ResourceAccessHandler, method, service, resourcePath string) fiber.Handler { - return func(ctx fiber.Ctx) error { - authorizationHeader := ctx.Get("Authorization") - if authorizationHeader == "" { - return writeError(ctx, logger, httputil.NewError(apierror.TypeMissingAuthorization, "Authorization token not provided", http.StatusUnauthorized, nil, nil)) - } - - if _, ok := authzHandler.(*FenceUserAccessHandler); !ok { - return writeError(ctx, logger, httputil.NewError("internal_server_error", "Invalid JWT handler configuration", http.StatusInternalServerError, nil, nil)) - } - - allowed, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, method, service, resourcePath) - if err != nil { - if serverErr, ok := err.(*ggmw.ServerError); ok { - return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) - } - if accessErr, ok := err.(*AccessError); ok { - return writeError(ctx, logger, httputil.NewError(serviceErrorType(accessErr.StatusCode), accessErr.Message, accessErr.StatusCode, nil, nil)) - } - return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) - } - if !allowed { - return writeError(ctx, logger, httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User does not have required %s permission on resource %s", method, "/programs"), http.StatusForbidden, map[string]any{"resource": resourcePath, "method": method}, nil)) - } - return ctx.Next() - } -} - func ProjectConfigAuth(logger arborist.Logger, authzHandler ResourceAccessHandler, method string) fiber.Handler { return func(ctx fiber.Ctx) error { authorizationHeader := ctx.Get("Authorization") @@ -245,6 +217,110 @@ func GitProjectAuth(logger arborist.Logger, jwtHandler ResourceAccessHandler) fi } } +func GitProjectMutationAuth(logger arborist.Logger, authzHandler ResourceAccessHandler, method string) fiber.Handler { + return func(ctx fiber.Ctx) error { + authorizationHeader := ctx.Get("Authorization") + if authorizationHeader == "" { + return writeError(ctx, logger, httputil.NewError(apierror.TypeMissingAuthorization, "Authorization token not provided", http.StatusUnauthorized, nil, nil)) + } + organization := strings.TrimSpace(ctx.Params("orgTitle")) + project := strings.TrimSpace(ctx.Params("projectTitle")) + if organization == "" || project == "" { + return writeError(ctx, logger, httputil.NewError("invalid_request", "organization and project are required", http.StatusBadRequest, nil, nil)) + } + resourcePath := ProgramProjectResourcePath(organization, project) + allowed, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, method, "*", resourcePath) + if err != nil { + if serverErr, ok := err.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) + } + if !allowed { + anyList, listErr := authzHandler.GetAllowedResources(authorizationHeader, method, "*") + if listErr != nil { + if serverErr, ok := listErr.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, listErr.Error(), http.StatusForbidden, nil, nil)) + } + resources, conversionErr := convertAnyToStringSlice(anyList) + if conversionErr != nil { + return writeError(ctx, logger, conversionErr) + } + allowed = resourceListAllowsProjectAdminAction(resources, organization, project) + } + if !allowed { + return writeError(ctx, logger, httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User does not have required %s permission on resource %s", method, resourcePath), http.StatusForbidden, map[string]any{ + "resource": resourcePath, + "method": method, + "organization": organization, + "project": project, + }, nil)) + } + return ctx.Next() + } +} + +// GitProjectSetupAuth is for setup-adjacent routes that operate on an existing +// project or organization context, such as storage configuration. Do not place +// this in front of the bootstrap /git/projects/:orgTitle/:projectTitle/setup +// route because Arborist must decide whether the caller can create or attach to +// missing org/project resources. +func GitProjectSetupAuth(logger arborist.Logger, authzHandler ResourceAccessHandler) fiber.Handler { + return func(ctx fiber.Ctx) error { + authorizationHeader := ctx.Get("Authorization") + if authorizationHeader == "" { + return writeError(ctx, logger, httputil.NewError(apierror.TypeMissingAuthorization, "Authorization token not provided", http.StatusUnauthorized, nil, nil)) + } + organization := strings.TrimSpace(ctx.Params("orgTitle")) + project := strings.TrimSpace(ctx.Params("projectTitle")) + if organization == "" || project == "" { + return writeError(ctx, logger, httputil.NewError("invalid_request", "organization and project are required", http.StatusBadRequest, nil, nil)) + } + projectResource := ProgramProjectResourcePath(organization, project) + projectReadable, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, "read", "*", projectResource) + if err != nil { + if serverErr, ok := err.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) + } + if projectReadable { + return ctx.Next() + } + orgProjectsResource := fmt.Sprintf("/programs/%s/projects", organization) + orgCreateAllowed, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, "create-descendant", "arborist", orgProjectsResource) + if err != nil { + if serverErr, ok := err.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) + } + if orgCreateAllowed { + return ctx.Next() + } + orgResource := fmt.Sprintf("/programs/%s", organization) + orgManageAllowed, err := authzHandler.CheckResourceServiceAccess(authorizationHeader, "manage-owners", "arborist", orgResource) + if err != nil { + if serverErr, ok := err.(*AccessError); ok { + return writeError(ctx, logger, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil)) + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil)) + } + if orgManageAllowed { + return ctx.Next() + } + return writeError(ctx, logger, httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User does not have required setup permission on resource %s", projectResource), http.StatusForbidden, map[string]any{ + "resource": projectResource, + "organization": organization, + "project": project, + "org_resource": orgResource, + "projects_path": orgProjectsResource, + }, nil)) + } +} + func GitOrganizationAuth(logger arborist.Logger, jwtHandler ResourceAccessHandler) fiber.Handler { return func(ctx fiber.Ctx) error { if jwtHandler == nil { @@ -308,35 +384,6 @@ func writeError(ctx fiber.Ctx, logger arborist.Logger, response *httputil.ErrorR return response.Write(ctx) } -// Migrated from internal/authz -func GetProjectsFromToken(ctx fiber.Ctx, authzHandler ResourceAccessHandler, method string, service string) ([]any, *httputil.ErrorResponse) { - token := ctx.Get("Authorization") - if token == "" { - return nil, httputil.NewError(apierror.TypeMissingAuthorization, "Authorization token not provided", http.StatusUnauthorized, nil, nil) - } - anyList, err := authzHandler.GetAllowedResources(token, method, service) - if err != nil { - if serverErr, ok := err.(*AccessError); ok { - return nil, httputil.NewError(serviceErrorType(serverErr.StatusCode), serverErr.Message, serverErr.StatusCode, nil, nil) - } - return nil, httputil.NewError(apierror.TypeAuthorizationServiceError, err.Error(), http.StatusForbidden, nil, nil) - } - return anyList, nil -} - -// Migrated from internal/authz -func ParseAccess(resourceList []string, resource string, method string) *httputil.ErrorResponse { - if len(resourceList) == 0 { - return httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User is not allowed to %s on any resource path", method), http.StatusForbidden, map[string]any{"resource": resource, "method": method}, nil) - } - for _, v := range resourceList { - if v == resource { - return nil - } - } - return httputil.NewError(apierror.TypeForbidden, fmt.Sprintf("User is not allowed to %s on resource path: %s", method, resource), http.StatusForbidden, map[string]any{"resource": resource, "method": method}, nil) -} - func CleanAccessToken(raw string) string { token := strings.TrimSpace(raw) if strings.HasPrefix(strings.ToLower(token), "bearer ") { diff --git a/internal/server/middleware/fence_user_test.go b/internal/server/middleware/fence_user_test.go index 7eb7fa5..35ac389 100644 --- a/internal/server/middleware/fence_user_test.go +++ b/internal/server/middleware/fence_user_test.go @@ -2,22 +2,6 @@ package middleware import "testing" -func TestSnapshotAllows(t *testing.T) { - raw := []any{ - map[string]any{"service": "*", "method": "read"}, - map[string]any{"service": "arborist", "method": "create-descendant"}, - } - if !snapshotAllows(raw, "read", "*") { - t.Fatalf("expected wildcard read to match") - } - if !snapshotAllows(raw, "create-descendant", "arborist") { - t.Fatalf("expected arborist create-descendant to match") - } - if snapshotAllows(raw, "update", "*") { - t.Fatalf("did not expect update to match") - } -} - func TestFenceUserEndpoint(t *testing.T) { token := "Bearer eyJhbGciOiJub25lIn0.eyJpc3MiOiJodHRwczovL2V4YW1wbGUub3JnL3VzZXIifQ." endpoint, err := fenceUserEndpoint(token) diff --git a/internal/server/middleware/git_test.go b/internal/server/middleware/git_test.go index bfcae95..405aae8 100644 --- a/internal/server/middleware/git_test.go +++ b/internal/server/middleware/git_test.go @@ -158,6 +158,114 @@ func TestRequireAuthorizationAllowsBearerHeader(t *testing.T) { } } +func TestGitProjectMutationAuthAllowsExactProjectResource(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/edit-connect", GitProjectMutationAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/org-a/projects/proj-a"}}, "update"), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("POST", "/git/projects/org-a/proj-a/edit-connect", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestGitProjectMutationAuthRejectsDifferentProjectResource(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/edit-connect", GitProjectMutationAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/org-a/projects/other"}}, "update"), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("POST", "/git/projects/org-a/proj-a/edit-connect", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusForbidden { + t.Fatalf("expected 403, got %d", resp.StatusCode) + } +} + +func TestGitProjectMutationAuthAllowsAdminWildcardResource(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Post("/git/projects/:orgTitle/:projectTitle/edit-connect", GitProjectMutationAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"*"}}, "update"), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("POST", "/git/projects/org-a/proj-a/edit-connect", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestGitProjectSetupAuthAllowsOrgProjectCreate(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Put("/git/projects/:orgTitle/:projectTitle/setup", GitProjectSetupAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/org-a/projects"}}), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("PUT", "/git/projects/org-a/proj-a/setup", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestGitProjectSetupAuthAllowsOrgManageOwners(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Put("/git/projects/:orgTitle/:projectTitle/setup", GitProjectSetupAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/org-a"}}), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("PUT", "/git/projects/org-a/proj-a/setup", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestGitProjectSetupAuthRejectsMissingProjectAndOrgAccess(t *testing.T) { + logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} + app := fiber.New() + app.Put("/git/projects/:orgTitle/:projectTitle/setup", GitProjectSetupAuth(logger, fakeJWTAllowedResourceHandler{resources: []any{"/programs/other"}}), func(ctx fiber.Ctx) error { + return ctx.SendStatus(fiber.StatusOK) + }) + + req := httptest.NewRequest("PUT", "/git/projects/org-a/proj-a/setup", nil) + req.Header.Set("Authorization", "Bearer test") + resp, err := app.Test(req) + if err != nil { + t.Fatalf("unexpected app.Test error: %v", err) + } + if resp.StatusCode != fiber.StatusForbidden { + t.Fatalf("expected 403, got %d", resp.StatusCode) + } +} + func TestParseResourceAccessSnapshotPrefersAuthzBlock(t *testing.T) { snapshot, err := parseResourceAccessSnapshot(map[string]any{ "authz": map[string]any{ @@ -285,19 +393,6 @@ func TestProjectConfigAuthAllowsAdminWildcardResource(t *testing.T) { } } -func TestSnapshotAllowsAcceptsWildcardMethod(t *testing.T) { - raw := []any{ - map[string]any{ - "method": "*", - "service": "*", - }, - } - - if !snapshotAllows(raw, "delete", "*") { - t.Fatalf("expected wildcard method/service snapshot entry to allow delete") - } -} - func TestGitProjectAuthForbiddenIncludesRequestAccessDetails(t *testing.T) { logger := &geckologging.Handler{Logger: log.New(io.Discard, "", 0)} app := fiber.New() diff --git a/internal/server/middleware/middleware.go b/internal/server/middleware/middleware.go index 75cd8c8..3e659c6 100644 --- a/internal/server/middleware/middleware.go +++ b/internal/server/middleware/middleware.go @@ -7,8 +7,10 @@ import ( "io" "net/http" "strings" + "time" "github.com/calypr/gecko/apierror" + "github.com/calypr/gecko/internal/httpclient" "github.com/calypr/gecko/internal/httputil" "github.com/golang-jwt/jwt/v5" ) @@ -35,7 +37,7 @@ type FenceUserAccessHandler struct { func NewFenceUserAccessHandler(client *http.Client) *FenceUserAccessHandler { if client == nil { - client = http.DefaultClient + client = httpclient.NewServiceClient(30 * time.Second) } return &FenceUserAccessHandler{client: client} } @@ -145,28 +147,6 @@ func parseResourceAccessSnapshot(payload map[string]any) (ResourceAccessSnapshot return snapshot, nil } -func snapshotAllows(raw any, method, service string) bool { - entries, ok := raw.([]any) - if !ok { - return false - } - for _, entry := range entries { - record, ok := entry.(map[string]any) - if !ok { - continue - } - entryMethod, _ := record["method"].(string) - entryService, _ := record["service"].(string) - if entryMethod != method && entryMethod != "*" { - continue - } - if entryService == "*" || service == "*" || entryService == service { - return true - } - } - return false -} - func ResourceAccessAllows(snapshot ResourceAccessSnapshot, resourcePath, method, service string) bool { entries := snapshot[resourcePath] for _, entry := range entries { diff --git a/internal/server/server.go b/internal/server/server.go index df8a45e..8375806 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -9,6 +9,7 @@ import ( "github.com/bmeg/grip/gripql" "github.com/calypr/gecko/internal/git" geckologging "github.com/calypr/gecko/internal/logging" + "github.com/calypr/gecko/internal/presentation" httpapi "github.com/calypr/gecko/internal/server/http" servermw "github.com/calypr/gecko/internal/server/middleware" "github.com/calypr/gecko/internal/thumbnail" @@ -19,15 +20,16 @@ import ( ) type Server struct { - db *sqlx.DB - jwtApp arborist.JWTDecoder - Logger *geckologging.Handler - stmts *arborist.CachedStmts - qdrantClient *qdrant.Client - gripqlClient *gripql.Client - gripGraphName string - gitService *git.GitService - thumbnailStore thumbnail.Manager + db *sqlx.DB + jwtApp arborist.JWTDecoder + Logger *geckologging.Handler + stmts *arborist.CachedStmts + qdrantClient *qdrant.Client + gripqlClient *gripql.Client + gripGraphName string + gitService *git.GitService + thumbnailStore thumbnail.Manager + presentationStore presentation.Manager } func NewServer() *Server { return &Server{} } @@ -69,6 +71,11 @@ func (server *Server) WithThumbnailStore(store thumbnail.Manager) *Server { return server } +func (server *Server) WithPresentationStore(store presentation.Manager) *Server { + server.presentationStore = store + return server +} + func (server *Server) Init() (*Server, error) { if server.jwtApp == nil { return nil, errors.New("gecko server initialized without JWT app") @@ -99,8 +106,8 @@ func (server *Server) Init() (*Server, error) { func (server *Server) MakeRouter() *fiber.App { app := fiber.New(fiber.Config{ ReadBufferSize: 32 * 1024, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, }) app.Use(func(ctx fiber.Ctx) error { @@ -116,14 +123,15 @@ func (server *Server) MakeRouter() *fiber.App { app.Use(servermw.RequestLogger(server.Logger)) httpapi.Register(app, httpapi.Dependencies{ - DB: server.db, - Logger: server.Logger, - JWTApp: server.jwtApp, - QdrantClient: server.qdrantClient, - GripqlClient: server.gripqlClient, - GripGraphName: server.gripGraphName, - GitService: server.gitService, - ThumbnailStore: server.thumbnailStore, + DB: server.db, + Logger: server.Logger, + JWTApp: server.jwtApp, + QdrantClient: server.qdrantClient, + GripqlClient: server.gripqlClient, + GripGraphName: server.gripGraphName, + GitService: server.gitService, + ThumbnailStore: server.thumbnailStore, + PresentationStore: server.presentationStore, }) return app } diff --git a/internal/storageaudit/evidence.go b/internal/storageaudit/evidence.go new file mode 100644 index 0000000..6a736dd --- /dev/null +++ b/internal/storageaudit/evidence.go @@ -0,0 +1,81 @@ +package storageaudit + +import "strings" + +type EvidenceStatus string + +const ( + EvidenceVerified EvidenceStatus = "verified" + EvidenceUnknown EvidenceStatus = "unknown" +) + +const ( + OperationInventory = "inventory" + OperationList = "list" + OperationMetadata = "metadata" + OperationDownload = "download" +) + +type Probe struct { + Operation string + Status string + ErrorKind string + ValidationStatus string +} + +type Assessment struct { + Status EvidenceStatus + Present bool + Missing bool + MappingBroken bool + MetadataMismatch bool + HasInventoryMiss bool + HasExactEvidence bool +} + +func Assess(probes []Probe, bucketObserved bool) Assessment { + assessment := Assessment{Status: EvidenceUnknown} + if bucketObserved { + assessment.Present = true + assessment.Status = EvidenceVerified + } + for _, probe := range probes { + operation := strings.TrimSpace(probe.Operation) + status := strings.TrimSpace(probe.Status) + errorKind := strings.TrimSpace(probe.ErrorKind) + validation := strings.TrimSpace(probe.ValidationStatus) + + if operation == OperationInventory && (status == "unknown" || errorKind == "inventory_miss") { + assessment.HasInventoryMiss = true + continue + } + isExact := operation == OperationList || operation == OperationMetadata + if isExact { + assessment.HasExactEvidence = true + } + if status == "present" { + assessment.Present = true + assessment.Status = EvidenceVerified + } + // S3-compatible backends can report a false miss for LIST or HEAD while a + // download-compatible GET still succeeds. Only an explicit GET-style probe + // may prove absence and unlock destructive repair. + if operation == OperationDownload && status == "not_found" && (errorKind == "" || errorKind == "object_not_found") { + assessment.Missing = true + assessment.Status = EvidenceVerified + } + switch errorKind { + case "missing_access_url", "scope_not_found", "credential_missing": + assessment.MappingBroken = true + assessment.Status = EvidenceVerified + } + if validation == "mismatched" { + assessment.MetadataMismatch = true + assessment.Status = EvidenceVerified + } + } + if assessment.Present { + assessment.Missing = false + } + return assessment +} diff --git a/internal/storageaudit/evidence_test.go b/internal/storageaudit/evidence_test.go new file mode 100644 index 0000000..8b13246 --- /dev/null +++ b/internal/storageaudit/evidence_test.go @@ -0,0 +1,76 @@ +package storageaudit + +import "testing" + +func TestAssessTreatsInventoryMissAsUnknown(t *testing.T) { + assessment := Assess([]Probe{{ + Operation: OperationInventory, + Status: "unknown", + ErrorKind: "inventory_miss", + ValidationStatus: "unverifiable", + }}, false) + if assessment.Status != EvidenceUnknown || assessment.Missing || !assessment.HasInventoryMiss { + t.Fatalf("expected an unverified inventory candidate, got %+v", assessment) + } +} + +func TestAssessTreatsExactListMissAsUnconfirmed(t *testing.T) { + assessment := Assess([]Probe{{ + Operation: OperationList, + Status: "not_found", + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }}, false) + if assessment.Status != EvidenceUnknown || assessment.Missing || !assessment.HasExactEvidence { + t.Fatalf("expected exact LIST absence to require metadata confirmation, got %+v", assessment) + } +} + +func TestAssessTreatsMetadataMissAsUnconfirmed(t *testing.T) { + assessment := Assess([]Probe{{ + Operation: OperationMetadata, + Status: "not_found", + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }}, false) + if assessment.Status != EvidenceUnknown || assessment.Missing || !assessment.HasExactEvidence { + t.Fatalf("expected metadata absence to require a download-compatible probe, got %+v", assessment) + } +} + +func TestAssessTreatsDownloadMissAsVerified(t *testing.T) { + assessment := Assess([]Probe{{ + Operation: OperationDownload, + Status: "not_found", + ErrorKind: "object_not_found", + ValidationStatus: "unverifiable", + }}, false) + if assessment.Status != EvidenceVerified || !assessment.Missing { + t.Fatalf("expected a download miss to verify absence, got %+v", assessment) + } +} + +func TestAssessLetsPresentLocatorWinOverMissingAlias(t *testing.T) { + assessment := Assess([]Probe{ + {Operation: OperationList, Status: "not_found", ErrorKind: "object_not_found"}, + {Operation: OperationList, Status: "present", ValidationStatus: "matched"}, + }, false) + if assessment.Status != EvidenceVerified || !assessment.Present || assessment.Missing { + t.Fatalf("expected any present canonical locator to connect the record, got %+v", assessment) + } +} + +func TestAutomaticRepairRequiresVerifiedMissingEvidence(t *testing.T) { + if !AllowsAutomaticRepair("bucket_only_object", EvidenceVerified) { + t.Fatal("verified bucket-only inventory evidence should allow bucket deletion") + } + if AllowsAutomaticRepair("syfon_git_no_bucket", EvidenceUnknown) { + t.Fatal("unknown evidence must not allow automatic repair") + } + if !AllowsAutomaticRepair("syfon_git_no_bucket", EvidenceVerified) { + t.Fatal("verified missing evidence should allow the missing-record repair") + } + if AllowsAutomaticRepair("bucket_syfon_no_git", EvidenceVerified) { + t.Fatal("absence from Git is informational and must not auto-delete storage") + } +} diff --git a/internal/storageaudit/policy.go b/internal/storageaudit/policy.go new file mode 100644 index 0000000..6a08d76 --- /dev/null +++ b/internal/storageaudit/policy.go @@ -0,0 +1,15 @@ +package storageaudit + +import "strings" + +func AllowsAutomaticRepair(kind string, evidenceStatus EvidenceStatus) bool { + if evidenceStatus != EvidenceVerified { + return false + } + switch strings.TrimSpace(kind) { + case "bucket_only_object", "storage_object_missing", "syfon_git_no_bucket", "syfon_missing_bucket_object": + return true + default: + return false + } +} diff --git a/internal/storageaudit/types.go b/internal/storageaudit/types.go new file mode 100644 index 0000000..f86c81f --- /dev/null +++ b/internal/storageaudit/types.go @@ -0,0 +1,301 @@ +package storageaudit + +type Evidence struct { + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` + Buckets []string `json:"buckets,omitempty"` + Keys []string `json:"keys,omitempty"` + StorageOperations []string `json:"storage_operations,omitempty"` + ProbeStatuses []string `json:"probe_statuses,omitempty"` + ValidationStates []string `json:"validation_states,omitempty"` + ErrorKinds []string `json:"error_kinds,omitempty"` + Errors []string `json:"errors,omitempty"` + BucketEvaluation string `json:"bucket_evaluation,omitempty"` +} + +type ProjectDiffAuditRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` +} + +type ProjectDiffFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids"` + RecordCount int `json:"record_count"` + SizeBytes int64 `json:"size_bytes,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` + RecommendedAction string `json:"recommended_action"` + Evidence *Evidence `json:"evidence,omitempty"` +} + +type ProjectDiffSummary struct { + CountsByKind map[string]int `json:"counts_by_kind"` + TotalFindings int `json:"total_findings"` + IndexedPathCount int `json:"indexed_path_count"` + ExpectedPathCount int `json:"expected_path_count"` + MatchedPathCount int `json:"matched_path_count"` + IncludesRepoManifest bool `json:"includes_repo_manifest"` + ScannedRecordCount int `json:"scanned_record_count"` +} + +type ProjectDiffAuditResponse struct { + Findings []ProjectDiffFinding `json:"findings"` + Summary ProjectDiffSummary `json:"summary"` + PathPrefix string `json:"path_prefix"` +} + +type CleanupAuditRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + CheckStorage bool `json:"check_storage,omitempty"` + SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` +} + +type ChainAuditRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + Refresh bool `json:"refresh,omitempty"` + ProbeMode string `json:"probe_mode,omitempty"` + ValidationMode string `json:"validation_mode,omitempty"` + BucketInventoryMode string `json:"bucket_inventory_mode,omitempty"` + BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` + FindingKind string `json:"finding_kind,omitempty"` + FindingLimit int `json:"finding_limit,omitempty"` + ForceAuditRefresh bool `json:"force_audit_refresh,omitempty"` + ForceBucketInventoryRefresh bool `json:"force_bucket_inventory_refresh,omitempty"` +} + +type ChainFinding struct { + Kind string `json:"kind"` + EvidenceStatus string `json:"evidence_status"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + SourcePaths []string `json:"source_paths,omitempty"` + ObjectIDs []string `json:"object_ids"` + Records []CleanupRecordAudit `json:"records,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + BucketObjectURL string `json:"bucket_object_url,omitempty"` + ResolvedBucket string `json:"resolved_bucket,omitempty"` + ResolvedKey string `json:"resolved_key,omitempty"` + ProbeStatus string `json:"probe_status,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + Error string `json:"error,omitempty"` + RecordCount int `json:"record_count"` + SizeBytes int64 `json:"size_bytes,omitempty"` + BucketSizeBytes int64 `json:"bucket_size_bytes,omitempty"` + RecommendedAction string `json:"recommended_action"` + SuggestedFix string `json:"suggested_fix,omitempty"` + SuggestedAction string `json:"suggested_action,omitempty"` + Actionability string `json:"actionability,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + SupportsDryRun bool `json:"supports_dry_run,omitempty"` + Evidence *Evidence `json:"evidence,omitempty"` +} + +// GitOnlySyfonRegistrationRequest names Git LFS paths that the caller wants +// registered in Syfon after Gecko has revalidated their scoped bucket objects. +type GitOnlySyfonRegistrationRequest struct { + Ref string `json:"ref,omitempty"` + ExpectedGitRevision string `json:"expected_git_revision"` + RepoPaths []string `json:"repo_paths"` +} + +type GitOnlySyfonRegistrationResult struct { + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + GitSizeBytes int64 `json:"git_size_bytes,omitempty"` + BucketObjectURL string `json:"bucket_object_url,omitempty"` + BucketSizeBytes int64 `json:"bucket_size_bytes,omitempty"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + ObjectID string `json:"object_id,omitempty"` +} + +type GitOnlySyfonRegistrationResponse struct { + GitRevision string `json:"git_revision"` + Results []GitOnlySyfonRegistrationResult `json:"results"` +} + +type ChainSummary struct { + CountsByKind map[string]int `json:"counts_by_kind"` + TotalFindings int `json:"total_findings"` + ReturnedFindings int `json:"returned_findings"` + FindingsTruncated bool `json:"findings_truncated"` + FindingLimit int `json:"finding_limit,omitempty"` + ValidationMode string `json:"validation_mode,omitempty"` + GitRevision string `json:"git_revision,omitempty"` + SyfonRevision string `json:"syfon_revision,omitempty"` + ObservedAt string `json:"observed_at,omitempty"` + BucketObjectCount int `json:"bucket_object_count"` + SyfonRecordCount int `json:"syfon_record_count"` + GitTrackedFileCount int `json:"git_tracked_file_count"` + BucketPathExists *bool `json:"bucket_path_exists,omitempty"` + BucketPathObjectURL string `json:"bucket_path_object_url,omitempty"` + BucketSummaryMode string `json:"bucket_summary_mode,omitempty"` + BucketInventoryAvailable bool `json:"bucket_inventory_available"` + BucketInventoryError string `json:"bucket_inventory_error,omitempty"` + AuditCacheHit bool `json:"audit_cache_hit,omitempty"` + AuditCachedAt string `json:"audit_cached_at,omitempty"` + AuditCacheAgeSeconds int64 `json:"audit_cache_age_seconds,omitempty"` + AuditRefreshDurationMs int64 `json:"audit_refresh_duration_ms,omitempty"` + AuditCacheSource string `json:"audit_cache_source,omitempty"` + AuditCacheError string `json:"audit_cache_error,omitempty"` +} + +type ChainIssueGroup struct { + Kind string `json:"kind"` + FindingCount int `json:"finding_count"` + PathCount int `json:"path_count"` + RecordCount int `json:"record_count"` + ObjectCount int `json:"object_count"` + TotalBytes int64 `json:"total_bytes,omitempty"` +} + +type ChainAuditResponse struct { + Findings []ChainFinding `json:"findings"` + Groups []ChainIssueGroup `json:"groups,omitempty"` + Summary ChainSummary `json:"summary"` + PathPrefix string `json:"path_prefix"` + BucketPathPrefix string `json:"bucket_path_prefix,omitempty"` +} + +type CleanupAccessProbe struct { + URL string `json:"url"` + Operation string `json:"operation,omitempty"` + Provider string `json:"provider,omitempty"` + Bucket string `json:"bucket,omitempty"` + Key string `json:"key,omitempty"` + Path string `json:"path,omitempty"` + Exists *bool `json:"exists,omitempty"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + MetaSHA256 string `json:"meta_sha256,omitempty"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` + ValidationStatus string `json:"validation_status,omitempty"` + SizeMatch *bool `json:"size_match,omitempty"` + NameMatch *bool `json:"name_match,omitempty"` + SHA256Match *bool `json:"sha256_match,omitempty"` + ValidationMismatches []string `json:"validation_mismatches,omitempty"` +} + +type CleanupAccessMethod struct { + AccessID string `json:"access_id,omitempty"` + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Headers []string `json:"headers,omitempty"` +} + +type CleanupRecordAudit struct { + ObjectID string `json:"object_id"` + Checksum string `json:"checksum,omitempty"` + NormalizedPath string `json:"normalized_path,omitempty"` + CleanupScope string `json:"cleanup_scope"` + AccessURLs []string `json:"access_urls,omitempty"` + AccessMethods []CleanupAccessMethod `json:"access_methods,omitempty"` + AccessProbes []CleanupAccessProbe `json:"access_probes"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` + SizeBytes int64 `json:"size,omitempty"` + LastUpdated string `json:"updated_time,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` +} + +type CleanupFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + Checksum string `json:"checksum,omitempty"` + ObjectIDs []string `json:"object_ids"` + Records []CleanupRecordAudit `json:"records"` + RecommendedAction string `json:"recommended_action"` + RepoDeleteCandidate bool `json:"repo_delete_candidate"` + CleanupScope string `json:"cleanup_scope"` + SizeBytes int64 `json:"total_bytes,omitempty"` + LastUpdated string `json:"last_updated,omitempty"` + DownloadCount int64 `json:"download_count,omitempty"` + LastDownload string `json:"last_download_time,omitempty"` + Actionability string `json:"actionability,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + SupportsDryRun bool `json:"supports_dry_run,omitempty"` + Evidence *Evidence `json:"evidence,omitempty"` +} + +type CleanupApplyAction struct { + Action string `json:"action"` + Kind string `json:"kind,omitempty"` + NormalizedPath string `json:"normalized_path,omitempty"` +} + +type CleanupApplyFinding struct { + Kind string `json:"kind"` + NormalizedPath string `json:"normalized_path"` + ObjectIDs []string `json:"object_ids,omitempty"` + Records []CleanupRecordAudit `json:"records,omitempty"` + BucketObjectURL string `json:"bucket_object_url,omitempty"` + BucketObjectURLs []string `json:"bucket_object_urls,omitempty"` + AccessURLs []string `json:"access_urls,omitempty"` + AvailableActions []string `json:"available_actions,omitempty"` + DefaultAction string `json:"default_action,omitempty"` + SuggestedAction string `json:"suggested_action,omitempty"` + Evidence *Evidence `json:"evidence,omitempty"` +} + +type CleanupAuditSummary struct { + CountsByKind map[string]int `json:"counts_by_kind"` + TotalFindings int `json:"total_findings"` + ManualFindingCount int `json:"manual_finding_count"` + RepoDeleteCandidateCount int `json:"repo_delete_candidate_count"` + StaleDuplicateCount int `json:"stale_duplicate_count"` + RepoOrphanCount int `json:"repo_orphan_count"` +} + +type CleanupAuditResponse struct { + Findings []CleanupFinding `json:"findings"` + Summary CleanupAuditSummary `json:"summary"` + ExpectedPathCount int `json:"expected_path_count"` + IncludesRepoManifest bool `json:"includes_repo_manifest"` + PathPrefix string `json:"path_prefix"` +} + +type CleanupApplyRequest struct { + GitSubpath string `json:"git_subpath,omitempty"` + Ref string `json:"ref,omitempty"` + DeleteRepoOrphans bool `json:"delete_repo_orphans,omitempty"` + DeleteStaleDuplicates bool `json:"delete_stale_duplicates,omitempty"` + DeleteBucketOnlyObjects bool `json:"delete_bucket_only_objects,omitempty"` + RepairBrokenBucketMappings bool `json:"repair_broken_bucket_mappings,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + SelectedRepoPaths []string `json:"selected_repo_paths,omitempty"` + Actions []CleanupApplyAction `json:"actions,omitempty"` + Findings []CleanupApplyFinding `json:"findings,omitempty"` +} + +type CleanupPurgeResult struct { + ObjectID string `json:"object_id"` + Success *bool `json:"success"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` +} + +type CleanupApplyResponse struct { + DeletedRecordIDs []string `json:"deleted_record_ids"` + DeletedBucketObjectURLs []string `json:"deleted_bucket_object_urls"` + UpdatedRecordIDs []string `json:"updated_record_ids"` + PurgeResults []CleanupPurgeResult `json:"purge_results"` + RepoDeletePaths []string `json:"repo_delete_paths"` + ManualPaths []string `json:"manual_paths"` + SkippedPaths []string `json:"skipped_paths"` + DryRun bool `json:"dry_run"` +} diff --git a/main.go b/main.go index 1237b71..bdf9c14 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( "github.com/calypr/gecko/internal/git" integrationfence "github.com/calypr/gecko/internal/integrations/fence" integrationgithub "github.com/calypr/gecko/internal/integrations/github" + "github.com/calypr/gecko/internal/presentation" server "github.com/calypr/gecko/internal/server" "github.com/calypr/gecko/internal/thumbnail" ) @@ -86,6 +87,7 @@ func main() { }) serverBuilder = serverBuilder.WithGitService(gitService) serverBuilder = serverBuilder.WithThumbnailStore(thumbnail.NewFilesystemStore(gitDataDir)) + serverBuilder = serverBuilder.WithPresentationStore(presentation.NewFilesystemStore(gitDataDir)) } if qdrantHost != "" && qdrantPort != 0 { diff --git a/tests/integration/middleware_test.go b/tests/integration/middleware_test.go index e5a4dfc..092aee5 100644 --- a/tests/integration/middleware_test.go +++ b/tests/integration/middleware_test.go @@ -122,17 +122,6 @@ func TestGeneralAuthMware_ConvertAnyToStringSliceError(t *testing.T) { assert.Contains(t, body, "Element 123 is not a string") } -func TestGeneralAuthMware_ParseAccessDenied(t *testing.T) { - srv := setupServer() - app := fiber.New() - app.Get("/:projectId", servermw.GeneralAuth(srv.Logger, &MockJWTHandler{AllowedResources: []string{"/programs/other/projects/test"}}, "read", "*"), func(c fiber.Ctx) error { return c.SendStatus(http.StatusOK) }) - req := httptest.NewRequest(http.MethodGet, "/ohsu-test", nil) - req.Header.Set("Authorization", "Bearer dummy") - resp, body := runFiber(app, req, t) - assert.Equal(t, http.StatusForbidden, resp.StatusCode) - assert.Contains(t, body, "User is not allowed to read on resource path") -} - func TestConfigAuthMiddleware_MethodNotAllowed(t *testing.T) { srv := setupServer() app := fiber.New() @@ -154,26 +143,6 @@ func TestConfigAuthMiddleware_Nav_PublicGET(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) } -func TestBaseConfigsAuthMiddleware_NoAuthorization(t *testing.T) { - srv := setupServer() - app := fiber.New() - app.Get("/", servermw.BaseConfigsAuth(srv.Logger, &MockJWTHandler{}, "read", "*", "/programs"), func(c fiber.Ctx) error { return c.SendStatus(http.StatusOK) }) - resp, body := runFiber(app, httptest.NewRequest(http.MethodGet, "/", nil), t) - assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) - assert.Contains(t, body, "Authorization token not provided") -} - -func TestBaseConfigsAuthMiddleware_InvalidJWTHandler(t *testing.T) { - srv := setupServer() - app := fiber.New() - app.Get("/", servermw.BaseConfigsAuth(srv.Logger, &MockJWTHandler{}, "read", "*", "/programs"), func(c fiber.Ctx) error { return c.SendStatus(http.StatusOK) }) - req := httptest.NewRequest(http.MethodGet, "/", nil) - req.Header.Set("Authorization", "Bearer dummy") - resp, body := runFiber(app, req, t) - assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) - assert.Contains(t, body, "Invalid JWT handler configuration") -} - func TestConfigAuthMiddleware_Project_PublicGET(t *testing.T) { srv := setupServer() app := fiber.New()